app/vmui: fix last 6 months time range picker

Previously it was incorrectly selecting 6 minutes instead of 6 months.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1960

.dockerignore

@@ -4,3 +4,4 @@ gocache-for-docker
 victoria-metrics-data
 vmstorage-data
 vmselect-cache
+.vscode

.github/dependabot.yml

@@ -21,6 +21,6 @@ updates:
     schedule:
       interval: "daily"
   - package-ecosystem: "npm"
-    directory: "/app/vmui"
+    directory: "/app/vmui/packages/vmui"
     schedule:
       interval: "weekly"

.github/workflows/check-licenses.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/main.yml

@@ -16,15 +16,15 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.16
+          go-version: 1.17
         id: go
-      - name: Dependencies
-        run: |
-          go get -u golang.org/x/lint/golint
-          go get -u github.com/kisielk/errcheck
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.29.0
       - name: Code checkout
         uses: actions/checkout@master
+      - name: Dependencies
+        run: |
+          make install-golint
+          make install-errcheck
+          make install-golangci-lint
       - name: Build
         env:
           GO111MODULE: on

.gitignore

@@ -4,6 +4,7 @@
 *.pprof
 /bin
 .idea
+.vscode
 *.test
 *.swp
 /gocache-for-docker

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2021 VictoriaMetrics, Inc.
+   Copyright 2019-2022 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -24,6 +24,8 @@ all: \
 
 include app/*/Makefile
 include deployment/*/Makefile
+include snap/local/Makefile
+
 
 clean:
 	rm -rf bin/*
@@ -84,9 +86,12 @@ vmutils-windows-amd64: \
 	vmauth-windows-amd64 \
 	vmctl-windows-amd64
 
-release-snap:
-	snapcraft
-	snapcraft upload "victoriametrics_$(PKG_TAG)_multi.snap" --release beta,edge,candidate
+
+publish-release:
+	git checkout $(TAG) && $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
 
 release: \
 	release-victoria-metrics \
@@ -174,6 +179,7 @@ release-vmutils-windows-generic: \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
 
+
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -261,7 +267,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.42.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
 
 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd

README.md

@@ -13,46 +13,13 @@
 VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.
 
 VictoriaMetrics is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
-in [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), in [Snap packages](https://snapcraft.io/victoriametrics)
-and in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics follow [these instructions](#how-to-start-victoriametrics).
+[Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap packages](https://snapcraft.io/victoriametrics)
+and [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and follow [these instructions](#how-to-start-victoriametrics).
 Then read [Prometheus setup](#prometheus-setup) and [Grafana setup](#grafana-setup) docs.
 
 Cluster version of VictoriaMetrics is available [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
 
-[Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics.
-See [features available in enterprise package](https://victoriametrics.com/enterprise.html).
-Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
-
-
-## Case studies and talks
-
-Case studies:
-
-* [AbiosGaming](https://docs.victoriametrics.com/CaseStudies.html#abiosgaming)
-* [adidas](https://docs.victoriametrics.com/CaseStudies.html#adidas)
-* [Adsterra](https://docs.victoriametrics.com/CaseStudies.html#adsterra)
-* [ARNES](https://docs.victoriametrics.com/CaseStudies.html#arnes)
-* [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
-* [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
-* [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
-* [Dreamteam](https://docs.victoriametrics.com/CaseStudies.html#dreamteam)
-* [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
-* [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
-* [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
-* [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
-* [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
-* [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
-* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
-* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
-* [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia)
-* [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets)
-* [Synthesio](https://docs.victoriametrics.com/CaseStudies.html#synthesio)
-* [Wedos.com](https://docs.victoriametrics.com/CaseStudies.html#wedoscom)
-* [Wix.com](https://docs.victoriametrics.com/CaseStudies.html#wixcom)
-* [Zerodha](https://docs.victoriametrics.com/CaseStudies.html#zerodha)
-* [zhihu](https://docs.victoriametrics.com/CaseStudies.html#zhihu)
-
-See also [articles and slides about VictoriaMetrics from our users](https://docs.victoriametrics.com/Articles.html#third-party-articles-and-slides-about-victoriametrics)
+[Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics. See [features available in enterprise package](https://victoriametrics.com/products/enterprise/). Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 
 ## Prominent features
@@ -89,12 +56,43 @@ VictoriaMetrics has the following prominent features:
   * [Native binary format](#how-to-import-data-in-native-format).
 * It supports metrics' relabeling. See [these docs](#relabeling) for details.
 * It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
-* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/enterprise.html).
+* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/).
 * It has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 See also [various Articles about VictoriaMetrics](https://docs.victoriametrics.com/Articles.html).
 
 
+## Case studies and talks
+
+Case studies:
+
+* [AbiosGaming](https://docs.victoriametrics.com/CaseStudies.html#abiosgaming)
+* [adidas](https://docs.victoriametrics.com/CaseStudies.html#adidas)
+* [Adsterra](https://docs.victoriametrics.com/CaseStudies.html#adsterra)
+* [ARNES](https://docs.victoriametrics.com/CaseStudies.html#arnes)
+* [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
+* [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
+* [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
+* [Dreamteam](https://docs.victoriametrics.com/CaseStudies.html#dreamteam)
+* [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
+* [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
+* [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
+* [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
+* [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
+* [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
+* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
+* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
+* [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia)
+* [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets)
+* [Synthesio](https://docs.victoriametrics.com/CaseStudies.html#synthesio)
+* [Wedos.com](https://docs.victoriametrics.com/CaseStudies.html#wedoscom)
+* [Wix.com](https://docs.victoriametrics.com/CaseStudies.html#wixcom)
+* [Zerodha](https://docs.victoriametrics.com/CaseStudies.html#zerodha)
+* [zhihu](https://docs.victoriametrics.com/CaseStudies.html#zhihu)
+
+See also [articles and slides about VictoriaMetrics from our users](https://docs.victoriametrics.com/Articles.html#third-party-articles-and-slides-about-victoriametrics)
+
+
 ## Operation
 
 ## How to start VictoriaMetrics
@@ -418,9 +416,15 @@ The `/api/v1/export` endpoint should return the following response:
 Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read via the following APIs:
 
 * [Graphite API](#graphite-api-usage)
-* [Prometheus querying API](#prometheus-querying-api-usage). VictoriaMetrics supports `__graphite__` pseudo-label for selecting time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). For example, `{__graphite__="foo.*.bar"}` is equivalent to `{__name__=~"foo[.][^.]*[.]bar"}`, but it works faster and it is easier to use when migrating from Graphite to VictoriaMetrics.
+* [Prometheus querying API](#prometheus-querying-api-usage). See also [selecting Graphite metrics](#selecting-graphite-metrics).
 * [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml)
 
+## Selecting Graphite metrics
+
+VictoriaMetrics supports `__graphite__` pseudo-label for selecting time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). For example, `{__graphite__="foo.*.bar"}` is equivalent to `{__name__=~"foo[.][^.]*[.]bar"}`, but it works faster and it is easier to use when migrating from Graphite to VictoriaMetrics. See [docs for Graphite paths and wildcards](https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards). VictoriaMetrics also supports [label_graphite_group](https://docs.victoriametrics.com/MetricsQL.html#label_graphite_group) function for extracting the given groups from Graphite metric name.
+
+The `__graphite__` pseudo-label supports e.g. alternate regexp filters such as `(value1|...|valueN)`. They are transparently converted to `{value1,...,valueN}` syntax [used in Graphite](https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards). This allows using [multi-value template variables in Grafana](https://grafana.com/docs/grafana/latest/variables/formatting-multi-value-variables/) inside `__graphite__` pseudo-label. For example, Grafana expands `{__graphite__=~"foo.($bar).baz"}` into `{__graphite__=~"foo.(x|y).baz"}` if `$bar` template variable contains `x` and `y` values. In this case the query is automatically converted into `{__graphite__=~"foo.{x,y}.baz"}` before execution.
+
 ## How to send data from OpenTSDB-compatible agents
 
 VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
@@ -517,9 +521,10 @@ All the Prometheus querying API handlers can be prepended with `/prometheus` pre
 ### Prometheus querying API enhancements
 
 VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg, which can be used for enforcing additional label filters for queries. For example,
-`/api/v1/query_range?extra_label=user_id=123&query=<query>` would automatically add `{user_id="123"}` label filter to the given `<query>`. This functionality can be used
-for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query arg is automatically set by auth proxy sitting
-in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+`/api/v1/query_range?extra_label=user_id=123&extra_label=group_id=456&query=<query>` would automatically add `{user_id="123",group_id="456"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+
+VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example,
+`/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
 
 VictoriaMetrics accepts relative times in `time`, `start` and `end` query args additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt).
 For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
@@ -556,18 +561,16 @@ VictoriaMetrics supports the following Graphite APIs, which are needed for [Grap
 
 All the Graphite handlers can be pre-pended with `/graphite` prefix. For example, both `/graphite/metrics/find` and `/metrics/find` should work.
 
-VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg for all the Graphite APIs. This arg can be used for limiting the scope of time series
-visible to the given tenant. It is expected that the `extra_label` query arg is automatically set by auth proxy sitting in front of VictoriaMetrics.
+VictoriaMetrics accepts optional query args: `extra_label=<label_name>=<label_value>` and `extra_filters[]=series_selector` query args for all the Graphite APIs. These args can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query arg is automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+
 [Contact us](mailto:sales@victoriametrics.com) if you need assistance with such a proxy.
 
-VictoriaMetrics supports `__graphite__` pseudo-label for filtering time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html).
-For example, `{__graphite__="foo.*.bar"}` is equivalent to `{__name__=~"foo[.][^.]*[.]bar"}`, but it works faster
-and it is easier to use when migrating from Graphite to VictoriaMetrics.
+VictoriaMetrics supports `__graphite__` pseudo-label for filtering time series with Graphite-compatible filters in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). See [these docs](#selecting-graphite-metrics).
 
 
 ### Graphite Render API usage
 
-[VictoriaMetrics Enterprise](https://victoriametrics.com/enterprise.html) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
+[VictoriaMetrics Enterprise](https://victoriametrics.com/products/enterprise/) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
 at `/render` endpoint, which is used by [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
 When configuring Graphite datasource in Grafana, the `Storage-Step` http request header must be set to a step between Graphite data points stored in VictoriaMetrics. For example, `Storage-Step: 10s` would mean 10 seconds distance between Graphite datapoints stored in VictoriaMetrics.
 Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
@@ -613,6 +616,10 @@ Query history can be navigated by holding `Ctrl` (or `Cmd` on MacOS) and pressin
 
 When querying the [backfilled data](https://docs.victoriametrics.com/#backfilling), it may be useful disabling response cache by clicking `Enable cache` checkbox.
 
+VMUI automatically adjusts the interval between datapoints on the graph depending on the horizontal resolution and on the selected time range. The step value can be customized by clickhing `Override step value` checkbox.
+
+VMUI allows investigating correlations between two queries on the same graph. Just click `+Query` button, enter the second query in the newly appeared input field and press `Ctrl+Enter`. Results for both queries should be displayed simultaneously on the same graph. Every query has its own vertical scale, which is displayed on the left and the right side of the graph. Lines for the second query are dashed.
+
 See the [example VMUI at VictoriaMetrics playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/?g0.expr=100%20*%20sum(rate(process_cpu_seconds_total))%20by%20(job)&g0.range_input=1d).
 
 
@@ -625,7 +632,7 @@ to your needs or when testing bugfixes.
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make victoria-metrics` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics` binary and puts it into the `bin` folder.
 
@@ -641,7 +648,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make victoria-metrics-arm` or `make victoria-metrics-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-arm` or `victoria-metrics-arm64` binary respectively and puts it into the `bin` folder.
 
@@ -655,7 +662,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 `Pure Go` mode builds only Go code without [cgo](https://golang.org/cmd/cgo/) dependencies.
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make victoria-metrics-pure` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-pure` binary and puts it into the `bin` folder.
 
@@ -766,33 +773,10 @@ when new data is ingested into it.
 
 VictoriaMetrics provides the following handlers for exporting data:
 
-* `/api/v1/export/native` for exporting data in native binary format. This is the most efficient format for data export.
-  See [these docs](#how-to-export-data-in-native-format) for details.
 * `/api/v1/export` for exporing data in JSON line format. See [these docs](#how-to-export-data-in-json-line-format) for details.
 * `/api/v1/export/csv` for exporting data in CSV. See [these docs](#how-to-export-csv-data) for details.
-
-
-### How to export data in native format
-
-Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
-where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
-for metrics to export. Use `{__name__=~".*"}` selector for fetching all the time series.
-
-On large databases you may experience problems with limit on unique timeseries (default value is 300000). In this case you need to adjust `-search.maxUniqueTimeseries` parameter:
-
-```bash
-# count unique timeseries in database
-wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq '.data[0]'
-
-# relaunch victoriametrics with search.maxUniqueTimeseries more than value from previous command
-```
-
-Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
-unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
-
-The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
-The native export format may change in incompatible way between VictoriaMetrics releases, so the data exported from the release X
-can fail to be imported into VictoriaMetrics release Y.
+* `/api/v1/export/native` for exporting data in native binary format. This is the most efficient format for data export.
+  See [these docs](#how-to-export-data-in-native-format) for details.
 
 
 ### How to export data in JSON line format
@@ -813,7 +797,7 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val
 
 Optional `max_rows_per_line` arg may be added to the request for limiting the maximum number of rows exported per each JSON line.
 Optional `reduce_mem_usage=1` arg may be added to the request for reducing memory usage when exporting big number of time series.
-In this case the output may contain multiple lines with distinct samples for the same time series.
+In this case the output may contain multiple lines with samples for the same time series.
 
 Pass `Accept-Encoding: gzip` HTTP header in the request to `/api/v1/export` in order to reduce network bandwidth during exporing big amounts
 of time series data. This enables gzip compression for the exported data. Example for exporting gzipped data:
@@ -826,6 +810,9 @@ The maximum duration for each request to `/api/v1/export` is limited by `-search
 
 Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-data-in-json-line-format).
 
+The [deduplication](#deduplication) is applied to the data exported via `/api/v1/export` by default. The deduplication
+isn't applied if `reduce_mem_usage=1` query arg is passed to the request.
+
 
 ### How to export CSV data
 
@@ -850,6 +837,33 @@ unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) val
 
 The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
 
+The [deduplication](#deduplication) is applied for the data exported in CSV by default. It is possible to export raw data without de-duplication by passing `reduce_mem_usage=1` query arg to `/api/v1/export/csv`.
+
+
+### How to export data in native format
+
+Send a request to `http://<victoriametrics-addr>:8428/api/v1/export/native?match[]=<timeseries_selector_for_export>`,
+where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
+for metrics to export. Use `{__name__=~".*"}` selector for fetching all the time series.
+
+On large databases you may experience problems with limit on unique timeseries (default value is 300000). In this case you need to adjust `-search.maxUniqueTimeseries` parameter:
+
+```bash
+# count unique timeseries in database
+wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq '.data[0]'
+
+# relaunch victoriametrics with search.maxUniqueTimeseries more than value from previous command
+```
+
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
+unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+
+The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
+The native export format may change in incompatible way between VictoriaMetrics releases, so the data exported from the release X
+can fail to be imported into VictoriaMetrics release Y.
+
+The [deduplication](#deduplication) isn't applied for the data exported in native format. It is expected that the de-duplication is performed during data import.
+
 
 ## How to import time series data
 
@@ -869,36 +883,6 @@ Time series data can be imported via any supported ingestion protocol:
 * `/api/v1/import/prometheus` for importing data in Prometheus exposition format. See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
 
 
-### How to import data in native format
-
-The specification of VictoriaMetrics' native format may yet change and is not formally documented yet. So currently we do not recommend that external clients attempt to pack their own metrics in native format file.
-
-If you have a native format file obtained via [/api/v1/export/native](#how-to-export-data-in-native-format) however this is the most efficient protocol for importing data in.
-
-```bash
-# Export the data from <source-victoriametrics>:
-curl http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin
-
-# Import the data to <destination-victoriametrics>:
-curl -X POST http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin
-```
-
-Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import/native` for importing gzipped data:
-
-```bash
-# Export gzipped data from <source-victoriametrics>:
-curl -H 'Accept-Encoding: gzip' http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin.gz
-
-# Import gzipped data to <destination-victoriametrics>:
-curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin.gz
-```
-
-Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
-For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
-
-Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
-
-
 ### How to import data in JSON line format
 
 Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
@@ -929,6 +913,26 @@ Note that it could be required to flush response cache after importing historica
 VictoriaMetrics parses input JSON lines one-by-one. It loads the whole JSON line in memory, then parses it and then saves the parsed samples into persistent storage. This means that VictoriaMetrics can occupy big amounts of RAM when importing too long JSON lines. The solution is to split too long JSON lines into smaller lines. It is OK if samples for a single time series are split among multiple JSON lines.
 
 
+### How to import data in native format
+
+The specification of VictoriaMetrics' native format may yet change and is not formally documented yet. So currently we do not recommend that external clients attempt to pack their own metrics in native format file.
+
+If you have a native format file obtained via [/api/v1/export/native](#how-to-export-data-in-native-format) however this is the most efficient protocol for importing data in.
+
+```bash
+# Export the data from <source-victoriametrics>:
+curl http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin
+
+# Import the data to <destination-victoriametrics>:
+curl -X POST http://destination-victoriametrics:8428/api/v1/import/native -T exported_data.bin
+```
+
+Extra labels may be added to all the imported time series by passing `extra_label=name=value` query args.
+For example, `/api/v1/import/native?extra_label=foo=bar` would add `"foo":"bar"` label to all the imported time series.
+
+Note that it could be required to flush response cache after importing historical data. See [these docs](#backfilling) for detail.
+
+
 ### How to import CSV data
 
 Arbitrary CSV data can be imported via `/api/v1/import/csv`. The CSV data is imported according to the provided `format` query arg.
@@ -1004,6 +1008,13 @@ It should return something like the following:
 {"metric":{"__name__":"foo","bar":"baz"},"values":[123],"timestamps":[1594370496905]}
 ```
 
+Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import/prometheus` for importing gzipped data:
+
+```bash
+# Import gzipped data to <destination-victoriametrics>:
+curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import/prometheus -T prometheus_data.gz
+```
+
 Extra labels may be added to all the imported metrics by passing `extra_label=name=value` query args.
 For example, `/api/v1/import/prometheus?extra_label=foo=bar` would add `{foo="bar"}` label to all the imported metrics.
 
@@ -1022,6 +1033,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc
 
 VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points
 to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries.
+The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`.
 See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2).
 
 Example contents for `-relabelConfig` file:
@@ -1071,7 +1083,7 @@ It is recommended leaving the following amounts of spare resources:
 
 * 50% of free RAM for reducing the probability of OOM (out of memory) crashes and slowdowns during temporary spikes in workload.
 * 50% of spare CPU for reducing the probability of slowdowns during temporary spikes in workload.
-* At least 30% of free storage space at the directory pointed by `-storageDataPath` command-line flag.
+* At least 30% of free storage space at the directory pointed by `-storageDataPath` command-line flag. See also `-storage.minFreeDiskSpaceBytes` command-line flag description [here](#list-of-command-line-flags).
 
 
 ## High availability
@@ -1132,20 +1144,25 @@ write data to the same VictoriaMetrics instance. These vmagent or Prometheus ins
 
 Retention is configured with `-retentionPeriod` command-line flag. For instance, `-retentionPeriod=3` means
 that the data will be stored for 3 months and then deleted.
-Data is split in per-month subdirectories inside `<-storageDataPath>/data/small` and `<-storageDataPath>/data/big` folders.
-Directories for months outside the configured retention are deleted on the first day of new month.
+Data is split in per-month partitions inside `<-storageDataPath>/data/small` and `<-storageDataPath>/data/big` folders.
+Data partitions outside the configured retention are deleted on the first day of new month.
+
+Each partition consists of one or more data parts with the following name pattern `rowsCount_blocksCount_minTimestamp_maxTimestamp`.
+Data parts outside of the configured retention are eventually deleted during [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+
 In order to keep data according to `-retentionPeriod` max disk space usage is going to be `-retentionPeriod` + 1 month.
 For example if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
+
+VictoriaMetrics supports retention smaller than 1 month. For example, `-retentionPeriod=5d` would set data retention for 5 days.
+Please note, time range covered by data part is not limited by retention period unit. Hence, data part may contain data
+for multiple days and will be deleted only when fully outside of the configured retention.
+
 It is safe to extend `-retentionPeriod` on existing data. If `-retentionPeriod` is set to lower
 value than before then data outside the configured period will be eventually deleted.
 
-VictoriaMetrics supports retention smaller than 1 month. For example, `-retentionPeriod=5d` would set data retention for 5 days.
-Older data is eventually deleted during [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
-
-
 ## Multiple retentions
 
-Just start multiple VictoriaMetrics instances with distinct values for the following flags:
+A single instance of VictoriaMetrics supports only a single retention, which can be configured via `-retentionPeriod` command-line flag. If you need multiple retentions, then you may start multiple VictoriaMetrics instances with distinct values for the following flags:
 
 * `-retentionPeriod`
 * `-storageDataPath`, so the data for each retention period is saved in a separate directory
@@ -1154,24 +1171,20 @@ Just start multiple VictoriaMetrics instances with distinct values for the follo
 Then set up [vmauth](https://docs.victoriametrics.com/vmauth.html) in front of VictoriaMetrics instances,
 so it could route requests from particular user to VictoriaMetrics with the desired retention.
 The same scheme could be implemented for multiple tenants in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+See [these docs](https://docs.victoriametrics.com/guides/guide-vmcluster-multiple-retention-setup.html) for multi-retention setup details.
 
 
 ## Downsampling
 
-There is no downsampling support at the moment, but:
+[VictoriaMetrics Enterprise](https://victoriametrics.com/products/enterprise/) supports multi-level downsampling with `-downsampling.period` command-line flag. For example:
 
-* VictoriaMetrics is optimized for querying big amounts of raw data. See benchmark results for heavy queries
-  in [this article](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
-* VictoriaMetrics has good compression for on-disk data. See [this article](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
-  for details.
-* The downsampling doesn't improve query performance on a long time range if the time range contains big number of time series due to [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). The query performance depends on the number of unique time series on the selected time range, while downsampling doesn't reduce the number of unique time series in the database - it can reduce only the number of samples per each time series.
+* `-downsampling.period=30d:5m` instructs VictoriaMetrics to [deduplicate](#deduplication) samples older than 30 days with 5 minutes interval.
 
-These properties reduce the need of downsampling. We plan to implement downsampling in the future.
-See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/36) for details.
+* `-downsampling.period=30d:5m,180d:1h` instructs VictoriaMetrics to deduplicate samples older than 30 days with 5 minutes interval and to deduplicate samples older than 180 days with 1 hour interval.
 
-It is possible to (ab)use [-dedup.minScrapeInterval](#deduplication) for basic downsampling.
-For instance, if interval between the ingested data points is 15s, then `-dedup.minScrapeInterval=5m` will leave
-only a single data point out of 20 initial data points per each 5m interval.
+Downsampling is applied independently per each time series. It can reduce disk space usage and improve query performance if it is applied to time series with big number of samples per each series. The downsampling doesn't improve query performance if the database contains big number of time series with small number of samples per each series (aka [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate)), since downsampling doesn't reduce the number of time series. So the majority of time is spent on searching for the matching time series. It is possible to use recording rules in [vmalert](https://docs.victoriametrics.com/vmalert.html) in order to reduce the number of time series. See [these docs](https://docs.victoriametrics.com/vmalert.html#downsampling-and-aggregation-via-vmalert).
+
+The downsampling can be evaluated for free by downloading and using enterprise binaries from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 
 ## Multi-tenancy
@@ -1213,6 +1226,8 @@ Consider setting the following command-line flags:
 * `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
 * `-forceMergeAuthKey` for protecting `/internal/force_merge` endpoint. See [force merge docs](#forced-merge).
 * `-search.resetCacheAuthKey` for protecting `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
+* `-configAuthKey` for protecting `/config` endpoint, since it may contain sensitive information such as passwords.
+- `-pprofAuthKey` for protecting `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
 
 Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
 For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`.
@@ -1248,8 +1263,9 @@ or Prometheus by adding the corresponding scrape config to it.
 Alternatively they can be self-scraped by setting `-selfScrapeInterval` command-line flag to duration greater than 0.
 For example, `-selfScrapeInterval=10s` would enable self-scraping of `/metrics` page with 10 seconds interval.
 
-There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176).
-There is also an [alternative dashboard for clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11831).
+There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176). There is also an [alternative dashboard for clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11831).
+
+Graphs on these dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 
 It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.com/vmalert.html) or in Prometheus from [this config](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).
 
@@ -1366,8 +1382,7 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet
   This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
   metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
 
-* If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then use `{__graphite__="foo.*.baz"}` syntax for selecting such metrics.
-  This expression is equivalent to `{__name__=~"foo[.][^.]*[.]baz"}`, but it works faster and it is easier to use when migrating from Graphite.
+* If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then `{__graphite__="foo.*.baz"}` filter can be used for selecting such metrics. See [these docs](#selecting-graphite-metrics) for details.
 
 * VictoriaMetrics ignores `NaN` values during data ingestion.
 
@@ -1392,6 +1407,7 @@ See [vmctl docs](https://docs.victoriametrics.com/vmctl.html) for more details.
 ## Backfilling
 
 VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data).
+See [how to backfill data with recording rules in vmalert](https://docs.victoriametrics.com/vmalert.html#rules-backfilling).
 Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.
 
 It is recommended disabling query cache with `-search.disableCache` command-line flag when writing
@@ -1432,6 +1448,18 @@ We also provide [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanag
 Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 
+## Benchmarks
+
+Note, that vendors (including VictoriaMetrics) are often biased when doing such tests. E.g. they try highlighting 
+the best parts of their product, while highlighting the worst parts of competing products. 
+So we encourage users and all independent third parties to conduct their becnhmarks for various products 
+they are evaluating in production and publish the results.
+
+As a reference, please see [benchmarks](https://docs.victoriametrics.com/Articles.html#benchmarks) conducted by
+VictoriaMetrics team. Please also see the [helm chart](https://github.com/VictoriaMetrics/benchmark) 
+for running ingestion benchmarks based on node_exporter metrics.
+
+
 ## Profiling
 
 VictoriaMetrics provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
@@ -1485,9 +1513,11 @@ Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics
 Feel free asking any questions regarding VictoriaMetrics:
 
 * [slack](https://slack.victoriametrics.com/)
+* [linkedin](https://www.linkedin.com/company/victoriametrics/)
 * [reddit](https://www.reddit.com/r/VictoriaMetrics/)
 * [telegram-en](https://t.me/VictoriaMetrics_en)
 * [telegram-ru](https://t.me/VictoriaMetrics_ru1)
+* [articles and talks about VictoriaMetrics in Russian](https://github.com/denisgolius/victoriametrics-ru-links)
 * [google groups](https://groups.google.com/forum/#!forum/victorametrics-users)
 
 If you like VictoriaMetrics and want to contribute, then we need the following:
@@ -1514,7 +1544,7 @@ Report bugs and propose new features [here](https://github.com/VictoriaMetrics/V
 
 ## VictoriaMetrics Logo
 
-[Zip](VM_logo.zip) contains three folders with different image orientations (main color and inverted version).
+[Zip](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/VM_logo.zip) contains three folders with different image orientations (main color and inverted version).
 
 Files included in each folder:
 
@@ -1549,17 +1579,22 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
 ```
   -bigMergeConcurrency int
     	The maximum number of CPU cores to use for big merges. Default value is used if set to 0
+  -configAuthKey string
+    	Authorization key for accessing /config page. It must be passed via authKey query arg
   -csvTrimTimestamp duration
     	Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -datadog.maxInsertRequestSize size
     	The maximum size in bytes of a single DataDog POST request to /api/v1/series
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
   -dedup.minScrapeInterval duration
-    	Leave only the first sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details
+    	Leave only the first sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling
   -deleteAuthKey string
     	authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries
   -denyQueriesOutsideRetention
     	Whether to deny queries outside of the configured -retentionPeriod. When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
+  -downsampling.period array
+    	Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details
+    	Supports an array of values separated by comma or specified via multiple flags.
   -dryRun
     	Whether to check only -promscrape.config and then exit. Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse
   -enableTCP6
@@ -1568,6 +1603,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
+  -eula
+    	By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf
   -finalMergeDelay duration
     	The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge
   -forceFlushAuthKey string
@@ -1640,15 +1677,17 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -maxInsertRequestSize size
     	The maximum size in bytes of a single Prometheus remote_write API request
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 33554432)
+  -maxLabelValueLen int
+    	The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 16384)
   -maxLabelsPerTimeseries int
-    	The maximum number of labels accepted per time series. Superfluous labels are dropped (default 30)
+    	The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
   -memory.allowedBytes size
     	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
     	TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
   -opentsdbListenAddr string
@@ -1661,7 +1700,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -opentsdbhttpTrimTimestamp duration
     	Trim timestamps for OpenTSDB HTTP data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -precisionBits int
     	The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss (default 64)
   -promscrape.cluster.memberNum int
@@ -1671,7 +1710,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.cluster.replicationFactor int
     	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
   -promscrape.config string
-    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
   -promscrape.config.dryRun
     	Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
   -promscrape.config.strictParse
@@ -1716,6 +1755,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
   -promscrape.maxDroppedTargets int
     	The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
+  -promscrape.maxResponseHeadersSize size
+    	The maximum size of http response headers from Prometheus scrape targets
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096)
   -promscrape.maxScrapeSize size
     	The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
@@ -1735,7 +1777,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.suppressScrapeErrors
     	Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
   -relabelConfig string
-    	Optional path to a file with relabeling rules, which are applied to all the ingested metrics. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
+    	Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
   -relabelDebug
     	Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs
   -retentionPeriod value
@@ -1747,6 +1789,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	Whether to disable automatic response cache reset if a sample with timestamp outside -search.cacheTimestampOffset is inserted into VictoriaMetrics
   -search.disableCache
     	Whether to disable response caching. This may be useful during data backfilling
+  -search.graphiteMaxPointsPerSeries int
+    	The maximum number of points per series Graphite render API can return (default 1000000)
+  -search.graphiteStorageStep duration
+    	The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overriden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API (default 10s)
   -search.latencyOffset duration
     	The time when data points become visible in query results after the collection. Too small value can result in incomplete last points for query results (default 30s)
   -search.logSlowQueryDuration duration

app/victoria-metrics/main.go

@@ -25,7 +25,7 @@ import (
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details")
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
 		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
 )
@@ -51,7 +51,7 @@ func main() {
 
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
-	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
+	storage.SetDedupInterval(*minScrapeInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
@@ -90,15 +90,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/vmui", "Web UI"},
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/config", "-promscrape.config contents"},
-			{"/metrics", "available service metrics"},
-			{"/flags", "command-line flags"},
-			{"/api/v1/status/tsdb", "tsdb status page"},
-			{"/api/v1/status/top_queries", "top queries"},
-			{"/api/v1/status/active_queries", "active queries"},
+			{"vmui", "Web UI"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"api/v1/status/tsdb", "tsdb status page"},
+			{"api/v1/status/top_queries", "top queries"},
+			{"api/v1/status/active_queries", "active queries"},
 		})
 		return true
 	}

app/vmagent/README.md

@@ -46,7 +46,7 @@ to `vmagent` such as the ability to push metrics instead of pulling them. We did
 Please download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
 and configure the following flags to the `vmagent` binary in order to start scraping Prometheus targets:
 
-* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`)
+* `-promscrape.config` with the path to Prometheus config file (usually located at `/etc/prometheus/prometheus.yml`). The path can point either to local file or to http url.
 * `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified multiple times to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
@@ -214,15 +214,16 @@ The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders w
 
 ## Loading scrape configs from multiple files
 
-`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory plus a `single_scrape_config.yml` file:
+`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url:
 
 ```yml
 scrape_config_files:
 - configs/*.yml
 - single_scrape_config.yml
+- https://config-server/scrape_config.yml
 ```
 
-Every referred file can contain arbitrary number of any [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:
+Every referred file can contain arbitrary number of [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:
 
 ```yml
 - job_name: foo
@@ -279,7 +280,7 @@ The relabeling can be defined in the following places:
 
 * At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target.
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics.
-* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
+* At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
 
 You can read more about relabeling in the following articles:
@@ -300,7 +301,6 @@ You can read more about relabeling in the following articles:
 * If the metric disappears from the list of scraped metrics, then stale marker is sent to this particular metric.
 * If the scrape target becomes temporarily unavailable, then stale markers are sent for all the metrics scraped from this target.
 * If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
-* Stale markers are sent for all the scraped metrics on graceful shutdown of `vmagent`.
 
 Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series) for details.
 
@@ -428,7 +428,7 @@ These limits are approximate, so `vmagent` can underflow/overflow the limit by a
 
 `vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. We recommend setting up regular scraping of this page
 either through `vmagent` itself or by Prometheus so that the exported metrics may be analyzed later.
-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
+Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add a review to the dashboard.
 
 `vmagent` also exports the status for various targets at the following handlers:
@@ -452,10 +452,11 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 
 * If `vmagent` uses too big amounts of memory, then the following options can help:
   * Disabling staleness tracking with `-promscrape.noStaleMarkers` option. See [these docs](#prometheus-staleness-markers).
-  * Enabling stream parsing mode. See [these docs](#stream-parsing-mode).
+  * Enabling stream parsing mode if `vmagent` scrapes targets with millions of metrics per target. See [these docs](#stream-parsing-mode).
   * Reducing the number of output queues with `-remoteWrite.queues` command-line option.
   * Reducing the amounts of RAM vmagent can use for in-memory buffering with `-memory.allowedPercent` or `-memory.allowedBytes` command-line option. Another option is to reduce memory limits in Docker and/or Kuberntes if `vmagent` runs under these systems.
   * Reducing the number of CPU cores vmagent can use by passing `GOMAXPROCS=N` environment variable to `vmagent`, where `N` is the desired limit on CPU cores. Another option is to reduce CPU limits in Docker or Kubernetes if `vmagent` runs under these systems.
+  * Passing `-promscrape.dropOriginalLabels` command-line option to `vmagent`, so it drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page. This reduces memory usage when scraping big number of targets at the cost of reduced debuggability for improperly configured per-target relabeling.
 
 * When `vmagent` scrapes many unreliable targets, it can flood the error log with scrape errors. These errors can be suppressed
   by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`
@@ -464,17 +465,9 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 * The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
   This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default the `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM. Therefore big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if a big number of scrape targets are dropped during relabeling.
 
-* If `vmagent` scrapes a big number of targets then the `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
-  This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+* We recommend you increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page grows constantly. It is also recommended increasing `-remoteWrite.maxBlockSize` and `-remoteWrite.maxRowsPerBlock` command-line options in this case. This can improve data ingestion performance to the configured remote storage systems at the cost of higher memory usage.
 
-* If `vmagent` scrapes targets with millions of metrics per target (for example, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
-  we recommend enabling [stream parsing mode](#stream-parsing-mode) in order to reduce memory usage during scraping.
-
-* We recommend you increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page grows constantly.
-
-* If you see gaps in the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, try increasing `-remoteWrite.queues`.
-  Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage. Therefore it starts dropping the buffered data
-  if the on-disk buffer size exceeds `-remoteWrite.maxDiskUsagePerURL`.
+* If you see gaps in the data pushed by `vmagent` to remote storage when `-remoteWrite.maxDiskUsagePerURL` is set, try increasing `-remoteWrite.queues`. Such gaps may appear because `vmagent` cannot keep up with sending the collected data to remote storage. Therefore it starts dropping the buffered data if the on-disk buffer size exceeds `-remoteWrite.maxDiskUsagePerURL`.
 
 * `vmagent` drops data blocks if remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses. The number of dropped blocks can be monitored via `vmagent_remotewrite_packets_dropped_total` metric exported at [/metrics page](#monitoring).
 
@@ -527,7 +520,7 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 
 ## Kafka integration
 
-[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read and write metrics from / to Kafka:
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` can read and write metrics from / to Kafka:
 
 * [Reading metrics from Kafka](#reading-metrics-from-kafka)
 * [Writing metrics to Kafka](#writing-metrics-to-kafka)
@@ -537,7 +530,7 @@ The enterprise version of vmagent is available for evaluation at [releases](http
 
 ### Reading metrics from Kafka
 
-[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:
 
 * `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). Messages in this format can be sent by vmagent - see [these docs](#writing-metrics-to-kafka).
 * `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/).
@@ -573,7 +566,7 @@ data_format = "influx"
 
 #### Command-line flags for Kafka consumer
 
-These command-line flags are available only in [enterprise](https://victoriametrics.com/enterprise.html) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+These command-line flags are available only in [enterprise](https://victoriametrics.com/products/enterprise/) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
 
 ```
   -kafka.consumer.topic array
@@ -606,7 +599,7 @@ These command-line flags are available only in [enterprise](https://victoriametr
 
 ### Writing metrics to Kafka
 
-[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` writes data to Kafka with `at-least-once` semantics if `-remoteWrite.url` contains e.g. Kafka url. For example, if `vmagent` is started with `-remoteWrite.url=kafka://localhost:9092/?topic=prom-rw`, then it would send Prometheus remote_write messages to Kafka bootstrap server at `localhost:9092` with the topic `prom-rw`. These messages can be read later from Kafka by another `vmagent` - see [these docs](#reading-metrics-from-kafka) for details.
+[Enterprise version](https://victoriametrics.com/products/enterprise/) of `vmagent` writes data to Kafka with `at-least-once` semantics if `-remoteWrite.url` contains e.g. Kafka url. For example, if `vmagent` is started with `-remoteWrite.url=kafka://localhost:9092/?topic=prom-rw`, then it would send Prometheus remote_write messages to Kafka bootstrap server at `localhost:9092` with the topic `prom-rw`. These messages can be read later from Kafka by another `vmagent` - see [these docs](#reading-metrics-from-kafka) for details.
 
 Additional Kafka options can be passed as query params to `-remoteWrite.url`. For instance, `kafka://localhost:9092/?topic=prom-rw&client.id=my-favorite-id` sets `client.id` Kafka option to `my-favorite-id`. The full list of Kafka options is available [here](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
 
@@ -635,7 +628,7 @@ We recommend using [binary releases](https://github.com/VictoriaMetrics/Victoria
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmagent` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds the `vmagent` binary and puts it into the `bin` folder.
 
@@ -664,7 +657,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmagent-arm` or `make vmagent-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics)
    It builds `vmagent-arm` or `vmagent-arm64` binary respectively and puts it into the `bin` folder.
 
@@ -707,6 +700,8 @@ vmagent collects metrics data via popular data ingestion protocols and routes th
 
 See the docs at https://docs.victoriametrics.com/vmagent.html .
 
+  -configAuthKey string
+    	Authorization key for accessing /config page. It must be passed via authKey query arg
   -csvTrimTimestamp duration
     	Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -datadog.maxInsertRequestSize size
@@ -790,7 +785,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
     	TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
   -opentsdbListenAddr string
@@ -803,7 +798,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -opentsdbhttpTrimTimestamp duration
     	Trim timestamps for OpenTSDB HTTP data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -promscrape.cluster.memberNum int
     	The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster
   -promscrape.cluster.membersCount int
@@ -811,7 +806,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.cluster.replicationFactor int
     	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
   -promscrape.config string
-    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+    	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
   -promscrape.config.dryRun
     	Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
   -promscrape.config.strictParse
@@ -856,6 +851,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
   -promscrape.maxDroppedTargets int
     	The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
+  -promscrape.maxResponseHeadersSize size
+    	The maximum size of http response headers from Prometheus scrape targets
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096)
   -promscrape.maxScrapeSize size
     	The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
@@ -895,7 +893,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.maxBlockSize size
-    	The maximum size in bytes of unpacked request to send to remote storage. It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics
+    	The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
   -remoteWrite.maxDailySeries int
     	The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
@@ -904,6 +902,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -remoteWrite.maxHourlySeries int
     	The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
+  -remoteWrite.maxRowsPerBlock int
+    	The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 10000)
   -remoteWrite.multitenantURL array
     	Base path for multitenant remote storage URL to write data to. See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url
     	Supports an array of values separated by comma or specified via multiple flags.
@@ -931,7 +931,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
     	Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
-    	Optional path to file with relabel_config entries. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
+    	Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
   -remoteWrite.relabelDebug
     	Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs
   -remoteWrite.roundDigits array
@@ -966,7 +966,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
-    	Optional path to relabel config for the corresponding -remoteWrite.url
+    	Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelDebug array
     	Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs

app/vmagent/main.go

@@ -50,6 +50,7 @@ var (
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
 		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
@@ -157,12 +158,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/targets", "discovered targets list"},
-			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"/config", "-promscrape.config contents"},
-			{"/metrics", "available service metrics"},
-			{"/flags", "command-line flags"},
-			{"/-/reload", "reload configuration"},
+			{"targets", "discovered targets list"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"-/reload", "reload configuration"},
 		})
 		return true
 	}
@@ -235,26 +236,26 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{"valid":true}`)
 		return true
 	case "/datadog/api/v1/check_run":
 		datadogCheckRunRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/intake/":
 		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
 	case "/targets":
@@ -262,13 +263,21 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
 	case "/config":
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
 		promscrapeConfigRequests.Inc()
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
@@ -382,19 +391,19 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{"valid":true}`)
 		return true
 	case "datadog/api/v1/check_run":
 		datadogCheckRunRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "datadog/intake/":
 		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
 	default:

app/vmagent/remotewrite/client.go

@@ -105,11 +105,11 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		if !strings.Contains(pURL, "://") {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
 		}
-		urlProxy, err := url.Parse(pURL)
+		pu, err := url.Parse(pURL)
 		if err != nil {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
 		}
-		tr.Proxy = http.ProxyURL(urlProxy)
+		tr.Proxy = http.ProxyURL(pu)
 	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
@@ -165,7 +165,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if username != "" || password != "" || passwordFile != "" {
 		basicAuthCfg = &promauth.BasicAuthConfig{
 			Username:     username,
-			Password:     password,
+			Password:     promauth.NewSecret(password),
 			PasswordFile: passwordFile,
 		}
 	}
@@ -179,7 +179,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	if clientSecretFile != "" || clientSecret != "" {
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
-			ClientSecret:     clientSecret,
+			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
@@ -254,7 +254,7 @@ func (c *client) sendBlockHTTP(block []byte) bool {
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
@@ -295,6 +295,17 @@ again:
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
+		body, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		l := logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+		if err != nil {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+				"failed to read response body: %s",
+				len(block), c.sanitizedURL, statusCode, err)
+		} else {
+			l.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+				len(block), c.sanitizedURL, statusCode, string(body))
+		}
 		// Just drop block on 409 and 400 status codes like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149

app/vmagent/remotewrite/pendingseries.go

@@ -20,16 +20,10 @@ import (
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
-	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
-		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
+	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 )
 
-// the maximum number of rows to send per each block.
-const maxRowsPerBlock = 10000
-
-// the maximum number of labels to send per each block.
-const maxLabelsPerBlock = 10 * maxRowsPerBlock
-
 type pendingSeries struct {
 	mu sync.Mutex
 	wr writeRequest
@@ -153,10 +147,13 @@ func (wr *writeRequest) adjustSampleValues() {
 
 func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
 		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
-		if len(wr.samples) >= maxRowsPerBlock || len(wr.labels) >= maxLabelsPerBlock {
+		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
 			wr.flush()
 			tssDst = wr.tss

app/vmagent/remotewrite/relabel.go

@@ -15,12 +15,14 @@ import (
 var (
 	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
+		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
 		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
 	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
 		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
-	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
-	relabelDebug       = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+		"The path can point either to local file or to http url")
+	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
 		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
 		"This is useful for debugging the relabeling configs")
 )

app/vmagent/remotewrite/remotewrite.go

@@ -171,12 +171,12 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
 
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / maxRowsPerBlock / 100
-	if maxInmemoryBlocks > 400 {
+	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks / *queues > 100 {
 		// There is no much sense in keeping higher number of blocks in memory,
 		// since this means that the producer outperforms consumer and the queue
 		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 400
+		maxInmemoryBlocks = 100 * *queues
 	}
 	if maxInmemoryBlocks < 2 {
 		maxInmemoryBlocks = 2
@@ -274,6 +274,9 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -283,7 +286,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 			samplesCount += len(tss[i].Samples)
 			labelsCount += len(tss[i].Labels)
 			i++
-			if samplesCount >= maxRowsPerBlock || labelsCount >= maxLabelsPerBlock {
+			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
 				break
 			}
 		}
@@ -301,11 +304,7 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if len(tssBlock) > 0 {
-			for _, rwctx := range rwctxs {
-				rwctx.Push(tssBlock)
-			}
-		}
+		pushBlockToRemoteStorages(rwctxs, tssBlock)
 		if rctx != nil {
 			rctx.reset()
 		}
@@ -315,6 +314,23 @@ func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	}
 }
 
+func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
+	if len(tssBlock) == 0 {
+		// Nothing to push
+		return
+	}
+	// Push block to remote storages in parallel in order to reduce the time needed for sending the data to multiple remote storage systems.
+	var wg sync.WaitGroup
+	for _, rwctx := range rwctxs {
+		wg.Add(1)
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
+			rwctx.Push(tssBlock)
+		}(rwctx)
+	}
+	wg.Wait()
+}
+
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
 func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
 	if !*sortLabels {
@@ -374,6 +390,8 @@ var labelsHashBufPool bytesutil.ByteBufferPool
 func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
+		// Do not use logger.WithThrottler() here, since this will increase CPU usage
+		// because every call to logSkippedSeries will result to a call to labelsToString.
 		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
 	default:
 	}

app/vmalert/README.md

@@ -2,7 +2,11 @@
 
 `vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
 or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
-rules against configured address. It is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
+rules against configured `-datasource.url`. For sending alerting notifications
+vmalert relies on [Alertmanager]((https://github.com/prometheus/alertmanager)) configured via `-notifier.url` flag.
+Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+protocol and require `-remoteWrite.url` to be configured.
+Vmalert is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
 implementation and aims to be compatible with its syntax.
 
 ## Features
@@ -18,12 +22,12 @@ implementation and aims to be compatible with its syntax.
 * Lightweight without extra dependencies.
 
 ## Limitations
-* `vmalert` execute queries against remote datasource which has reliability risks because of network.
-It is recommended to configure alerts thresholds and rules expressions with understanding that network request
-may fail;
-* by default, rules execution is sequential within one group, but persisting of execution results to remote
-storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
-recording rule is reused in next one;
+* `vmalert` execute queries against remote datasource which has reliability risks because of the network.
+It is recommended to configure alerts thresholds and rules expressions with the understanding that network 
+requests may fail;
+* by default, rules execution is sequential within one group, but persistence of execution results to remote
+storage is asynchronous. Hence, user shouldn't rely on chaining of recording rules when result of previous
+recording rule is reused in the next one;
 
 ## QuickStart
 
@@ -33,29 +37,36 @@ git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
 ```
-The build binary will be placed to `VictoriaMetrics/bin` folder.
+The build binary will be placed in `VictoriaMetrics/bin` folder.
 
 To start using `vmalert` you will need the following things:
 * list of rules - PromQL/MetricsQL expressions to execute;
-* datasource address - reachable VictoriaMetrics instance for rules execution;
-* notifier address - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
-aggregating alerts and sending notifications.
+* datasource address - reachable MetricsQL endpoint to run queries against;
+* notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
+aggregating alerts, and sending notifications.
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-compatible storage address for storing recording rules results and alerts state in for of timeseries.
+  compatible storage to persist rules and alerts state info;
+* remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
 
 Then configure `vmalert` accordingly:
 ```
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
-    -notifier.url=http://localhost:9093 \    # AlertManager URL
+    -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
     -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
-    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules
+    -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
     -remoteRead.url=http://localhost:8428 \  # MetricsQL compatible datasource to restore alerts state from
     -external.label=cluster=east-1 \         # External label to be applied for each rule
     -external.label=replica=a                # Multiple external labels may be set
 ```
 
-See the fill list of configuration flags in [configuration](#configuration) section.
+Note there's a separate `remoteRead.url` to allow writing results of
+alerting/recording rules into a different storage than the initial data that's
+queried. This allows using `vmalert` to aggregate data from a short-term,
+high-frequency, high-cardinality storage into a long-term storage with
+decreased cardinality and a bigger interval between samples.
+
+See the full list of configuration flags in [configuration](#configuration) section.
 
 If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
 to specify different `external.label` flags in order to define which `vmalert` generated rules or alerts.
@@ -85,15 +96,27 @@ name: <string>
 [ concurrency: <integer> | default = 1 ]
 
 # Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus".
-# By default "prometheus" rule type is used.
+# By default "prometheus" type is used.
 [ type: <string> ]
 
-# Optional list of label filters applied to every rule's
-# request withing a group. Is compatible only with VM datasource.
-# See more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+# Warning: DEPRECATED
+# Please use `params` instead:
+#   params:
+#     extra_label: ["job=nodeexporter", "env=prod"]
 extra_filter_labels:
   [ <labelname>: <labelvalue> ... ]
 
+# Optional list of HTTP URL parameters
+# applied for all rules requests within a group
+# For example:
+#  params:
+#    nocache: ["1"]                # disable caching for vmselect
+#    denyPartialResponse: ["true"] # fail if one or more vmstorage nodes returned an error
+#    extra_label: ["env=dev"]      # apply additional label filter "env=dev" for all requests 
+# see more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+params:
+  [ <string>: [<string>, ...]]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment
@@ -113,14 +136,14 @@ expression and then act according to the Rule type.
 
 There are two types of Rules:
 * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) -
-Alerting rules allows to define alert conditions via `expr` field and to send notifications
+Alerting rules allow defining alert conditions via `expr` field and to send notifications to
 [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
 * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -
-Recording rules allows to define `expr` which result will be than backfilled to configured
+Recording rules allow defining `expr` which result will be then backfilled to configured
 `-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
 expensive expressions and save their result as a new set of time series.
 
-`vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels
+`vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
 within one group.
 
 #### Alerting rules
@@ -130,17 +153,13 @@ The syntax for alerting rule is the following:
 # The name of the alert. Must be a valid metric name.
 alert: <string>
 
-# Optional type for the rule. Supported values: "graphite", "prometheus".
-# By default "prometheus" rule type is used.
-[ type: <string> ]
-
 # The expression to evaluate. The expression language depends on the type value.
-# By default PromQL/MetricsQL expression is used. If type="graphite", then the expression
+# By default PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
 # Alerts are considered firing once they have been returned for this long.
-# Alerts which have not yet fired for long enough are considered pending.
+# Alerts which have not yet been fired for long enough are considered pending.
 # If param is omitted or set to 0 then alerts will be immediately considered
 # as firing once they return.
 [ for: <duration> | default = 0s ]
@@ -166,12 +185,8 @@ The syntax for recording rules is following:
 # The name of the time series to output to. Must be a valid metric name.
 record: <string>
 
-# Optional type for the rule. Supported values: "graphite", "prometheus".
-# By default "prometheus" rule type is used.
-[ type: <string> ]
-
 # The expression to evaluate. The expression language depends on the type value.
-# By default MetricsQL expression is used. If type="graphite", then the expression
+# By default MetricsQL expression is used. If group.type="graphite", then the expression
 # must contain valid Graphite expression.
 expr: <string>
 
@@ -189,19 +204,19 @@ For recording rules to work `-remoteWrite.url` must be specified.
 the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
 * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
 into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
-These are regular time series and may be queried from VM just as any other time series.
-The state stored to the configured address on every rule evaluation.
+These are regular time series and maybe queried from VM just as any other time series.
+The state is stored to the configured address on every rule evaluation.
 * `-remoteRead.url` - URL to VictoriaMetrics (Single) or vmselect (Cluster). `vmalert` will try to restore alerts state
 from configured address by querying time series with name `ALERTS_FOR_STATE`.
 
-Both flags are required for the proper state restoring. Restore process may fail if time series are missing
+Both flags are required for proper state restoration. Restore process may fail if time series are missing
 in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
 or received state doesn't match current `vmalert` rules configuration.
 
 
 ### Multitenancy
 
-There are the following approaches for alerting and recording rules across
+There are the following approaches exist for alerting and recording rules across
 [multiple tenants](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy):
 
 * To run a separate `vmalert` instance per each tenant.
@@ -214,7 +229,7 @@ There are the following approaches for alerting and recording rules across
   rules to `AccountID=123`.
 
 * To specify `tenant` parameter per each alerting and recording group if
-  [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used
+  [enterprise version of vmalert](https://victoriametrics.com/products/enterprise/) is used
   with `-clusterMode` command-line flag. For example:
 
 ```yaml
@@ -234,12 +249,119 @@ If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-re
 contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`.
 `vmalert` automatically adds the specified tenant to urls per each recording rule in this case.
 
+If `-clusterMode` is enabled and the `tenant` in a particular group is missing, then the tenant value
+is obtained from `-defaultTenant.prometheus` or `-defaultTenant.graphite` depending on the `type` of the group.
+
 The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files
 at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise`
 tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).
 
+### Topology examples
 
-### WEB
+The following sections are showing how `vmalert` may be used and configured 
+for different scenarios. 
+
+Please note, not all flags in examples are required: 
+* `-remoteWrite.url` and `-remoteRead.url` are optional and are needed only if
+you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
+* `-notifier.url` is optional and is needed only if you have alerting rules.
+
+#### Single-node VictoriaMetrics
+
+The simplest configuration where one single-node VM server is used for
+rules execution, storing recording rules results and alerts state.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                    # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093          # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert single" width="500" src="vmalert_single.png">
+
+
+#### Cluster VictoriaMetrics
+
+In [cluster mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html)
+VictoriaMetrics has separate components for writing and reading path:
+`vminsert` and `vmselect` components respectively. `vmselect` is used for executing rules expressions
+and `vminsert` is used to persist recording rules results and alerts state.
+Cluster mode could have multiple `vminsert` and `vmselect` components. 
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml  \                                # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for executing rules expressions
+    -remoteWrite.url=http://vminsert:8480/insert/0/prometheuss  # vminsert addr to persist alerts state and recording rules results
+    -remoteRead.url=http://vmselect:8481/select/0/prometheus    # vmselect addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager:9093                      # AlertManager addr to send alerts when they trigger
+```
+
+<img alt="vmalert cluster" src="vmalert_cluster.png">
+
+In case when you want to spread the load on these components - add balancers before them and configure
+`vmalert` with balancer's addresses. Please, see more about VM's cluster architecture
+[here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview).
+
+#### HA vmalert
+
+For HA user can run multiple identically configured `vmalert` instances.
+It means all of them will execute the same rules, write state and results to
+the same destinations, and send alert notifications to multiple configured
+Alertmanagers.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=rules.yml \                   # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://victoriametrics:8428 \   # VM-single addr for executing rules expressions
+    -remoteWrite.url=http://victoriametrics:8428 \  # VM-single addr to persist alerts state and recording rules results
+    -remoteRead.url=http://victoriametrics:8428 \   # VM-single addr for restoring alerts state after restart
+    -notifier.url=http://alertmanager1:9093 \       # Multiple AlertManager addresses to send alerts when they trigger
+    -notifier.url=http://alertmanagerN:9093         # The same alert will be sent to all configured notifiers
+```
+
+<img alt="vmalert ha" width="800px" src="vmalert_ha.png">
+
+To avoid recording rules results and alerts state duplication in VictoriaMetrics server
+don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
+
+Alertmanager will automatically deduplicate alerts with identical labels, so ensure that
+all `vmalert`s are having the same config.
+
+Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/)
+for Alertmanagers for better reliability.
+
+This example uses single-node VM server for the sake of simplicity. 
+Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed. 
+
+
+#### Downsampling and aggregation via vmalert
+
+Example shows how to build a topology where `vmalert` will process data from one cluster
+and write results into another. Such clusters may be called as "hot" (low retention,
+high-speed disks, used for operative monitoring) and "cold" (long term retention,
+slower/cheaper disks, low resolution data). With help of `vmalert`, user can setup
+recording rules to process raw data from "hot" cluster (by applying additional transformations
+or reducing resolution) and push results to "cold" cluster.
+
+`vmalert` configuration flags:
+```
+./bin/vmalert -rule=downsampling-rules.yml \                                        # Path to the file with rules configuration. Supports wildcard
+    -datasource.url=http://raw-cluster-vmselect:8481/select/0/prometheus            # vmselect addr for executing recordi ng rules expressions
+    -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheuss   # vminsert addr to persist recording rules results
+```
+
+<img alt="vmalert multi cluster" src="vmalert_multicluster.png">
+
+Please note, [replay](#rules-backfilling) feature may be used for transforming historical data.
+
+Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only recording rules are used.
+
+
+### Web
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
 * `http://<vmalert-addr>` - UI;
@@ -257,12 +379,12 @@ vmalert sends requests to `<-datasource.url>/render?format=json` during evaluati
 if the corresponding group or rule contains `type: "graphite"` config option. It is expected that the `<-datasource.url>/render`
 implements [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) for `format=json`.
 When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VM do not forget
-to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on query type.
+to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.
 
 ## Rules backfilling
 
 vmalert supports alerting and recording rules backfilling (aka `replay`). In replay mode vmalert
-can read the same rules configuration as normally, evaluate them on the given time range and backfill
+can read the same rules configuration as normal, evaluate them on the given time range and backfill
 results via remote write to the configured storage. vmalert supports any PromQL/MetricsQL compatible
 data source for backfilling.
 
@@ -308,19 +430,19 @@ max range per request:  8h20m0s
 In `replay` mode all groups are executed sequentially one-by-one. Rules within the group are
 executed sequentially as well (`concurrency` setting is ignored). Vmalert sends rule's expression
 to [/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) endpoint
-of the configured `-datasource.url`. Returned data then processed according to the rule type and
-backfilled to `-remoteWrite.url` via [Remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
+of the configured `-datasource.url`. Returned data is then processed according to the rule type and
+backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
 Vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
 Vmalert automatically disables caching on VictoriaMetrics side by sending `nocache=1` param. It allows
 to prevent cache pollution and unwanted time range boundaries adjustment during backfilling.
 
 #### Recording rules
 
-Result of recording rules `replay` should match with results of normal rules evaluation.
+The result of recording rules `replay` should match with results of normal rules evaluation.
 
 #### Alerting rules
 
-Result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
+The result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
 To see if `replayed` alert has fired in the past use the following PromQL/MetricsQL expression:
 ```
 ALERTS{alertname="your_alertname", alertstate="firing"}
@@ -333,7 +455,7 @@ There are following non-required `replay` flags:
 
 * `-replay.maxDatapointsPerQuery` - the max number of data points expected to receive in one request.
 In two words, it affects the max time range for every `/query_range` request. The higher the value,
-the less requests will be issued during `replay`.
+the fewer requests will be issued during `replay`.
 * `-replay.ruleRetryAttempts` - when datasource fails to respond vmalert will make this number of retries
 per rule before giving up.
 * `-replay.rulesDelay` - delay between sequential rules execution. Important in cases if there are chaining
@@ -355,13 +477,15 @@ See full description for these flags in `./vmalert --help`.
 We recommend setting up regular scraping of this page either through `vmagent` or by Prometheus so that the exported
 metrics may be analyzed later.
 
-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview.
+Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview. Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add
 a review to the dashboard.
 
 
 ## Configuration
 
+### Flags
+
 Pass `-help` to `vmalert` in order to see the full list of supported
 command-line flags with their descriptions.
 
@@ -459,7 +583,7 @@ The shortlist of configuration flags is the following:
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
     	Optional basic auth password for -notifier.url
     	Supports an array of values separated by comma or specified via multiple flags.
@@ -482,10 +606,10 @@ The shortlist of configuration flags is the following:
     	Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
     	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.url array
-    	Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+    	Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
     	Supports an array of values separated by comma or specified via multiple flags.
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -remoteRead.basicAuth.password string
     	Optional basic auth password for -remoteRead.url
   -remoteRead.basicAuth.passwordFile string
@@ -583,12 +707,32 @@ The shortlist of configuration flags is the following:
     	Show VictoriaMetrics version
 ```
 
+### Hot config reload
 `vmalert` supports "hot" config reload via the following methods:
 * send SIGHUP signal to `vmalert` process;
 * send GET request to `/-/reload` endpoint;
 * configure `-rule.configCheckInterval` flag for periodic reload
 on config change.
 
+### URL params
+
+To set additional URL params for `datasource.url`, `remoteWrite.url` or `remoteRead.url`
+just add them in address: `-datasource.url=http://localhost:8428?nocache=1`.
+
+To set additional URL params for specific [group of rules](#Groups) modify
+the `params` group:
+```yaml
+groups:
+  - name: TestGroup
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
+```
+Please note, `params` are used only for executing rules expressions (requests to `datasource.url`).
+If there would be a conflict between URL params set in `datasource.url` flag and params in group definition
+the latter will have higher priority.
+
+
 ## Contributing
 
 `vmalert` is mostly designed and built by VictoriaMetrics community.
@@ -604,7 +748,7 @@ It is recommended using
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert` binary and puts it into the `bin` folder.
 
@@ -621,7 +765,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmalert-arm` or `make vmalert-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-arm` or `vmalert-arm64` binary respectively and puts it into the `bin` folder.
 

app/vmalert/alerting.go

@@ -58,7 +58,7 @@ type alertingRuleMetrics struct {
 
 func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
-		Type:         cfg.Type,
+		Type:         group.Type,
 		RuleID:       cfg.ID,
 		Name:         cfg.Alert,
 		Expr:         cfg.Expr,
@@ -69,9 +69,9 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		GroupName:    group.Name,
 		EvalInterval: group.Interval,
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &cfg.Type,
+			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
 		metrics: &alertingRuleMetrics{},
@@ -153,6 +153,13 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range series {
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			s.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			s.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		// extra labels could contain templates, so we expand them first
 		labels, err := expandLabels(s, qFn, ar)
 		if err != nil {
@@ -163,13 +170,6 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 			// so the hash key will be consistent on restore
 			s.SetLabel(k, v)
 		}
-		// set additional labels to identify group and rule name
-		if ar.Name != "" {
-			s.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			s.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
 		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
 		if err != nil {
 			return nil, fmt.Errorf("failed to create alert: %s", err)
@@ -225,6 +225,13 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			m.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			m.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		// extra labels could contain templates, so we expand them first
 		labels, err := expandLabels(m, qFn, ar)
 		if err != nil {
@@ -235,14 +242,6 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 			// so the hash key will be consistent on restore
 			m.SetLabel(k, v)
 		}
-		// set additional labels to identify group and rule name
-		// set additional labels to identify group and rule name
-		if ar.Name != "" {
-			m.SetLabel(alertNameLabel, ar.Name)
-		}
-		if !*disableAlertGroupLabel && ar.GroupName != "" {
-			m.SetLabel(alertGroupNameLabel, ar.GroupName)
-		}
 		h := hash(m)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels

app/vmalert/alerting_test.go

@@ -715,6 +715,44 @@ func TestAlertingRule_Template(t *testing.T) {
 				},
 			},
 		},
+		{
+			&AlertingRule{
+				Name:      "ExtraTemplating",
+				GroupName: "Testing",
+				Labels: map[string]string{
+					"name":     "alert_{{ $labels.alertname }}",
+					"group":    "group_{{ $labels.alertgroup }}",
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary":     `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+					"description": `Alert "{{ $labels.name }}({{ $labels.group }})" for instance {{ $labels.instance }}`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "instance", "foo"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, alertNameLabel, "ExtraTemplating",
+					"name", "alert_ExtraTemplating",
+					alertGroupNameLabel, "Testing",
+					"group", "group_Testing",
+					"instance", "foo")): {
+					Labels: map[string]string{
+						alertNameLabel:      "ExtraTemplating",
+						"name":              "alert_ExtraTemplating",
+						alertGroupNameLabel: "Testing",
+						"group":             "group_Testing",
+						"instance":          "foo",
+					},
+					Annotations: map[string]string{
+						"summary":     `Alert "ExtraTemplating(Testing)" for instance foo`,
+						"description": `Alert "alert_ExtraTemplating(group_Testing)" for instance foo`,
+					},
+				},
+			},
+		},
 	}
 	fakeGroup := Group{Name: "TestRule_Exec"}
 	for _, tc := range testCases {

app/vmalert/config/config.go

@@ -5,17 +5,18 @@ import (
 	"fmt"
 	"hash/fnv"
 	"io/ioutil"
+	"net/url"
 	"path/filepath"
 	"sort"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"gopkg.in/yaml.v2"
 )
 
 // Group contains list of Rules grouped into
@@ -30,6 +31,7 @@ type Group struct {
 	// ExtraFilterLabels is a list label filters applied to every rule
 	// request withing a group. Is compatible only with VM datasources.
 	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
+	// DEPRECATED: use Params field instead
 	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
 	// Labels is a set of label value pairs, that will be added to every rule.
 	// It has priority over the external labels.
@@ -37,6 +39,8 @@ type Group struct {
 	// Checksum stores the hash of yaml definition for this group.
 	// May be used to detect any changes like rules re-ordering etc.
 	Checksum string
+	// Optional HTTP URL parameters added to each rule request
+	Params url.Values `yaml:"params"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -56,12 +60,20 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if g.Type.Get() == "" {
 		g.Type.Set(datasource.NewPrometheusType())
 	}
-	// update rules with empty type.
-	for i, r := range g.Rules {
-		if r.Type.Get() == "" {
-			r.Type.Set(g.Type)
-			r.ID = HashRule(r)
-			g.Rules[i] = r
+
+	// backward compatibility with deprecated `ExtraFilterLabels` param
+	if len(g.ExtraFilterLabels) > 0 {
+		if g.Params == nil {
+			g.Params = url.Values{}
+		}
+		// Sort extraFilters for consistent order for query args across runs.
+		extraFilters := make([]string, 0, len(g.ExtraFilterLabels))
+		for k, v := range g.ExtraFilterLabels {
+			extraFilters = append(extraFilters, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(extraFilters)
+		for _, extraFilter := range extraFilters {
+			g.Params.Add("extra_label", extraFilter)
 		}
 	}
 
@@ -76,9 +88,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
-	if len(g.Rules) == 0 {
-		return fmt.Errorf("group %q can't contain no rules", g.Name)
-	}
 
 	uniqueRules := map[uint64]struct{}{}
 	for _, r := range g.Rules {
@@ -97,9 +106,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 			// its needed only for tests.
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
-			if r.Type.Get() != "" {
-				exprValidator = r.Type.ValidateExpr
-			}
 			if err := exprValidator(r.Expr); err != nil {
 				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
@@ -120,7 +126,6 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 // recording rule or alerting rule.
 type Rule struct {
 	ID          uint64
-	Type        datasource.Type    `yaml:"type,omitempty"`
 	Record      string             `yaml:"record,omitempty"`
 	Alert       string             `yaml:"alert,omitempty"`
 	Expr        string             `yaml:"expr"`
@@ -162,7 +167,6 @@ func HashRule(r Rule) uint64 {
 		h.Write([]byte("alerting"))
 		h.Write([]byte(r.Alert))
 	}
-	h.Write([]byte(r.Type.Get()))
 	kv := sortMap(r.Labels)
 	for _, i := range kv {
 		h.Write([]byte(i.key))
@@ -194,6 +198,7 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		fp = append(fp, matches...)
 	}
 	errGroup := new(utils.ErrGroup)
+	var isExtraFilterLabelsUsed bool
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
@@ -213,6 +218,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
+			if len(g.ExtraFilterLabels) > 0 {
+				isExtraFilterLabelsUsed = true
+			}
 			groups = append(groups, g)
 		}
 	}
@@ -222,6 +230,9 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
+	if isExtraFilterLabelsUsed {
+		logger.Warnf("field `extra_filter_labels` is deprecated - use `params` instead")
+	}
 	return groups, nil
 }
 

app/vmalert/config/config_test.go

@@ -7,10 +7,11 @@ import (
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
@@ -95,10 +96,6 @@ func TestGroup_Validate(t *testing.T) {
 			group:  &Group{},
 			expErr: "group name must be set",
 		},
-		{
-			group:  &Group{Name: "test"},
-			expErr: "contain no rules",
-		},
 		{
 			group: &Group{Name: "test",
 				Rules: []Rule{
@@ -267,7 +264,6 @@ func TestGroup_Validate(t *testing.T) {
 					},
 					{
 						Expr: "sum(up == 0 ) by (host)",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -283,7 +279,6 @@ func TestGroup_Validate(t *testing.T) {
 					},
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
-						Type: datasource.NewPrometheusType(),
 					},
 				},
 			},
@@ -478,6 +473,85 @@ concurrency: 16
 rules:
   - alert: ExampleAlertWithFor
     expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`params` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+    nocache: ["1"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+params:
+    nocache: ["0"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
 `)
 	})
 }
+
+func TestGroupParams(t *testing.T) {
+	f := func(t *testing.T, data string, expParams url.Values) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		got, exp := g.Params.Encode(), expParams.Encode()
+		if got != exp {
+			t.Fatalf("expected to have %q; got %q", exp, got)
+		}
+	}
+
+	t.Run("no params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{})
+	})
+
+	t.Run("params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+  nocache: ["1"]
+  denyPartialResponse: ["true"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
+	})
+
+	t.Run("extra labels", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+  env: prod
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"extra_label": {"env=prod", "job=victoriametrics"}})
+	})
+
+	t.Run("extra labels and params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+extra_filter_labels:
+  job: victoriametrics
+params:
+  nocache: ["1"]
+  extra_label: ["env=prod"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "extra_label": {"env=prod", "job=victoriametrics"}})
+	})
+}

app/vmalert/config/testdata/dir/rules-update0-good.rules

@@ -1,15 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 2s
-    concurrency: 2
-    type: prometheus
-    labels:
-      cluster: main
-    rules:
-      - alert: up
-        expr: up == 0
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/dir/rules-update1-good.rules

@@ -1,12 +0,0 @@
-groups:
-  - name: TestUpdateGroup
-    interval: 30s
-    type: graphite
-    rules:
-      - alert: up
-        expr: filterSeries(time('host.2',20),'>','0')
-        for: 30s
-      - alert: up graphite
-        expr: filterSeries(time('host.1',20),'>','0')
-        for: 30s
-        type: graphite

app/vmalert/config/testdata/rules0-good.rules

@@ -1,5 +1,8 @@
 groups:
   - name: groupGorSingleAlert
+    params:
+      nocache: ["1"]
+      denyPartialResponse: ["true"]
     rules:
       - alert: VMRows
         for: 10s

app/vmalert/config/testdata/rules2-good.rules

@@ -2,8 +2,11 @@ groups:
   - name: TestGroup
     interval: 2s
     concurrency: 2
-    extra_filter_labels:
+    extra_filter_labels: # deprecated param, use `params` instead
         job: victoriametrics
+    params:
+      denyPartialResponse: ["true"]
+      extra_label: ["env=dev"]
     rules:
       - alert: Conns
         expr: sum(vm_tcplistener_conns) by(instance) > 1

app/vmalert/config/testdata/rules3-good.rules

@@ -21,10 +21,3 @@ groups:
         annotations:
           summary: Too high connection number for {{$labels.instance}}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
-      - alert: HostDown
-        type: graphite
-        expr: filterSeries(sumSeries(host.receiver.interface.up),'last','=', 0)
-        for: 3m
-        annotations:
-          summary: Too high connection number for {{$labels.instance}}
-          description: "It is {{ $value }} connections for {{$labels.instance}}"

app/vmalert/config/testdata/rules4-good.rules

@@ -0,0 +1,8 @@
+groups:
+  - name: TestEmptyRules
+    interval: 2s
+    concurrency: 2
+    rules:
+
+  - name: TestNoRules
+    type: prometheus

app/vmalert/datasource/datasource.go

@@ -2,6 +2,7 @@ package datasource
 
 import (
 	"context"
+	"net/url"
 	"time"
 )
 
@@ -20,8 +21,7 @@ type QuerierBuilder interface {
 type QuerierParams struct {
 	DataSourceType     *Type
 	EvaluationInterval time.Duration
-	// see https://docs.victoriametrics.com/#prometheus-querying-api-enhancements
-	ExtraLabels map[string]string
+	QueryParams        url.Values
 }
 
 // Metric is the basic entity which should be return by datasource

app/vmalert/datasource/init.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
@@ -40,9 +41,9 @@ type Param struct {
 }
 
 // Init creates a Querier from provided flag values.
-// Provided extraParams will be added as GET params to
+// Provided extraParams will be added as GET params for
 // each request.
-func Init(extraParams []Param) (QuerierBuilder, error) {
+func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
@@ -56,11 +57,11 @@ func Init(extraParams []Param) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 
+	if extraParams == nil {
+		extraParams = url.Values{}
+	}
 	if *roundDigits > 0 {
-		extraParams = append(extraParams, Param{
-			Key:   "round_digits",
-			Value: fmt.Sprintf("%d", *roundDigits),
-		})
+		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
 	}
 
 	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)

app/vmalert/datasource/type.go

@@ -7,9 +7,6 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
-const graphiteType = "graphite"
-const prometheusType = "prometheus"
-
 // Type represents data source type
 type Type struct {
 	name string
@@ -17,12 +14,16 @@ type Type struct {
 
 // NewPrometheusType returns prometheus datasource type
 func NewPrometheusType() Type {
-	return Type{name: prometheusType}
+	return Type{
+		name: "prometheus",
+	}
 }
 
 // NewGraphiteType returns graphite datasource type
 func NewGraphiteType() Type {
-	return Type{name: graphiteType}
+	return Type{
+		name: "graphite",
+	}
 }
 
 // NewRawType returns datasource type from raw string
@@ -44,19 +45,19 @@ func (t *Type) Set(d Type) {
 // String implements String interface with default value.
 func (t Type) String() string {
 	if t.name == "" {
-		return prometheusType
+		return "prometheus"
 	}
 	return t.name
 }
 
 // ValidateExpr validates query expression with datasource ql.
 func (t *Type) ValidateExpr(expr string) error {
-	switch t.name {
-	case graphiteType:
+	switch t.String() {
+	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
 			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
 		}
-	case "", prometheusType:
+	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
@@ -72,12 +73,13 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err := unmarshal(&s); err != nil {
 		return err
 	}
+	if s == "" {
+		s = "prometheus"
+	}
 	switch s {
-	case "":
-		s = prometheusType
-	case graphiteType, prometheusType:
+	case "graphite", "prometheus":
 	default:
-		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, prometheusType, graphiteType)
+		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
 	t.name = s
 	return nil

app/vmalert/datasource/vm.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
@@ -22,8 +23,7 @@ type VMStorage struct {
 
 	dataSourceType     Type
 	evaluationInterval time.Duration
-	extraLabels        []string
-	extraParams        []Param
+	extraParams        url.Values
 	disablePathAppend  bool
 }
 
@@ -47,9 +47,7 @@ func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
 		s.dataSourceType = *params.DataSourceType
 	}
 	s.evaluationInterval = params.EvaluationInterval
-	for k, v := range params.ExtraLabels {
-		s.extraLabels = append(s.extraLabels, fmt.Sprintf("%s=%s", k, v))
-	}
+	s.extraParams = params.QueryParams
 	return s
 }
 
@@ -80,10 +78,10 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	}
 
 	ts := time.Now()
-	switch s.dataSourceType.name {
-	case "", prometheusType:
+	switch s.dataSourceType.String() {
+	case "prometheus":
 		s.setPrometheusInstantReqParams(req, query, ts)
-	case graphiteType:
+	case "graphite":
 		s.setGraphiteReqParams(req, query, ts)
 	default:
 		return nil, fmt.Errorf("engine not found: %q", s.dataSourceType.name)
@@ -98,7 +96,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	}()
 
 	parseFn := parsePrometheusResponse
-	if s.dataSourceType.name != prometheusType {
+	if s.dataSourceType.name != "prometheus" {
 		parseFn = parseGraphiteResponse
 	}
 	return parseFn(req, resp)
@@ -108,7 +106,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 // For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 // Graphite type isn't supported.
 func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
-	if s.dataSourceType.name != prometheusType {
+	if s.dataSourceType.name != "prometheus" {
 		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType.name)
 	}
 	req, err := s.newRequestPOST()
@@ -150,7 +148,7 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
 		if auth := s.authCfg.GetAuthHeader(); auth != "" {
 			req.Header.Set("Authorization", auth)

app/vmalert/datasource/vm_graphite_api.go

@@ -54,6 +54,14 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("format", "json")
 	q.Set("target", query)
 	from := "-5min"

app/vmalert/datasource/vm_prom_api.go

@@ -150,20 +150,24 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 
 func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
 	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
 	q.Set("query", query)
-	if s.evaluationInterval > 0 {
-		// set step as evaluationInterval by default
-		q.Set("step", s.evaluationInterval.String())
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
-	if s.queryStep > 0 {
-		// override step with user-specified value
-		q.Set("step", s.queryStep.String())
-	}
-	for _, l := range s.extraLabels {
-		q.Add("extra_label", l)
-	}
-	for _, p := range s.extraParams {
-		q.Add(p.Key, p.Value)
+	if s.queryStep > 0 { // override step with user-specified value
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
 }

app/vmalert/datasource/vm_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"strconv"
 	"strings"
@@ -20,7 +21,7 @@ var (
 	basicAuthPass = "bar"
 	baCfg         = &promauth.BasicAuthConfig{
 		Username: basicAuthName,
-		Password: basicAuthPass,
+		Password: promauth.NewSecret(basicAuthPass),
 	}
 	query       = "vm_rows"
 	queryRender = "constantLine(10)"
@@ -435,15 +436,28 @@ func TestRequestParams(t *testing.T) {
 				queryStep: time.Minute,
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, time.Minute, timestamp.Unix())
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(time.Minute.Seconds()), timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"round digits",
+			"step to seconds",
 			false,
 			&VMStorage{
-				extraParams: []Param{{"round_digits", "10"}},
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(evalInterval.Seconds()), tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
 			},
 			func(t *testing.T, r *http.Request) {
 				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
@@ -451,45 +465,32 @@ func TestRequestParams(t *testing.T) {
 			},
 		},
 		{
-			"extra labels",
-			false,
-			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&time=%d", query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
-			},
-		},
-		{
-			"extra labels range",
+			"prometheus extra params range",
 			true,
 			&VMStorage{
-				extraLabels: []string{
-					"env=prod",
-					"query=es=cape",
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("end=%d&extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&start=%d",
+				exp := fmt.Sprintf("end=%d&max_lookback=1h&nocache=1&query=%s&start=%d",
 					timestamp.Unix(), query, timestamp.Unix())
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
 		{
-			"extra params",
+			"graphite extra params",
 			false,
 			&VMStorage{
-				extraParams: []Param{
-					{Key: "nocache", Value: "1"},
-					{Key: "max_lookback", Value: "1h"},
+				dataSourceType: NewGraphiteType(),
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("max_lookback=1h&nocache=1&query=%s&time=%d", query, timestamp.Unix())
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
@@ -501,14 +502,14 @@ func TestRequestParams(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			switch tc.vm.dataSourceType.name {
-			case "", prometheusType:
+			switch tc.vm.dataSourceType.String() {
+			case "prometheus":
 				if tc.queryRange {
 					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
 				} else {
 					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
 				}
-			case graphiteType:
+			case "graphite":
 				tc.vm.setGraphiteReqParams(req, query, timestamp)
 			}
 			tc.checkFn(t, req)
@@ -519,7 +520,7 @@ func TestRequestParams(t *testing.T) {
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
 	if got != exp {
-		t.Errorf("expected to get %q; got %q", exp, got)
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }
 

app/vmalert/group.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"net/url"
 	"sync"
 	"time"
 
@@ -27,8 +28,8 @@ type Group struct {
 	Concurrency int
 	Checksum    string
 
-	ExtraFilterLabels map[string]string
-	Labels            map[string]string
+	Labels map[string]string
+	Params url.Values
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -71,14 +72,14 @@ func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[s
 
 func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
-		Type:              cfg.Type,
-		Name:              cfg.Name,
-		File:              cfg.File,
-		Interval:          cfg.Interval.Duration(),
-		Concurrency:       cfg.Concurrency,
-		Checksum:          cfg.Checksum,
-		ExtraFilterLabels: cfg.ExtraFilterLabels,
-		Labels:            cfg.Labels,
+		Type:        cfg.Type,
+		Name:        cfg.Name,
+		File:        cfg.File,
+		Interval:    cfg.Interval.Duration(),
+		Concurrency: cfg.Concurrency,
+		Checksum:    cfg.Checksum,
+		Params:      cfg.Params,
+		Labels:      cfg.Labels,
 
 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
@@ -198,7 +199,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 	// group.Start function
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
-	g.ExtraFilterLabels = newGroup.ExtraFilterLabels
+	g.Params = newGroup.Params
 	g.Labels = newGroup.Labels
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
@@ -226,7 +227,7 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 
 	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
 	if !skipRandSleepOnGroupStart {
-		randSleep := uint64(float64(g.Interval) * (float64(uint32(g.ID())) / (1 << 32)))
+		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
 		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
 		if randSleep < sleepOffset {
 			randSleep += uint64(g.Interval)
@@ -283,14 +284,15 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 		case <-t.C:
 			g.metrics.iterationTotal.Inc()
 			iterationStart := time.Now()
-			resolveDuration := getResolveDuration(g.Interval)
-			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, resolveDuration)
-			for err := range errs {
-				if err != nil {
-					logger.Errorf("group %q: %s", g.Name, err)
+			if len(g.Rules) > 0 {
+				resolveDuration := getResolveDuration(g.Interval)
+				errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, resolveDuration)
+				for err := range errs {
+					if err != nil {
+						logger.Errorf("group %q: %s", g.Name, err)
+					}
 				}
 			}
-
 			g.metrics.iterationDuration.UpdateDuration(iterationStart)
 		}
 	}
@@ -354,6 +356,7 @@ var (
 	execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)
 
 	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )
 
 func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
@@ -367,6 +370,7 @@ func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Dur
 
 	if len(tss) > 0 && e.rw != nil {
 		for _, ts := range tss {
+			remoteWriteTotal.Inc()
 			if err := e.rw.Push(ts); err != nil {
 				remoteWriteErrors.Inc()
 				return fmt.Errorf("rule %q: remote write failure: %w", rule, err)

app/vmalert/group_test.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 )
@@ -108,17 +107,6 @@ func TestUpdateWith(t *testing.T) {
 				{Record: "foo5"},
 			},
 		},
-		{
-			"update datasource type",
-			[]config.Rule{
-				{Alert: "foo1", Type: datasource.NewPrometheusType()},
-				{Alert: "foo3", Type: datasource.NewGraphiteType()},
-			},
-			[]config.Rule{
-				{Alert: "foo1", Type: datasource.NewGraphiteType()},
-				{Alert: "foo10", Type: datasource.NewPrometheusType()},
-			},
-		},
 	}
 
 	for _, tc := range testCases {

app/vmalert/main.go

@@ -97,6 +97,9 @@ func main() {
 		if err != nil {
 			logger.Fatalf("failed to init remoteWrite: %s", err)
 		}
+		if rw == nil {
+			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
+		}
 		notifier.InitTemplateFunc(eu)
 		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
@@ -104,8 +107,7 @@ func main() {
 		}
 		// prevent queries from caching and boundaries aligning
 		// when querying VictoriaMetrics datasource.
-		noCache := datasource.Param{Key: "nocache", Value: "1"}
-		q, err := datasource.Init([]datasource.Param{noCache})
+		q, err := datasource.Init(url.Values{"nocache": {"1"}})
 		if err != nil {
 			logger.Fatalf("failed to init datasource: %s", err)
 		}
@@ -284,13 +286,13 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			// config didn't change - skip it
 			continue
 		}
-		groupsCfg = newGroupsCfg
-		if err := m.update(ctx, groupsCfg, false); err != nil {
+		if err := m.update(ctx, newGroupsCfg, false); err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		groupsCfg = newGroupsCfg
 		configSuccess.Set(1)
 		configTimestamp.Set(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)

app/vmalert/main_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
@@ -99,6 +100,8 @@ groups:
 		querierBuilder: &fakeQuerier{},
 		groups:         make(map[uint64]*Group),
 		labels:         map[string]string{},
+		notifiers:      []notifier.Notifier{&fakeNotifier{}},
+		rw:             &remotewrite.Client{},
 	}
 
 	syncCh := make(chan struct{})

app/vmalert/manager.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"sort"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -85,12 +87,31 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) er
 }
 
 func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore bool) error {
+	var rrPresent, arPresent bool
 	groupsRegistry := make(map[uint64]*Group)
 	for _, cfg := range groupsCfg {
+		for _, r := range cfg.Rules {
+			if rrPresent && arPresent {
+				continue
+			}
+			if r.Record != "" {
+				rrPresent = true
+			}
+			if r.Alert != "" {
+				arPresent = true
+			}
+		}
 		ng := newGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
 		groupsRegistry[ng.ID()] = ng
 	}
 
+	if rrPresent && m.rw == nil {
+		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
+	}
+	if arPresent && m.notifiers == nil {
+		return fmt.Errorf("config contains alerting rules but `-notifier.url` isn't set")
+	}
+
 	type updateItem struct {
 		old *Group
 		new *Group
@@ -142,13 +163,13 @@ func (g *Group) toAPI() APIGroup {
 		// encode as string to avoid rounding
 		ID: fmt.Sprintf("%d", g.ID()),
 
-		Name:              g.Name,
-		Type:              g.Type.String(),
-		File:              g.File,
-		Interval:          g.Interval.String(),
-		Concurrency:       g.Concurrency,
-		ExtraFilterLabels: g.ExtraFilterLabels,
-		Labels:            g.Labels,
+		Name:        g.Name,
+		Type:        g.Type.String(),
+		File:        g.File,
+		Interval:    g.Interval.String(),
+		Concurrency: g.Concurrency,
+		Params:      urlValuesToStrings(g.Params),
+		Labels:      g.Labels,
 	}
 	for _, r := range g.Rules {
 		switch v := r.(type) {
@@ -160,3 +181,24 @@ func (g *Group) toAPI() APIGroup {
 	}
 	return ag
 }
+
+func urlValuesToStrings(values url.Values) []string {
+	if len(values) < 1 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(values))
+	for k := range values {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var res []string
+	for _, k := range keys {
+		params := values[k]
+		for _, v := range params {
+			res = append(res, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	return res
+}

app/vmalert/manager_test.go

@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"net/url"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -12,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 )
 
 func TestMain(m *testing.M) {
@@ -113,18 +115,6 @@ func TestManagerUpdate(t *testing.T) {
 			Name: "ExampleAlertAlwaysFiring",
 			Expr: "sum by(job) (up == 1)",
 		}
-		ExampleAlertGraphite = &AlertingRule{
-			Name: "up graphite",
-			Expr: "filterSeries(time('host.1',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
-		ExampleAlertGraphite2 = &AlertingRule{
-			Name: "up",
-			Expr: "filterSeries(time('host.2',20),'>','0')",
-			Type: datasource.NewGraphiteType(),
-			For:  defaultEvalInterval,
-		}
 	)
 
 	testCases := []struct {
@@ -226,28 +216,15 @@ func TestManagerUpdate(t *testing.T) {
 				},
 			},
 		},
-		{
-			name:       "update prometheus to graphite type",
-			initPath:   "config/testdata/dir/rules-update0-good.rules",
-			updatePath: "config/testdata/dir/rules-update1-good.rules",
-			want: []*Group{
-				{
-					File:     "config/testdata/dir/rules-update1-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     datasource.NewGraphiteType(),
-					Name:     "TestUpdateGroup",
-					Rules: []Rule{
-						ExampleAlertGraphite2,
-						ExampleAlertGraphite,
-					},
-				},
-			},
-		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group), querierBuilder: &fakeQuerier{}}
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				notifiers:      []notifier.Notifier{&fakeNotifier{}},
+			}
 
 			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
 			if err := m.update(ctx, cfgInit, false); err != nil {
@@ -276,6 +253,78 @@ func TestManagerUpdate(t *testing.T) {
 	}
 }
 
+func TestManagerUpdateNegative(t *testing.T) {
+	testCases := []struct {
+		notifiers []notifier.Notifier
+		rw        *remotewrite.Client
+		cfg       config.Group
+		expErr    string
+	}{
+		{
+			nil,
+			nil,
+			config.Group{Name: "Recording rule only",
+				Rules: []config.Rule{
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			nil,
+			config.Group{Name: "Alerting rule only",
+				Rules: []config.Rule{
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+		{
+			[]notifier.Notifier{&fakeNotifier{}},
+			nil,
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Alert: "alert1", Expr: "up > 0"},
+					{Alert: "alert2", Expr: "up > 0"},
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			&remotewrite.Client{},
+			config.Group{Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Record: "record1", Expr: "max(up)"},
+					{Record: "record2", Expr: "max(up)"},
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.cfg.Name, func(t *testing.T) {
+			m := &manager{
+				groups:         make(map[uint64]*Group),
+				querierBuilder: &fakeQuerier{},
+				notifiers:      tc.notifiers,
+				rw:             tc.rw,
+			}
+			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
+			if err == nil {
+				t.Fatalf("expected to get error; got nil")
+			}
+			if !strings.Contains(err.Error(), tc.expErr) {
+				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
+			}
+		})
+	}
+}
+
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {
 	t.Helper()
 	cfg, err := config.Parse(path, validateAnnotations, validateExpressions)

app/vmalert/notifier/alertmanager.go

@@ -32,7 +32,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
 	req = req.WithContext(ctx)
 	if am.basicAuthPass != "" {
 		req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)

app/vmalert/notifier/init.go

@@ -9,7 +9,7 @@ import (
 )
 
 var (
-	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
+	addrs             = flagutil.NewArray("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
 	basicAuthUsername = flagutil.NewArray("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword = flagutil.NewArray("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 
@@ -24,10 +24,6 @@ var (
 
 // Init creates a Notifier object based on provided flags.
 func Init(gen AlertURLGenerator) ([]Notifier, error) {
-	if len(*addrs) == 0 {
-		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
-	}
-
 	var notifiers []Notifier
 	for i, addr := range *addrs {
 		cert, key := tlsCertFile.GetOptionalArg(i), tlsKeyFile.GetOptionalArg(i)

app/vmalert/notifier/template_func.go

@@ -17,6 +17,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"net"
 	"net/url"
 	"regexp"
 	"strings"
@@ -26,6 +27,7 @@ import (
 	textTpl "text/template"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
 // metric is private copy of datasource.Metric,
@@ -61,6 +63,7 @@ var tmplFunc textTpl.FuncMap
 
 // InitTemplateFunc initiates template helper functions
 func InitTemplateFunc(externalURL *url.URL) {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
 	tmplFunc = textTpl.FuncMap{
 		/* Strings */
 
@@ -91,6 +94,24 @@ func InitTemplateFunc(externalURL *url.URL) {
 		// alias for https://golang.org/pkg/strings/#ToLower
 		"toLower": strings.ToLower,
 
+		// stripPort splits string into host and port, then returns only host.
+		"stripPort": func(hostPort string) string {
+			host, _, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				return hostPort
+			}
+			return host
+		},
+
+		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
+		"parseDuration": func(d string) (float64, error) {
+			ms, err := metricsql.DurationValue(d, 0)
+			if err != nil {
+				return 0, err
+			}
+			return float64(ms) / 1000, nil
+		},
+
 		/* Numbers */
 
 		// humanize converts given number to a human readable format

app/vmalert/recording.go

@@ -60,7 +60,7 @@ func (rr *RecordingRule) ID() uint64 {
 
 func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *RecordingRule {
 	rr := &RecordingRule{
-		Type:    cfg.Type,
+		Type:    group.Type,
 		RuleID:  cfg.ID,
 		Name:    cfg.Record,
 		Expr:    cfg.Expr,
@@ -68,9 +68,9 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
 		q: qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType:     &cfg.Type,
+			DataSourceType:     &group.Type,
 			EvaluationInterval: group.Interval,
-			ExtraLabels:        group.ExtraFilterLabels,
+			QueryParams:        group.Params,
 		}),
 	}
 

app/vmalert/utils/auth.go

@@ -10,7 +10,7 @@ func AuthConfig(baUser, baPass, baFile, bearerToken, bearerTokenFile string) (*p
 	if baUser != "" || baPass != "" || baFile != "" {
 		baCfg = &promauth.BasicAuthConfig{
 			Username:     baUser,
-			Password:     baPass,
+			Password:     promauth.NewSecret(baPass),
 			PasswordFile: baFile,
 		}
 	}

app/vmalert/vmalert_cluster.excalidraw

@@ -0,0 +1,525 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 794,
+      "versionNonce": 1855937036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 659,
+      "versionNonce": 247957684,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347948032,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1670,
+      "versionNonce": 2021681972,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 750.3317260742188,
+      "y": 226.5509033203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348054411
+    },
+    {
+      "type": "text",
+      "version": 1311,
+      "versionNonce": 1283453068,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 759.2862243652344,
+      "y": 238.68240356445312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert:8480",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 897,
+      "versionNonce": 1983434892,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 61.744873046875,
+      "y": 224.9600830078125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638348077724
+    },
+    {
+      "type": "text",
+      "version": 719,
+      "versionNonce": 457402292,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 70.69937133789062,
+      "y": 237.33523559570312,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 152,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect:8481",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1098,
+      "versionNonce": 1480603788,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 371.7434387207031,
+      "y": 398.50787353515625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348083348
+    },
+    {
+      "type": "text",
+      "version": 864,
+      "versionNonce": 1115813900,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 393.73809814453125,
+      "y": 410.5976257324219,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348009180,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2405,
+      "versionNonce": 959767732,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 556.6860961914062,
+      "y": 252.2582773408825,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 184.9195556640625,
+      "height": 1.6022679018915937,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348054411,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.0344528515859526,
+        "gap": 10.57574462890625
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.039393828258510157,
+        "gap": 8.72607421875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          184.9195556640625,
+          -1.6022679018915937
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 1173,
+      "versionNonce": 1248255756,
+      "isDeleted": false,
+      "id": "wRO0q9xKPHc8e8XPPsQWh",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 406.0439244722469,
+      "y": 246.80533728741074,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 157.86774649373126,
+      "height": 0.1417938392881979,
+      "seed": 656189364,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348077725,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.13736472619498497,
+        "gap": 16.306295254315614
+      },
+      "endBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.013200835330936087,
+        "gap": 14.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -157.86774649373126,
+          0.1417938392881979
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 557,
+      "versionNonce": 995289780,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 552.4987182617188,
+      "y": 212.27996826171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 76,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348059525,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist alerts state\n\n\nand recording rules",
+      "baseline": 72,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 803,
+      "versionNonce": 1576507444,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 290.34130859375,
+      "y": 210.56927490234375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1471,
+      "versionNonce": 1361321140,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 484.18669893674246,
+      "y": 302.3424013553929,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 1.0484739253853945,
+      "height": 84.72775855671654,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348083348,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.010768924644894236,
+        "gap": 14.650750964767894
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.051051952959743775,
+        "gap": 11.437713623046818
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          1.0484739253853945,
+          84.72775855671654
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 576,
+      "versionNonce": 2112088460,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 530.5612182617188,
+      "y": 318.60687255859375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348023735,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 480,
+      "versionNonce": 1835119500,
+      "isDeleted": false,
+      "id": "ff5OkfgmkKLifS13_TFj3",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 291.37474060058594,
+      "y": 261.4861297607422,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 19,
+      "seed": 264004620,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347948032,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "restore state",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_ha.excalidraw

@@ -0,0 +1,911 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 906,
+      "versionNonce": 448716468,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 302.2676696777344,
+      "y": 275.59356689453125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "OTKoeHmKtqCxFArDbY-sP"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 762,
+      "versionNonce": 223660724,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 318.0538024902344,
+      "y": 296.6778106689453,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2"
+      ],
+      "updated": 1638355102070,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert1",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 2067,
+      "versionNonce": 817347852,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 329.6426086425781,
+      "y": 90.3275146484375,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA",
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355058507
+    },
+    {
+      "type": "text",
+      "version": 1717,
+      "versionNonce": 2040797452,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 349.01177978515625,
+      "y": 102.45901489257812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "m9_BptFOFxbV2sS_xJDu2",
+        "S_dOHQrhGmu8SFJzobJK7"
+      ],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1553,
+      "versionNonce": 39381044,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 173.67257690429688,
+      "y": 470.5100402832031,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "OTKoeHmKtqCxFArDbY-sP",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355221749
+    },
+    {
+      "type": "text",
+      "version": 1309,
+      "versionNonce": 1112872844,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 189.667236328125,
+      "y": 482.59979248046875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040938,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager1:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 835,
+      "versionNonce": 2036483596,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 525.83837890625,
+      "y": 147.33470153808594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 777,
+      "versionNonce": 990468492,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 617.0690307617188,
+      "y": 376.9822692871094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638355227582,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1112,
+      "versionNonce": 999136652,
+      "isDeleted": false,
+      "id": "mIu-d0lmShCxzMLD5iA_p",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 492.29505920410156,
+      "y": 272.3052215576172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 756416780,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "S_dOHQrhGmu8SFJzobJK7",
+        "XhPgLRBk-YhWAFcSQi9TJ",
+        "Ar-hcDLlzVSoTPs2MaywO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 970,
+      "versionNonce": 430760500,
+      "isDeleted": false,
+      "id": "ZqIR6SaLNDQl8s0zZbdnE",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 508.08119201660156,
+      "y": 293.38946533203125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 1477404084,
+      "groupIds": [
+        "jbBot-UNdMoWy2jPXC1u5"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355105020,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalertN",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "qZFezRGU4Chwxgvb2451t",
+      "type": "line",
+      "x": 449.93653869628906,
+      "y": 306.30010986328125,
+      "width": 19.48321533203125,
+      "height": 0.3865966796875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1833426828,
+      "version": 69,
+      "versionNonce": 871948300,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          19.48321533203125,
+          0.3865966796875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": null,
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": null
+    },
+    {
+      "type": "rectangle",
+      "version": 1649,
+      "versionNonce": 93981708,
+      "isDeleted": false,
+      "id": "gNHiZJKo0ap69ALDobtZ-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 518.5596466064453,
+      "y": 471.4539489746094,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 454422412,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "fsGFp4NW4JlrCdF0HR3uA",
+        "Ar-hcDLlzVSoTPs2MaywO",
+        "D6fkQH1E_MFbCuL697ArO"
+      ],
+      "updated": 1638355153683
+    },
+    {
+      "type": "text",
+      "version": 1412,
+      "versionNonce": 75038348,
+      "isDeleted": false,
+      "id": "O-zgjZBvt4RI1PrkBNQnb",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 534.3148040771484,
+      "y": 483.543701171875,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 211,
+      "height": 24,
+      "seed": 1026268980,
+      "groupIds": [
+        "fEAIeQ0DxLnI_rPlKPZqW"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355040939,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanagerN:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "m9_BptFOFxbV2sS_xJDu2",
+      "type": "arrow",
+      "x": 378.57185562792466,
+      "y": 262.1596984863281,
+      "width": 79.96146539019026,
+      "height": 111.53411865234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 141224884,
+      "version": 521,
+      "versionNonce": 283254540,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          79.96146539019026,
+          -111.53411865234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.23996018586441184,
+        "gap": 13.433868408203125
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": -0.14207100087207922,
+        "gap": 15.550811767578125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "S_dOHQrhGmu8SFJzobJK7",
+      "type": "arrow",
+      "x": 558.1990515577129,
+      "y": 263.4271240234375,
+      "width": 88.18940317574186,
+      "height": 114.15780639648438,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1004789812,
+      "version": 314,
+      "versionNonce": 1259171724,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355105019,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -88.18940317574186,
+          -114.15780639648438
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4315094049079832,
+        "gap": 8.878097534179688
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "focus": 0.14840834302306677,
+        "gap": 14.194549560546875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "fsGFp4NW4JlrCdF0HR3uA",
+      "type": "arrow",
+      "x": 356.6613630516658,
+      "y": 354.95416259765625,
+      "width": 278.74351860741064,
+      "height": 106.90020751953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 712384692,
+      "version": 334,
+      "versionNonce": 266493324,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355102070,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          278.74351860741064,
+          106.90020751953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.7721210277890177,
+        "gap": 7.228485107421875
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4493598001028791,
+        "gap": 9.599578857421875
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "OTKoeHmKtqCxFArDbY-sP",
+      "type": "arrow",
+      "x": 361.02563221226757,
+      "y": 356.2252197265625,
+      "width": 95.3594006000182,
+      "height": 105.52130126953125,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1992252596,
+      "version": 466,
+      "versionNonce": 472553100,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355221749,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -95.3594006000182,
+          105.52130126953125
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.39325294425055324,
+        "gap": 8.499542236328125
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.400636314282374,
+        "gap": 8.763519287109375
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "XhPgLRBk-YhWAFcSQi9TJ",
+      "type": "arrow",
+      "x": 561.7189961327852,
+      "y": 349.238037109375,
+      "width": 266.271510370216,
+      "height": 112.383544921875,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 588869260,
+      "version": 453,
+      "versionNonce": 584723212,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355150131,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -266.271510370216,
+          112.383544921875
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": -0.7084007026732048,
+        "gap": 4.8007049560546875
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": -0.4180430111580123,
+        "gap": 8.888458251953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "Ar-hcDLlzVSoTPs2MaywO",
+      "type": "arrow",
+      "x": 557.5999880535809,
+      "y": 352.71795654296875,
+      "width": 112.87813113656136,
+      "height": 106.3255615234375,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1508782476,
+      "version": 331,
+      "versionNonce": 1137561908,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355153683,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          112.87813113656136,
+          106.3255615234375
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "mIu-d0lmShCxzMLD5iA_p",
+        "focus": 0.4358123206211808,
+        "gap": 8.280624389648438
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": 0.4783744286829653,
+        "gap": 12.410430908203125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "WqHnv9_g3SdkLZuy4WSHa",
+      "type": "text",
+      "x": 186.5891876220703,
+      "y": 523.145263671875,
+      "width": 272,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1310147468,
+      "version": 281,
+      "versionNonce": 949157044,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "text": "Alertmanagers in cluster mode",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "id": "yA0kgvdF71wZbHJ7Cg2p8",
+      "type": "rectangle",
+      "x": 159.41685485839844,
+      "y": 440.7666015625,
+      "width": 625.9590148925781,
+      "height": 110.13494873046878,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1465986996,
+      "version": 366,
+      "versionNonce": 196040844,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939
+    },
+    {
+      "id": "D6fkQH1E_MFbCuL697ArO",
+      "type": "arrow",
+      "x": 422.4853057861328,
+      "y": 494.8779132311229,
+      "width": 90.5517578125,
+      "height": 0.12427004064892344,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 2025208204,
+      "version": 316,
+      "versionNonce": 2144102156,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638355040939,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          90.5517578125,
+          0.12427004064892344
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.08064204025505255,
+        "gap": 8.706283569335938
+      },
+      "endBinding": {
+        "elementId": "gNHiZJKo0ap69ALDobtZ-",
+        "focus": -0.05976220061952895,
+        "gap": 5.5225830078125
+      },
+      "startArrowhead": "arrow",
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 463,
+      "versionNonce": 2118914484,
+      "isDeleted": false,
+      "id": "p4EDvSKPqZzfM_SReybeq",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 274.6086883544922,
+      "y": 56.77886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 394,
+      "height": 19,
+      "seed": 1214462348,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355074872,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VictoriaMetrics with deduplication enabled",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 712,
+      "versionNonce": 737833612,
+      "isDeleted": false,
+      "id": "YfNFeOucMo8BJk2P2JRex",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 229.77923583984375,
+      "y": 227.84378051757812,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 448.5542297363281,
+      "height": 134.06329345703122,
+      "seed": 932207756,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355133244
+    },
+    {
+      "type": "text",
+      "version": 427,
+      "versionNonce": 1458527796,
+      "isDeleted": false,
+      "id": "oldM7Q_aJtpWd2jXQ0iKf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 238.9442901611328,
+      "y": 230.862548828125,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 225,
+      "height": 38,
+      "seed": 2131899572,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638355259358,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "vmalerts with identical \nconfigurations",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_multicluster.excalidraw

@@ -0,0 +1,915 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 791,
+      "versionNonce": 48874036,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 289.6802978515625,
+      "y": 399.3895568847656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp",
+        "pD9DcILMxa6GaR1U5YyMO",
+        "HPEwr85wL4IedW0AgdArp",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 658,
+      "versionNonce": 1653816076,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 311.22686767578125,
+      "y": 420.4738006591797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347796775,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 802,
+      "versionNonce": 995326644,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.05322265625,
+      "y": 228.65371704101562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 624,
+      "versionNonce": 707755700,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 640.7635803222656,
+      "y": 241.02886962890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701075,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "text",
+      "version": 802,
+      "versionNonce": 1974403340,
+      "isDeleted": false,
+      "id": "ia2QzZNl_tuvfY3ymLjyJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 401.241943359375,
+      "y": 342.4627990722656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 178,
+      "height": 38,
+      "seed": 157304972,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347863144,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute aggregating\nrecording rules",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "jrFZeFNsxlss7IsEZpA-J",
+      "type": "rectangle",
+      "x": 594.1509857177734,
+      "y": 192.09194946289062,
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 48379060,
+      "version": 665,
+      "versionNonce": 617456652,
+      "isDeleted": false,
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347763716
+    },
+    {
+      "id": "6ibhLp94HJFIdfP5HCEv8",
+      "type": "text",
+      "x": 609.7205657958984,
+      "y": 199.81723022460938,
+      "width": 225,
+      "height": 19,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1053164852,
+      "version": 256,
+      "versionNonce": 538595124,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347761092,
+      "text": "VM cluster with raw data",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 15
+    },
+    {
+      "type": "rectangle",
+      "version": 914,
+      "versionNonce": 1576666164,
+      "isDeleted": false,
+      "id": "R5v-yZCVJ97BkJqr0Qb7H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 807.5664215087891,
+      "y": 287.8628387451172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1794212620,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "bZXA8PH9gYu-clotqJ4f7",
+        "MGdu6GDIPNBAaEUr0Gt-a"
+      ],
+      "updated": 1638347737174
+    },
+    {
+      "type": "text",
+      "version": 725,
+      "versionNonce": 267455500,
+      "isDeleted": false,
+      "id": "pWRC_smX7TuOI8_8UrA4H",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 840.0209197998047,
+      "y": 300.2379913330078,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 421856180,
+      "groupIds": [
+        "BJNOAY1MY3Evr9B3qQtHf"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 843,
+      "versionNonce": 1244193972,
+      "isDeleted": false,
+      "id": "EqROOfYulSPsZm7ovxfQN",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 604.9091339111328,
+      "y": 345.2847442626953,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 2043521972,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347718537
+    },
+    {
+      "type": "text",
+      "version": 676,
+      "versionNonce": 143370892,
+      "isDeleted": false,
+      "id": "ddQH1nnmT7HbKW7Xmv4zx",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 642.6199798583984,
+      "y": 357.90354919433594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 335223180,
+      "groupIds": [
+        "ls6uq-W9bbVBM_UxAuyba"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "bZXA8PH9gYu-clotqJ4f7"
+      ],
+      "updated": 1638347701076,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "id": "bZXA8PH9gYu-clotqJ4f7",
+      "type": "arrow",
+      "x": 785.2667708463345,
+      "y": 367.2342882272049,
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1153029388,
+      "version": 420,
+      "versionNonce": 1726025228,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347704644,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "EqROOfYulSPsZm7ovxfQN",
+        "focus": 0.5247890891198189,
+        "gap": 8.36404562660789
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": -0.6931056940383433,
+        "gap": 9.943033572650961
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "arrow",
+      "version": 528,
+      "versionNonce": 1908259468,
+      "isDeleted": false,
+      "id": "MGdu6GDIPNBAaEUr0Gt-a",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 781.4456641707259,
+      "y": 248.777857603323,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1695061516,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347737174,
+      "startBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "R5v-yZCVJ97BkJqr0Qb7H",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 883,
+      "versionNonce": 845352076,
+      "isDeleted": false,
+      "id": "4pW8hvBu3bo1eMtFvg_gS",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 601.6520690917969,
+      "y": 473.7677536010742,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1678097076,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 703,
+      "versionNonce": 519371956,
+      "isDeleted": false,
+      "id": "w_i2hO06oLa0bWbyAfFzU",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 639.3624267578125,
+      "y": 486.14290618896484,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 340753036,
+      "groupIds": [
+        "3kSpFrIN3kg4jwjDKpNWw"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmselect",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 745,
+      "versionNonce": 879581324,
+      "isDeleted": false,
+      "id": "U5U-67wL5fPwvzlxOAF5A",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 592.7498321533203,
+      "y": 437.2059860229492,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 397.1286010742188,
+      "height": 209.62742614746097,
+      "seed": 912540724,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "pD9DcILMxa6GaR1U5YyMO"
+      ],
+      "updated": 1638347776748
+    },
+    {
+      "type": "text",
+      "version": 345,
+      "versionNonce": 1628116148,
+      "isDeleted": false,
+      "id": "0IGVrvICMVeZp2RCaqTeP",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "dotted",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 608.3194122314453,
+      "y": 444.93126678466797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 291,
+      "height": 19,
+      "seed": 68700428,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347784373,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "VM cluster with aggregated data",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 996,
+      "versionNonce": 2112461580,
+      "isDeleted": false,
+      "id": "9QMf0HXPE1W4M9S-zMJVO",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 806.1652679443359,
+      "y": 532.9768753051758,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 523607476,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "he-SpFjCxEQEWpWny2kKP",
+        "-pjrKo16rOsasM8viZPJ-",
+        "9WBH34em4CdU2OwzqYIl5",
+        "5W90eDBjtfZvkSuQTG0Iw"
+      ],
+      "updated": 1638347777173
+    },
+    {
+      "type": "text",
+      "version": 804,
+      "versionNonce": 1660832052,
+      "isDeleted": false,
+      "id": "WOrOD5vn6EtMUQRJomt3-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 838.6197662353516,
+      "y": 545.3520278930664,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 105,
+      "height": 24,
+      "seed": 1321420684,
+      "groupIds": [
+        "Eb2kWfz3ZWBu8cNul0h_c"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmstorage",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 925,
+      "versionNonce": 2052807604,
+      "isDeleted": false,
+      "id": "4dtJZpXEUxSK3biwFG7vd",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 603.5079803466797,
+      "y": 590.3987808227539,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 171.99359130859375,
+      "height": 44.74725341796875,
+      "seed": 1738524468,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "9WBH34em4CdU2OwzqYIl5",
+        "EyecK0YM9Cc8T6ju-nTOc"
+      ],
+      "updated": 1638347812431
+    },
+    {
+      "type": "text",
+      "version": 756,
+      "versionNonce": 2040967820,
+      "isDeleted": false,
+      "id": "rAbyooo-X08-86qjoK0WR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 641.2188262939453,
+      "y": 603.0175857543945,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 94,
+      "height": 24,
+      "seed": 948598284,
+      "groupIds": [
+        "punXEDFtHkSpcd9seAkZj"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "9WBH34em4CdU2OwzqYIl5"
+      ],
+      "updated": 1638347776748,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vminsert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 660,
+      "versionNonce": 1560678196,
+      "isDeleted": false,
+      "id": "9WBH34em4CdU2OwzqYIl5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 783.8656172818814,
+      "y": 612.3483247872634,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 99.87819315552736,
+      "height": 24.681162491468,
+      "seed": 1141424308,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": 0.5247890891198189,
+        "gap": 8.364045626608004
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": -0.6931056940383404,
+        "gap": 9.943033572650847
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          99.87819315552736,
+          -24.681162491468
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 768,
+      "versionNonce": 1653264948,
+      "isDeleted": false,
+      "id": "5W90eDBjtfZvkSuQTG0Iw",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 780.0445106062728,
+      "y": 493.8918941633816,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 104.0940537386266,
+      "height": 27.60697400233846,
+      "seed": 1852298380,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638347777173,
+      "startBinding": {
+        "elementId": "4pW8hvBu3bo1eMtFvg_gS",
+        "focus": -0.5921495247360469,
+        "gap": 6.398850205882127
+      },
+      "endBinding": {
+        "elementId": "9QMf0HXPE1W4M9S-zMJVO",
+        "focus": 0.7021471697457312,
+        "gap": 11.478007139455713
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          104.0940537386266,
+          27.60697400233846
+        ]
+      ]
+    },
+    {
+      "id": "HPEwr85wL4IedW0AgdArp",
+      "type": "arrow",
+      "x": 423.70701599121094,
+      "y": 428.34434509277344,
+      "width": 179.54901123046875,
+      "height": 182.9937744140625,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 389863732,
+      "version": 47,
+      "versionNonce": 1364399028,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347805500,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          179.54901123046875,
+          -182.9937744140625
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.6700023593531782,
+        "gap": 10.266586303710938
+      },
+      "endBinding": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "id": "EyecK0YM9Cc8T6ju-nTOc",
+      "type": "arrow",
+      "x": 424.7585906982422,
+      "y": 441.12806701660156,
+      "width": 174.29449462890625,
+      "height": 170.56607055664062,
+      "angle": 0,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 981082124,
+      "version": 92,
+      "versionNonce": 1261546252,
+      "isDeleted": false,
+      "boundElementIds": null,
+      "updated": 1638347812431,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          174.29449462890625,
+          170.56607055664062
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.6826568395144794,
+        "gap": 11.318161010742188
+      },
+      "endBinding": {
+        "elementId": "4dtJZpXEUxSK3biwFG7vd",
+        "focus": -0.8207814548026305,
+        "gap": 4.45489501953125
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 775,
+      "versionNonce": 95530764,
+      "isDeleted": false,
+      "id": "o-yIJ_WZzVubhbVODvhcC",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 407.27012634277344,
+      "y": 489.33091735839844,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 141,
+      "height": 19,
+      "seed": 775116812,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "wRO0q9xKPHc8e8XPPsQWh"
+      ],
+      "updated": 1638347824876,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist results",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/vmalert_single.excalidraw

@@ -0,0 +1,354 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 795,
+      "versionNonce": 1195460364,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 422.3502197265625,
+      "y": 215.55953979492188,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "miEbzHxOPXe4PEYvXiJp5",
+        "rcmiQfIWtfbTTlwxqr1sl",
+        "P-dpWlSTtnsux-zr5oqgF",
+        "oAToSPttH7aWoD_AqXGFX",
+        "Bpy5by47XGKB4yS99ZkuA",
+        "wRO0q9xKPHc8e8XPPsQWh",
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 660,
+      "versionNonce": 1034125236,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 443.89678955078125,
+      "y": 236.64378356933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle"
+    },
+    {
+      "type": "rectangle",
+      "version": 1690,
+      "versionNonce": 236788620,
+      "isDeleted": false,
+      "id": "dd52BjHfPMPRji9Tws7U-",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 344.9837951660156,
+      "y": 64.64248657226562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 269.336669921875,
+      "height": 44.74725341796875,
+      "seed": 1779959692,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "Bpy5by47XGKB4yS99ZkuA"
+      ],
+      "updated": 1638348176044
+    },
+    {
+      "type": "text",
+      "version": 1331,
+      "versionNonce": 945335476,
+      "isDeleted": false,
+      "id": "9TEzv0sVCHAkc46ou0oNF",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 364.35296630859375,
+      "y": 76.77398681640625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 234,
+      "height": 24,
+      "seed": 1617178804,
+      "groupIds": [
+        "2Lijjn3PwPQW_8KrcDmdu"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "victoriametrics:8428",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "rectangle",
+      "version": 1099,
+      "versionNonce": 671872012,
+      "isDeleted": false,
+      "id": "8-XFSbd6Zw96EUSJbJXZv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 360.6495666503906,
+      "y": 382.2536315917969,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 240.10644531249997,
+      "height": 44.74725341796875,
+      "seed": 99322124,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633
+    },
+    {
+      "type": "text",
+      "version": 865,
+      "versionNonce": 635839156,
+      "isDeleted": false,
+      "id": "GUs816aggGqUSdoEsSmea",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 382.64422607421875,
+      "y": 394.3433837890625,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 199,
+      "height": 24,
+      "seed": 1194745268,
+      "groupIds": [
+        "6obQBPHIfExBKfejeLLVO"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "alertmanager:9093",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 2444,
+      "versionNonce": 361351692,
+      "isDeleted": false,
+      "id": "Bpy5by47XGKB4yS99ZkuA",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.0465703465111,
+      "y": 204.98379516601562,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.7962088410123442,
+      "height": 86.86798095703125,
+      "seed": 357577356,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348176045,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "gap": 10.57574462890625,
+        "focus": 0.0344528515859526
+      },
+      "endBinding": {
+        "elementId": "dd52BjHfPMPRji9Tws7U-",
+        "gap": 8.72607421875,
+        "focus": -0.039393828258510157
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.7962088410123442,
+          -86.86798095703125
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 640,
+      "versionNonce": 1892951180,
+      "isDeleted": false,
+      "id": "RbVSa4PnOgAMtzoKb-DhW",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 500.609619140625,
+      "y": 134.11544799804688,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 188,
+      "height": 95,
+      "seed": 1989838604,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [],
+      "updated": 1638348163424,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "execute rules,\npersist alerts \nand recording rules,\nrestore state\n",
+      "baseline": 91,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    },
+    {
+      "type": "arrow",
+      "version": 1472,
+      "versionNonce": 768565516,
+      "isDeleted": false,
+      "id": "sxEhnxlbT7ldlSsmHDUHp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 486.5245361328125,
+      "y": 300.5478957313172,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 0.3521007656264601,
+      "height": 70.26802223743277,
+      "seed": 1818348300,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElementIds": [],
+      "updated": 1638348116633,
+      "startBinding": {
+        "elementId": "E9Run6wCm2chQ6JHrmc_y",
+        "focus": 1.1925203824459496,
+        "gap": 11.77154541015625
+      },
+      "endBinding": {
+        "elementId": "8-XFSbd6Zw96EUSJbJXZv",
+        "focus": 0.0441077573536454,
+        "gap": 11.437713623046875
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          -0.3521007656264601,
+          70.26802223743277
+        ]
+      ]
+    },
+    {
+      "type": "text",
+      "version": 577,
+      "versionNonce": 1646003636,
+      "isDeleted": false,
+      "id": "E9Run6wCm2chQ6JHrmc_y",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 498.29608154296875,
+      "y": 298.6573791503906,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 122,
+      "height": 38,
+      "seed": 1836541708,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElementIds": [
+        "sxEhnxlbT7ldlSsmHDUHp"
+      ],
+      "updated": 1638348116633,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "send alert \nnotifications",
+      "baseline": 34,
+      "textAlign": "left",
+      "verticalAlign": "top"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/web.go

@@ -68,7 +68,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/api/v1/alerts":
@@ -77,7 +77,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
 	case "/-/reload":
@@ -102,7 +102,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 				httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
 				return true
 			}
-			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			w.Header().Set("Content-Type", "application/json")
 			w.Write(data)
 			return true
 		}

app/vmalert/web.qtpl

@@ -54,10 +54,10 @@
                  {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
                 <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
                 <p class="fs-6 fw-lighter">{%s g.File %}</p>
-                {% if len(g.ExtraFilterLabels) > 0 %}
-                    <div class="fs-6 fw-lighter">Extra filter labels
-                    {% for k, v := range g.ExtraFilterLabels %}
-                            <span class="float-left badge bg-primary">{%s k %}={%s v %}</span>
+                {% if len(g.Params) > 0 %}
+                    <div class="fs-6 fw-lighter">Extra params
+                    {% for _, param := range g.Params %}
+                            <span class="float-left badge bg-primary">{%s param %}</span>
                     {% endfor %}
                     </div>
                 {% endif %}

app/vmalert/web.qtpl.go

@@ -211,22 +211,18 @@ func StreamListGroups(qw422016 *qt422016.Writer, groups []APIGroup) {
 			qw422016.N().S(`</p>
                 `)
 //line app/vmalert/web.qtpl:57
-			if len(g.ExtraFilterLabels) > 0 {
+			if len(g.Params) > 0 {
 //line app/vmalert/web.qtpl:57
 				qw422016.N().S(`
-                    <div class="fs-6 fw-lighter">Extra filter labels
+                    <div class="fs-6 fw-lighter">Extra params
                     `)
 //line app/vmalert/web.qtpl:59
-				for k, v := range g.ExtraFilterLabels {
+				for _, param := range g.Params {
 //line app/vmalert/web.qtpl:59
 					qw422016.N().S(`
                             <span class="float-left badge bg-primary">`)
 //line app/vmalert/web.qtpl:60
-					qw422016.E().S(k)
-//line app/vmalert/web.qtpl:60
-					qw422016.N().S(`=`)
-//line app/vmalert/web.qtpl:60
-					qw422016.E().S(v)
+					qw422016.E().S(param)
 //line app/vmalert/web.qtpl:60
 					qw422016.N().S(`</span>
                     `)
@@ -986,7 +982,7 @@ func badgeState(state string) string {
 func streambadgeRestored(qw422016 *qt422016.Writer) {
 //line app/vmalert/web.qtpl:303
 	qw422016.N().S(`
-<span class="badge bg-warning text-dark" title="Alert state was restored after reload from remote storage">restored</span>
+<span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
 `)
 //line app/vmalert/web.qtpl:305
 }

app/vmalert/web_types.go

@@ -23,16 +23,16 @@ type APIAlert struct {
 
 // APIGroup represents Group for WEB view
 type APIGroup struct {
-	Name              string             `json:"name"`
-	Type              string             `json:"type"`
-	ID                string             `json:"id"`
-	File              string             `json:"file"`
-	Interval          string             `json:"interval"`
-	Concurrency       int                `json:"concurrency"`
-	ExtraFilterLabels map[string]string  `json:"extra_filter_labels"`
-	Labels            map[string]string  `json:"labels,omitempty"`
-	AlertingRules     []APIAlertingRule  `json:"alerting_rules"`
-	RecordingRules    []APIRecordingRule `json:"recording_rules"`
+	Name           string             `json:"name"`
+	Type           string             `json:"type"`
+	ID             string             `json:"id"`
+	File           string             `json:"file"`
+	Interval       string             `json:"interval"`
+	Concurrency    int                `json:"concurrency"`
+	Params         []string           `json:"params"`
+	Labels         map[string]string  `json:"labels,omitempty"`
+	AlertingRules  []APIAlertingRule  `json:"alerting_rules"`
+	RecordingRules []APIRecordingRule `json:"recording_rules"`
 }
 
 // APIAlertingRule represents AlertingRule for WEB view

app/vmauth/README.md

@@ -3,7 +3,7 @@
 `vmauth` is a simple auth proxy, router and [load balancer](#load-balancing) for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
 It reads auth credentials from `Authorization` http header ([Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) and `Bearer token` is supported),
 matches them against configs pointed by [-auth.config](#auth-config) command-line flag and proxies incoming HTTP requests to the configured per-user `url_prefix` on successful match.
-
+The `-auth.config` can point to either local file or to http url.
 
 ## Quick start
 
@@ -26,12 +26,10 @@ Pass `-help` to `vmauth` in order to see all the supported command-line flags wi
 Feel free [contacting us](mailto:info@victoriametrics.com) if you need customized auth proxy for VictoriaMetrics with the support of LDAP, SSO, RBAC, SAML,
 accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.com/vmgateway.html).
 
-
 ## Load balancing
 
 Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls. In the latter case `vmauth` balances load among the configured urls in a round-robin manner. This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
 
-
 ## Auth config
 
 `-auth.config` is represented in the following simple `yml` format:
@@ -43,6 +41,7 @@ Each `url_prefix` in the [-auth.config](#auth-config) may contain either a singl
 users:
   # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
@@ -117,12 +116,12 @@ users:
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
     headers:
-    - "X-Scope-OrgID: abc"```
+    - "X-Scope-OrgID: abc"
+```
 
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
 
-
 ## Security
 
 Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:
@@ -140,21 +139,27 @@ Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termi
 
 It is recommended protecting `/-/reload` endpoint with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
 
-
 ## Monitoring
 
 `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
 either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
 
+`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
+
+```yml
+users:
+- username: "secret_user"
+  name: "foobar"
+  # other config options here
+```
 
 ## How to build from sources
 
 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmauth` is located in `vmutils-*` archives there.
 
-
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmauth` binary and puts it into the `bin` folder.
 
@@ -177,7 +182,6 @@ by setting it via `<ROOT_IMAGE>` environment variable. For example, the followin
 ROOT_IMAGE=scratch make package-vmauth
 ```
 
-
 ## Profiling
 
 `vmauth` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
@@ -198,7 +202,6 @@ The command for collecting CPU profile waits for 30 seconds before returning.
 
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
 
-
 ## Advanced usage
 
 Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
@@ -211,7 +214,7 @@ vmauth authenticates and authorizes incoming requests and proxies them to Victor
 See the docs at https://docs.victoriametrics.com/vmauth.html .
 
   -auth.config string
-    	Path to auth config. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
+    	Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
   -enableTCP6
     	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
@@ -239,7 +242,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -httpListenAddr string
     	TCP address to listen for http connections (default ":8427")
   -logInvalidAuthTokens
-    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmagent_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
+    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -262,9 +265,9 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-    	Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -pprofAuthKey string
-    	Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -reloadAuthKey string
     	Auth key for /-/reload http endpoint. It must be passed as authKey=...
   -tls

app/vmauth/auth_config.go

@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"regexp"
@@ -14,6 +13,7 @@ import (
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/metrics"
@@ -21,8 +21,8 @@ import (
 )
 
 var (
-	authConfigPath = flag.String("auth.config", "", "Path to auth config. See https://docs.victoriametrics.com/vmauth.html "+
-		"for details on the format of this auth config")
+	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config")
 )
 
 // AuthConfig represents auth config.
@@ -32,6 +32,7 @@ type AuthConfig struct {
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
+	Name        string     `yaml:"name,omitempty"`
 	BearerToken string     `yaml:"bearer_token,omitempty"`
 	Username    string     `yaml:"username,omitempty"`
 	Password    string     `yaml:"password,omitempty"`
@@ -236,9 +237,9 @@ var authConfigWG sync.WaitGroup
 var stopCh chan struct{}
 
 func readAuthConfig(path string) (map[string]*UserInfo, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := fs.ReadFileOrHTTP(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read %q: %w", path, err)
+		return nil, err
 	}
 	m, err := parseAuthConfig(data)
 	if err != nil {
@@ -275,9 +276,12 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 		if byUsername[ui.Username] {
 			return nil, fmt.Errorf("duplicate username found; username: %q", ui.Username)
 		}
-		authToken := getAuthToken(ui.BearerToken, ui.Username, ui.Password)
-		if byAuthToken[authToken] != nil {
-			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", authToken, ui.BearerToken, ui.Username)
+		at1, at2 := getAuthTokens(ui.BearerToken, ui.Username, ui.Password)
+		if byAuthToken[at1] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at1)
+		}
+		if byAuthToken[at2] != nil {
+			return nil, fmt.Errorf("duplicate auth token found for bearer_token=%q, username=%q: %q", ui.BearerToken, ui.Username, at2)
 		}
 		if ui.URLPrefix != nil {
 			if err := ui.URLPrefix.sanitize(); err != nil {
@@ -299,21 +303,41 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 			return nil, fmt.Errorf("missing `url_prefix`")
 		}
 		if ui.BearerToken != "" {
+			name := "bearer_token"
+			if ui.Name != "" {
+				name = ui.Name
+			}
 			if ui.Password != "" {
 				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 			}
-			ui.requests = metrics.GetOrCreateCounter(`vmauth_user_requests_total{username="bearer_token"}`)
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byBearerToken[ui.BearerToken] = true
 		}
 		if ui.Username != "" {
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, ui.Username))
+			name := ui.Username
+			if ui.Name != "" {
+				name = ui.Name
+			}
+			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
 			byUsername[ui.Username] = true
 		}
-		byAuthToken[authToken] = ui
+		byAuthToken[at1] = ui
+		byAuthToken[at2] = ui
 	}
 	return byAuthToken, nil
 }
 
+func getAuthTokens(bearerToken, username, password string) (string, string) {
+	if bearerToken != "" {
+		// Accept the bearerToken as Basic Auth username with empty password
+		at1 := getAuthToken(bearerToken, "", "")
+		at2 := getAuthToken("", bearerToken, "")
+		return at1, at2
+	}
+	at := getAuthToken("", username, password)
+	return at, at
+}
+
 func getAuthToken(bearerToken, username, password string) string {
 	if bearerToken != "" {
 		return "Bearer " + bearerToken

app/vmauth/auth_config_test.go

@@ -290,6 +290,32 @@ users:
 				},
 			},
 		},
+		getAuthToken("", "foo", ""): {
+			BearerToken: "foo",
+			URLMap: []URLMap{
+				{
+					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
+				},
+			},
+		},
 	})
 }
 

app/vmauth/example_config.yml

@@ -4,6 +4,7 @@
 users:
   # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+  # Requests with the Basic Auth username=XXXX are proxied to http://localhost:8428 as well.
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 

app/vmauth/main.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -108,7 +109,7 @@ func proxyRequest(w http.ResponseWriter, r *http.Request) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	reverseProxy.ServeHTTP(w, r)
+	getReverseProxy().ServeHTTP(w, r)
 }
 
 var (
@@ -117,29 +118,42 @@ var (
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
 )
 
-var reverseProxy = &httputil.ReverseProxy{
-	Director: func(r *http.Request) {
-		targetURL := r.Header.Get("vm-target-url")
-		target, err := url.Parse(targetURL)
-		if err != nil {
-			logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
-		}
-		r.URL = target
-	},
-	Transport: func() *http.Transport {
-		tr := http.DefaultTransport.(*http.Transport).Clone()
-		// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
-		tr.DisableCompression = true
-		// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
-		tr.ForceAttemptHTTP2 = false
-		tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
-		if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
-			tr.MaxIdleConns = tr.MaxIdleConnsPerHost
-		}
-		return tr
-	}(),
-	FlushInterval: time.Second,
-	ErrorLog:      logger.StdErrorLogger(),
+var (
+	reverseProxy     *httputil.ReverseProxy
+	reverseProxyOnce sync.Once
+)
+
+func getReverseProxy() *httputil.ReverseProxy {
+	reverseProxyOnce.Do(initReverseProxy)
+	return reverseProxy
+}
+
+// initReverseProxy must be called after flag.Parse(), since it uses command-line flags.
+func initReverseProxy() {
+	reverseProxy = &httputil.ReverseProxy{
+		Director: func(r *http.Request) {
+			targetURL := r.Header.Get("vm-target-url")
+			target, err := url.Parse(targetURL)
+			if err != nil {
+				logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
+			}
+			r.URL = target
+		},
+		Transport: func() *http.Transport {
+			tr := http.DefaultTransport.(*http.Transport).Clone()
+			// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
+			tr.DisableCompression = true
+			// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
+			tr.ForceAttemptHTTP2 = false
+			tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
+			if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+				tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+			}
+			return tr
+		}(),
+		FlushInterval: time.Second,
+		ErrorLog:      logger.StdErrorLogger(),
+	}
 }
 
 func usage() {

app/vmbackup/README.md

@@ -6,11 +6,11 @@ Supported storage systems for backups:
 
 * [GCS](https://cloud.google.com/storage/). Example: `gs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
-* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/en/pacific/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`
 
-`vmbackup` supports incremental and full backups. Incremental backups created automatically if the destination path already contains data from the previous backup.
-Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+`vmbackup` supports incremental and full backups. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be sped up with `-origin` pointing to an already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
 data between the existing backup and new backup. It saves time and costs on data transfer.
 
 Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
@@ -34,7 +34,7 @@ vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-
 ```
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
-  There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
+  There is no need to stop VictoriaMetrics for creating backups since they are performed from immutable [instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
 * `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). `vmbackup` can create the snapshot on itself if `-snapshot.createURL` command-line flag is set to an url for creating snapshots. In this case `-snapshotName` flag isn't needed.
 * `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
@@ -54,7 +54,7 @@ It saves time and network bandwidth costs by performing server-side copy for the
 
 ### Incremental backups
 
-Incremental backups performed if `-dst` points to an already existing backup. In this case only new data uploaded to remote storage.
+Incremental backups are performed if `-dst` points to an already existing backup. In this case only new data is uploaded to remote storage.
 It saves time and network bandwidth costs when working with big backups:
 
 ```
@@ -87,7 +87,7 @@ Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
 This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
 or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
 
-Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
+Do not forget to remove old snapshots and backups when they are no longer needed in order to save storage costs.
 
 See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager.html) for automating smart backups.
 
@@ -97,19 +97,19 @@ See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager
 The backup algorithm is the following:
 
 1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
-2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
-3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
-4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+2. Determine which files in `-dst` are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+3. Determine which files in `-snapshotName` are missing in `-dst`. These are usually small new files and bigger merged files.
+4. Determine which files from step 3 exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
    These are usually the biggest and the oldest files, which are shared between backups.
 5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
 
-The algorithm splits source files into 100 MB chunks in the backup. Each chunk stored as a separate file in the backup.
+The algorithm splits source files into 1 GiB chunks in the backup. Each chunk is stored as a separate file in the backup.
 Such splitting minimizes the amounts of data to re-transfer after temporary errors.
 
 `vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
 
 - All the files in the snapshot are immutable.
-- Old files periodically merged into new files.
+- Old files are periodically merged into new files.
 - Smaller files have higher probability to be merged.
 - Consecutive snapshots share many identical files.
 
@@ -185,12 +185,32 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
   -dst string
     	Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
     	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -enableTCP6
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
     	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
     	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
+  -http.maxGracefulShutdownDuration duration
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+  -httpAuth.password string
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+    	TCP address for exporting metrics at /metrics page (default ":8420")
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -213,8 +233,14 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -origin string
     	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -s3ForcePathStyle
+    	Prefixing endpoint with bucket name when set false, true by default. (default true)
   -snapshot.createURL string
     	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set
   -snapshot.deleteURL string
@@ -223,6 +249,12 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
     	Name for the snapshot to backup. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set
   -storageDataPath string
     	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
   -version
     	Show VictoriaMetrics version
 ```
@@ -235,7 +267,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmbackup` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmbackup` binary and puts it into the `bin` folder.
 

app/vmbackup/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmbackup/snapshot"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
@@ -14,10 +15,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 var (
+	httpListenAddr    = flag.String("httpListenAddr", ":8420", "TCP address for exporting metrics at /metrics page")
 	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
 	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set")
 	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+
@@ -70,6 +73,8 @@ func main() {
 		}()
 	}
 
+	go httpserver.Serve(*httpListenAddr, nil)
+
 	srcFS, err := newSrcFS()
 	if err != nil {
 		logger.Fatalf("%s", err)
@@ -94,6 +99,13 @@ func main() {
 	srcFS.MustStop()
 	dstFS.MustStop()
 	originFS.MustStop()
+
+	startTime := time.Now()
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http server for metrics: %s", err)
+	}
+	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func usage() {

app/vmbackupmanager/README.md

@@ -1,6 +1,6 @@
 ## vmbackupmanager
 
-***vmbackupmanager is a part of [enterprise package](https://victoriametrics.com/enterprise.html). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
+***vmbackupmanager is a part of [enterprise package](https://victoriametrics.com/products/enterprise/). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
 
 The VictoriaMetrics backup manager automates regular backup procedures. It supports the following backup intervals: **hourly**, **daily**, **weekly** and **monthly**. Multiple backup intervals may be configured simultaneously. I.e. the backup manager creates hourly backups every hour, while it creates daily backups every day, etc. Backup manager must have read access to the storage data, so best practice is to install it on the same machine (or as a sidecar) where the storage node is installed.
 The backup service makes a backup every hour and puts it to the latest folder and then copies data to the folders which represent the backup intervals (hourly, daily, weekly and monthly)
@@ -98,11 +98,11 @@ The result on the GCS bucket
 
 - The root folder
   
-  ![root](root.png)
+  ![root](vmbackupmanager_root_folder.png)
 
 - The latest folder
 
-  ![latest](latest.png)
+  ![latest](vmbackupmanager_latest_folder.png)
 
 ## Backup Retention Policy
 
@@ -117,7 +117,7 @@ Backup retention policy is controlled by:
 
 Let’s assume we have a backup manager collecting daily backups for the past 10 days.
 
-![daily](rp_daily_1.png)
+![daily](vmbackupmanager_rp_daily_1.png)
 
 
 We enable backup retention policy for backup manager by using following configuration:
@@ -143,4 +143,4 @@ info    app/vmbackupmanager/retention.go:106    daily backups to delete [daily/2
 
 The result on the GCS bucket. We see only 3 daily backups:
 
-![daily](rp_daily_2.png)
+![daily](vmbackupmanager_rp_daily_2.png)

app/vmctl/README.md

@@ -560,6 +560,15 @@ results such as `average`, `rate`, etc.
  If multiple labels needs to be added, set flag for each label, for example, `--vm-extra-label label1=value1 --vm-extra-label label2=value2`.
  If timeseries already have label, that must be added with `--vm-extra-label` flag, flag has priority and will override label value from timeseries.
 
+### Rate limiting
+
+Limiting the rate of data transfer could help to reduce pressure on disk or on destination database.
+The rate limit may be set in bytes-per-second via `--vm-rate-limit` flag.
+
+Please note, you can also use [vmagent](https://docs.victoriametrics.com/vmagent.html) 
+as a proxy between `vmctl` and destination with `-remoteWrite.rateLimit` flag enabled.
+
+
 ## How to build
 
 It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmctl` is located in `vmutils-*` archives there.
@@ -567,7 +576,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmctl` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmctl` binary and puts it into the `bin` folder.
 
@@ -596,7 +605,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 #### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmctl-arm` or `make vmctl-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmctl-arm` or `vmctl-arm64` binary respectively and puts it into the `bin` folder.
 

app/vmctl/flags.go

@@ -7,7 +7,8 @@ import (
 )
 
 const (
-	globalSilent = "s"
+	globalSilent  = "s"
+	globalVerbose = "verbose"
 )
 
 var (
@@ -17,6 +18,11 @@ var (
 			Value: false,
 			Usage: "Whether to run in silent mode. If set to true no confirmation prompts will appear.",
 		},
+		&cli.BoolFlag{
+			Name:  globalVerbose,
+			Value: false,
+			Usage: "Whether to enable verbosity in logs output.",
+		},
 	}
 )
 
@@ -30,7 +36,10 @@ const (
 	vmBatchSize          = "vm-batch-size"
 	vmSignificantFigures = "vm-significant-figures"
 	vmRoundDigits        = "vm-round-digits"
-	vmExtraLabel         = "vm-extra-label"
+
+	// also used in vm-native
+	vmExtraLabel = "vm-extra-label"
+	vmRateLimit  = "vm-rate-limit"
 )
 
 var (
@@ -95,6 +104,11 @@ var (
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
 				"will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
+		&cli.Int64Flag{
+			Name: vmRateLimit,
+			Usage: "Optional data transfer rate limit in bytes per second.\n" +
+				"By default the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
+		},
 	}
 )
 
@@ -155,7 +169,7 @@ var (
 		&cli.IntFlag{
 			Name:  otsdbQueryLimit,
 			Usage: "Result limit on meta queries to OpenTSDB (affects both metric name and tag value queries, recommended to use a value exceeding your largest series)",
-			Value: 100e3,
+			Value: 100e6,
 		},
 		&cli.BoolFlag{
 			Name:  otsdbMsecsTime,
@@ -354,6 +368,11 @@ var (
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
 				"will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
+		&cli.Int64Flag{
+			Name: vmRateLimit,
+			Usage: "Optional data transfer rate limit in bytes per second.\n" +
+				"By default the rate limit is disabled. It can be useful for limiting load on source or destination databases.",
+		},
 	}
 )
 

app/vmctl/influx.go

@@ -30,7 +30,7 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st
 	}
 }
 
-func (ip *influxProcessor) run(silent bool) error {
+func (ip *influxProcessor) run(silent, verbose bool) error {
 	series, err := ip.ic.Explore()
 	if err != nil {
 		return fmt.Errorf("explore query failed: %s", err)
@@ -70,7 +70,7 @@ func (ip *influxProcessor) run(silent bool) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("influx error: %s", infErr)
 		case vmErr := <-ip.im.Errors():
-			return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
 		case seriesCh <- s:
 		}
 	}
@@ -80,7 +80,9 @@ func (ip *influxProcessor) run(silent bool) error {
 	ip.im.Close()
 	// drain import errors channel
 	for vmErr := range ip.im.Errors() {
-		return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+		if vmErr.Err != nil {
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
+		}
 	}
 	bar.Finish()
 	log.Println("Import finished!")

app/vmctl/limiter/limiter.go

@@ -0,0 +1,53 @@
+package limiter
+
+import (
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+)
+
+// NewLimiter creates a Limiter object
+// for the given perSecondLimit
+func NewLimiter(perSecondLimit int64) *Limiter {
+	return &Limiter{perSecondLimit: perSecondLimit}
+}
+
+// Limiter controls the amount of budget
+// that can be spent according to configured perSecondLimit
+type Limiter struct {
+	perSecondLimit int64
+
+	// mu protects budget and deadline from concurrent access.
+	mu sync.Mutex
+
+	// The current budget. It is increased by perSecondLimit every second.
+	budget int64
+
+	// The next deadline for increasing the budget by perSecondLimit
+	deadline time.Time
+}
+
+// Register blocks for amount of time
+// needed to process the given dataLen according
+// to the configured perSecondLimit.
+func (l *Limiter) Register(dataLen int) {
+	limit := l.perSecondLimit
+	if limit <= 0 {
+		return
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	for l.budget <= 0 {
+		if d := time.Until(l.deadline); d > 0 {
+			t := timerpool.Get(d)
+			<-t.C
+			timerpool.Put(t)
+		}
+		l.budget += limit
+		l.deadline = time.Now().Add(time.Second)
+	}
+	l.budget -= int64(dataLen)
+}

app/vmctl/limiter/writer.go

@@ -0,0 +1,37 @@
+package limiter
+
+import (
+	"io"
+)
+
+// NewWriteLimiter creates a new WriteLimiter object
+// for the give writer and Limiter.
+func NewWriteLimiter(w io.Writer, limiter *Limiter) *WriteLimiter {
+	return &WriteLimiter{
+		writer:  w,
+		limiter: limiter,
+	}
+}
+
+// WriteLimiter limits the amount of bytes written
+// per second via Write() method.
+// Must be created via NewWriteLimiter.
+type WriteLimiter struct {
+	writer  io.Writer
+	limiter *Limiter
+}
+
+// Close implements io.Closer
+// also calls Close for wrapped io.WriteCloser
+func (wl *WriteLimiter) Close() error {
+	if c, ok := wl.writer.(io.Closer); ok {
+		return c.Close()
+	}
+	return nil
+}
+
+// Write implements io.Writer
+func (wl *WriteLimiter) Write(p []byte) (n int, err error) {
+	wl.limiter.Register(len(p))
+	return wl.writer.Write(p)
+}

app/vmctl/main.go

@@ -18,6 +18,11 @@ import (
 )
 
 func main() {
+	var (
+		err      error
+		importer *vm.Importer
+	)
+
 	start := time.Now()
 	app := &cli.App{
 		Name:    "vmctl",
@@ -53,7 +58,7 @@ func main() {
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency))
-					return otsdbProcessor.run(c.Bool(globalSilent))
+					return otsdbProcessor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -82,14 +87,14 @@ func main() {
 					}
 
 					vmCfg := initConfigVM(c)
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
 
 					processor := newInfluxProcessor(influxClient, importer,
 						c.Int(influxConcurrency), c.String(influxMeasurementFieldSeparator))
-					return processor.run(c.Bool(globalSilent))
+					return processor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -100,7 +105,7 @@ func main() {
 					fmt.Println("Prometheus import mode")
 
 					vmCfg := initConfigVM(c)
-					importer, err := vm.NewImporter(vmCfg)
+					importer, err = vm.NewImporter(vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
@@ -123,7 +128,7 @@ func main() {
 						im: importer,
 						cc: c.Int(promConcurrency),
 					}
-					return pp.run(c.Bool(globalSilent))
+					return pp.run(c.Bool(globalSilent), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -138,6 +143,7 @@ func main() {
 					}
 
 					p := vmNativeProcessor{
+						rateLimit: c.Int64(vmRateLimit),
 						filter: filter{
 							match:     c.String(vmNativeFilterMatch),
 							timeStart: c.String(vmNativeFilterTimeStart),
@@ -166,12 +172,14 @@ func main() {
 	go func() {
 		<-c
 		fmt.Println("\r- Execution cancelled")
-		os.Exit(0)
+		if importer != nil {
+			importer.Close()
+		}
 	}()
 
-	err := app.Run(os.Args)
+	err = app.Run(os.Args)
 	if err != nil {
-		log.Fatal(err)
+		log.Println(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
 }
@@ -188,5 +196,6 @@ func initConfigVM(c *cli.Context) vm.Config {
 		SignificantFigures: c.Int(vmSignificantFigures),
 		RoundDigits:        c.Int(vmRoundDigits),
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
+		RateLimit:          c.Int64(vmRateLimit),
 	}
 }

app/vmctl/opentsdb.go

@@ -35,7 +35,7 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int) *otsdb
 	}
 }
 
-func (op *otsdbProcessor) run(silent bool) error {
+func (op *otsdbProcessor) run(silent, verbose bool) error {
 	log.Println("Loading all metrics from OpenTSDB for filters: ", op.oc.Filters)
 	var metrics []string
 	for _, filter := range op.oc.Filters {
@@ -111,7 +111,7 @@ func (op *otsdbProcessor) run(silent bool) error {
 					case otsdbErr := <-errCh:
 						return fmt.Errorf("opentsdb error: %s", otsdbErr)
 					case vmErr := <-op.im.Errors():
-						return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+						return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
 					case seriesCh <- queryObj{
 						Tr: tr, StartTime: startTime,
 						Series: series, Rt: opentsdb.RetentionMeta{
@@ -133,7 +133,9 @@ func (op *otsdbProcessor) run(silent bool) error {
 	}
 	op.im.Close()
 	for vmErr := range op.im.Errors() {
-		return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+		if vmErr.Err != nil {
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
+		}
 	}
 	log.Println("Import finished!")
 	log.Print(op.im.Stats())
@@ -143,7 +145,7 @@ func (op *otsdbProcessor) run(silent bool) error {
 func (op *otsdbProcessor) do(s queryObj) error {
 	start := s.StartTime - s.Tr.Start
 	end := s.StartTime - s.Tr.End
-	data, err := op.oc.GetData(s.Series, s.Rt, start, end)
+	data, err := op.oc.GetData(s.Series, s.Rt, start, end, op.oc.MsecsTime)
 	if err != nil {
 		return fmt.Errorf("failed to collect data for %v in %v:%v :: %v", s.Series, s.Rt, s.Tr, err)
 	}

app/vmctl/opentsdb/opentsdb.go

@@ -1,7 +1,6 @@
 package opentsdb
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -47,6 +46,7 @@ type Client struct {
 	Filters    []string
 	Normalize  bool
 	HardTS     int64
+	MsecsTime  bool
 }
 
 // Config contains fields required
@@ -83,12 +83,20 @@ type MetaResults struct {
 // Meta A meta object about a metric
 // only contain the tags/etc. and no data
 type Meta struct {
-	//tsuid  string
 	Metric string            `json:"metric"`
 	Tags   map[string]string `json:"tags"`
+	//tsuid  string
 }
 
-// Metric holds the time series data
+// OtsdbMetric is a single series in OpenTSDB's returned format
+type OtsdbMetric struct {
+	Metric        string
+	Tags          map[string]string
+	AggregateTags []string
+	Dps           map[int64]float64
+}
+
+// Metric holds the time series data in VictoriaMetrics format
 type Metric struct {
 	Metric     string
 	Tags       map[string]string
@@ -96,83 +104,6 @@ type Metric struct {
 	Values     []float64
 }
 
-// ExpressionOutput contains results from actual data queries
-type ExpressionOutput struct {
-	Outputs []qoObj     `json:"outputs"`
-	Query   interface{} `json:"query"`
-}
-
-// QoObj contains actual timeseries data from the returned data query
-type qoObj struct {
-	ID    string      `json:"id"`
-	Alias string      `json:"alias"`
-	Dps   [][]float64 `json:"dps"`
-	//dpsMeta interface{}
-	//meta    interface{}
-}
-
-// Expression objects format our data queries
-/*
-All of the following structs are to build a OpenTSDB expression object
-*/
-type Expression struct {
-	Time    timeObj     `json:"time"`
-	Filters []filterObj `json:"filters"`
-	Metrics []metricObj `json:"metrics"`
-	// this just needs to be an empty object, so the value doesn't matter
-	Expressions []int       `json:"expressions"`
-	Outputs     []outputObj `json:"outputs"`
-}
-
-type timeObj struct {
-	Start       int64  `json:"start"`
-	End         int64  `json:"end"`
-	Aggregator  string `json:"aggregator"`
-	Downsampler dSObj  `json:"downsampler"`
-}
-
-type dSObj struct {
-	Interval   string  `json:"interval"`
-	Aggregator string  `json:"aggregator"`
-	FillPolicy fillObj `json:"fillPolicy"`
-}
-
-type fillObj struct {
-	// we'll always hard-code to NaN here, so we don't need value
-	Policy string `json:"policy"`
-}
-
-type filterObj struct {
-	Tags []tagObj `json:"tags"`
-	ID   string   `json:"id"`
-}
-
-type tagObj struct {
-	Type    string `json:"type"`
-	Tagk    string `json:"tagk"`
-	Filter  string `json:"filter"`
-	GroupBy bool   `json:"groupBy"`
-}
-
-type metricObj struct {
-	ID         string  `json:"id"`
-	Metric     string  `json:"metric"`
-	Filter     string  `json:"filter"`
-	FillPolicy fillObj `json:"fillPolicy"`
-}
-
-type outputObj struct {
-	ID    string `json:"id"`
-	Alias string `json:"alias"`
-}
-
-/* End expression object structs */
-
-var (
-	exprOutput     = outputObj{ID: "a", Alias: "query"}
-	exprFillPolicy = fillObj{Policy: "nan"}
-)
-
 // FindMetrics discovers all metrics that OpenTSDB knows about (given a filter)
 // e.g. /api/suggest?type=metrics&q=system&max=100000
 func (c Client) FindMetrics(q string) ([]string, error) {
@@ -221,74 +152,119 @@ func (c Client) FindSeries(metric string) ([]Meta, error) {
 }
 
 // GetData actually retrieves data for a series at a specified time range
-func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64) (Metric, error) {
+// e.g. /api/query?start=1&end=200&m=sum:1m-avg-none:system.load5{host=host1}
+func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64, mSecs bool) (Metric, error) {
 	/*
-		Here we build the actual exp query we'll send to OpenTSDB
-
-		This is comprised of a number of different settings. We hard-code
-		a few to simplify the JSON object creation.
-		There are examples queries available, so not too much detail here...
+		First, build our tag string.
+		It's literally just key=value,key=value,...
 	*/
-	expr := Expression{}
-	expr.Outputs = []outputObj{exprOutput}
-	expr.Metrics = append(expr.Metrics, metricObj{ID: "a", Metric: series.Metric,
-		Filter: "f1", FillPolicy: exprFillPolicy})
-	expr.Time = timeObj{Start: start, End: end, Aggregator: rt.FirstOrder,
-		Downsampler: dSObj{Interval: rt.AggTime,
-			Aggregator: rt.SecondOrder,
-			FillPolicy: exprFillPolicy}}
-	var TagList []tagObj
+	tagStr := ""
 	for k, v := range series.Tags {
-		/*
-			every tag should be a literal_or because that's the closest to a full "==" that
-			this endpoint allows for
-		*/
-		TagList = append(TagList, tagObj{Type: "literal_or", Tagk: k,
-			Filter: v, GroupBy: true})
-	}
-	expr.Filters = append(expr.Filters, filterObj{ID: "f1", Tags: TagList})
-	// "expressions" is required in the query object or we get a 5xx, so force it to exist
-	expr.Expressions = make([]int, 0)
-	inputData, err := json.Marshal(expr)
-	if err != nil {
-		return Metric{}, fmt.Errorf("failed to marshal query JSON %s", err)
+		tagStr += fmt.Sprintf("%s=%s,", k, v)
 	}
+	// obviously we don't want trailing commas...
+	tagStr = strings.Trim(tagStr, ",")
 
-	q := fmt.Sprintf("%s/api/query/exp", c.Addr)
-	resp, err := http.Post(q, "application/json", bytes.NewBuffer(inputData))
+	/*
+		The aggregation policy should already be somewhat formatted:
+		FirstOrder (e.g. sum/avg/max/etc.)
+		SecondOrder (e.g. sum/avg/max/etc.)
+		AggTime	(e.g. 1m/10m/1d/etc.)
+		This will build into m=<FirstOrder>:<AggTime>-<SecondOrder>-none:
+		Or an example: m=sum:1m-avg-none
+	*/
+	aggPol := fmt.Sprintf("%s:%s-%s-none", rt.FirstOrder, rt.AggTime, rt.SecondOrder)
+
+	/*
+		Our actual query string:
+		Start and End are just timestamps
+		We then add the aggregation policy, the metric, and the tag set
+	*/
+	queryStr := fmt.Sprintf("start=%v&end=%v&m=%s:%s{%s}", start, end, aggPol,
+		series.Metric, tagStr)
+
+	q := fmt.Sprintf("%s/api/query?%s", c.Addr, queryStr)
+	resp, err := http.Get(q)
 	if err != nil {
 		return Metric{}, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
+	/*
+		There are three potential failures here, none of which should kill the entire
+		migration run:
+		1. bad response code
+		2. failure to read response body
+		3. bad format of response body
+	*/
 	if resp.StatusCode != 200 {
-		return Metric{}, fmt.Errorf("Bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
+		log.Println(fmt.Sprintf("bad response code from OpenTSDB query %v for %q...skipping", resp.StatusCode, q))
+		return Metric{}, nil
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return Metric{}, fmt.Errorf("could not retrieve series data from %q: %s", q, err)
+		log.Println("couldn't read response body from OpenTSDB query...skipping")
+		return Metric{}, nil
 	}
-	var output ExpressionOutput
+	var output []OtsdbMetric
 	err = json.Unmarshal(body, &output)
 	if err != nil {
-		return Metric{}, fmt.Errorf("failed to unmarshal response from %q: %s", q, err)
+		log.Println(fmt.Sprintf("couldn't marshall response body from OpenTSDB query (%s)...skipping", body))
+		return Metric{}, nil
 	}
-	if len(output.Outputs) < 1 {
+	/*
+		We expect results to look like:
+		[
+		  {
+			"metric": "zfs_filesystem.available",
+			"tags": {
+			  "rack": "6",
+			  "replica": "1",
+			  "host": "c7-bfyii-115",
+			  "pool": "dattoarray",
+			  "row": "c",
+			  "dc": "us-west-3",
+			  "group": "legonode"
+			},
+			"aggregateTags": [],
+			"dps": {
+			  "1626019200": 32490602877610.668,
+			  "1626033600": 32486439014058.668
+			}
+		  }
+		]
+		There are two things that could be bad here:
+			1. There are no actual stats returned (an empty array -> [])
+			2. There are aggregate tags in the results
+		An empty array doesn't cast to a OtsdbMetric struct well, and there's no reason to try, so we should just skip it
+		Because we're trying to migrate data without transformations, seeing aggregate tags could mean
+		we're dropping series on the floor.
+
+		In all "bad" cases, we don't end the migration, we just don't process that particular message
+	*/
+	if len(output) < 1 {
 		// no results returned...return an empty object without error
 		return Metric{}, nil
 	}
+	if len(output) > 1 {
+		// multiple series returned for a single query. We can't process this right, so...
+		return Metric{}, nil
+	}
+	if len(output[0].AggregateTags) > 0 {
+		// This failure means we've suppressed potential series somehow...
+		return Metric{}, nil
+	}
 	data := Metric{}
-	data.Metric = series.Metric
-	data.Tags = series.Tags
+	data.Metric = output[0].Metric
+	data.Tags = output[0].Tags
 	/*
 		We evaluate data for correctness before formatting the actual values
 		to skip a little bit of time if the series has invalid formatting
-
-		First step is to enforce Prometheus' data model
 	*/
 	data, err = modifyData(data, c.Normalize)
 	if err != nil {
-		return Metric{}, fmt.Errorf("invalid series data from %q: %s", q, err)
+		return Metric{}, nil
 	}
+
 	/*
 		Convert data from OpenTSDB's output format ([[ts,val],[ts,val]...])
 		to VictoriaMetrics format: {"timestamps": [ts,ts,ts...], "values": [val,val,val...]}
@@ -296,9 +272,13 @@ func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64) (
 		can be a float64, we have to initially cast _all_ objects that way
 		then convert the timestamp back to something reasonable.
 	*/
-	for _, tsobj := range output.Outputs[0].Dps {
-		data.Timestamps = append(data.Timestamps, int64(tsobj[0]))
-		data.Values = append(data.Values, tsobj[1])
+	for ts, val := range output[0].Dps {
+		if !mSecs {
+			data.Timestamps = append(data.Timestamps, ts*1000)
+		} else {
+			data.Timestamps = append(data.Timestamps, ts)
+		}
+		data.Values = append(data.Values, val)
 	}
 	return data, nil
 }
@@ -308,9 +288,13 @@ func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64) (
 func NewClient(cfg Config) (*Client, error) {
 	var retentions []Retention
 	offsetPrint := int64(time.Now().Unix())
+	// convert a number of days to seconds
+	offsetSecs := cfg.Offset * 24 * 60 * 60
 	if cfg.MsecsTime {
 		// 1000000 == Nanoseconds -> Milliseconds difference
 		offsetPrint = int64(time.Now().UnixNano() / 1000000)
+		// also bump offsetSecs to milliseconds
+		offsetSecs = offsetSecs * 1000
 	}
 	if cfg.HardTS > 0 {
 		/*
@@ -318,20 +302,16 @@ func NewClient(cfg Config) (*Client, error) {
 			Just present that if it is defined
 		*/
 		offsetPrint = cfg.HardTS
-	} else if cfg.Offset > 0 {
+	} else if offsetSecs > 0 {
 		/*
-			Our "offset" is the number of days we should step
+			Our "offset" is the number of days (in seconds) we should step
 			back before starting to scan for data
 		*/
-		if cfg.MsecsTime {
-			offsetPrint = offsetPrint - (cfg.Offset * 24 * 60 * 60 * 1000)
-		} else {
-			offsetPrint = offsetPrint - (cfg.Offset * 24 * 60 * 60)
-		}
+		offsetPrint = offsetPrint - offsetSecs
 	}
 	log.Println(fmt.Sprintf("Will collect data starting at TS %v", offsetPrint))
 	for _, r := range cfg.Retentions {
-		ret, err := convertRetention(r, cfg.Offset, cfg.MsecsTime)
+		ret, err := convertRetention(r, offsetSecs, cfg.MsecsTime)
 		if err != nil {
 			return &Client{}, fmt.Errorf("Couldn't parse retention %q :: %v", r, err)
 		}
@@ -344,6 +324,7 @@ func NewClient(cfg Config) (*Client, error) {
 		Filters:    cfg.Filters,
 		Normalize:  cfg.Normalize,
 		HardTS:     cfg.HardTS,
+		MsecsTime:  cfg.MsecsTime,
 	}
 	return client, nil
 }

app/vmctl/opentsdb/parser.go

@@ -87,6 +87,34 @@ func convertRetention(retention string, offset int64, msecTime bool) (Retention,
 	if len(chunks) != 3 {
 		return Retention{}, fmt.Errorf("invalid retention string: %q", retention)
 	}
+	queryLengthDuration, err := convertDuration(chunks[2])
+	if err != nil {
+		return Retention{}, fmt.Errorf("invalid ttl (second order) duration string: %q: %s", chunks[2], err)
+	}
+	// set ttl in milliseconds, unless we aren't using millisecond time in OpenTSDB...then use seconds
+	queryLength := queryLengthDuration.Milliseconds()
+	if !msecTime {
+		queryLength = queryLength / 1000
+	}
+	queryRange := queryLength
+	// bump by the offset so we don't look at empty ranges any time offset > ttl
+	queryLength += offset
+
+	// first/second order aggregations for queries defined in chunk 0...
+	aggregates := strings.Split(chunks[0], "-")
+	if len(aggregates) != 3 {
+		return Retention{}, fmt.Errorf("invalid aggregation string: %q", chunks[0])
+	}
+
+	aggTimeDuration, err := convertDuration(aggregates[1])
+	if err != nil {
+		return Retention{}, fmt.Errorf("invalid aggregation time duration string: %q: %s", aggregates[1], err)
+	}
+	aggTime := aggTimeDuration.Milliseconds()
+	if !msecTime {
+		aggTime = aggTime / 1000
+	}
+
 	rowLengthDuration, err := convertDuration(chunks[1])
 	if err != nil {
 		return Retention{}, fmt.Errorf("invalid row length (first order) duration string: %q: %s", chunks[1], err)
@@ -96,26 +124,35 @@ func convertRetention(retention string, offset int64, msecTime bool) (Retention,
 	if !msecTime {
 		rowLength = rowLength / 1000
 	}
-	ttlDuration, err := convertDuration(chunks[2])
-	if err != nil {
-		return Retention{}, fmt.Errorf("invalid ttl (second order) duration string: %q: %s", chunks[2], err)
+
+	var querySize int64
+	/*
+		The idea here is to ensure each individual query sent to OpenTSDB is *at least*
+		large enough to ensure no single query requests essentially 0 data.
+	*/
+	if rowLength > aggTime {
+		/*
+			We'll look at 2x the row size for each query we perform
+			This is a strange function, but the logic works like this:
+			1. we discover the "number" of ranges we should split the time range into
+			   This is found with queryRange / (rowLength * 4)...kind of a percentage query
+			2. we discover the actual size of each "chunk"
+			   This is second division step
+		*/
+		querySize = int64(queryRange / (queryRange / (rowLength * 4)))
+	} else {
+		/*
+			Unless the aggTime (how long a range of data we're requesting per individual point)
+			is greater than the row size. Then we'll need to use that to determine
+			how big each individual query should be
+		*/
+		querySize = int64(queryRange / (queryRange / (aggTime * 4)))
 	}
-	// set ttl in milliseconds, unless we aren't using millisecond time in OpenTSDB...then use seconds
-	ttl := ttlDuration.Milliseconds()
-	if !msecTime {
-		ttl = ttl / 1000
-	}
-	// bump by the offset so we don't look at empty ranges any time offset > ttl
-	ttl += offset
+
 	var timeChunks []TimeRange
 	var i int64
-	for i = offset; i <= ttl; i = i + rowLength {
-		timeChunks = append(timeChunks, TimeRange{Start: i + rowLength, End: i})
-	}
-	// first/second order aggregations for queries defined in chunk 0...
-	aggregates := strings.Split(chunks[0], "-")
-	if len(aggregates) != 3 {
-		return Retention{}, fmt.Errorf("invalid aggregation string: %q", chunks[0])
+	for i = offset; i <= queryLength; i = i + querySize {
+		timeChunks = append(timeChunks, TimeRange{Start: i + querySize, End: i})
 	}
 
 	ret := Retention{FirstOrder: aggregates[0],

app/vmctl/opentsdb/parser_test.go

@@ -8,7 +8,7 @@ func TestConvertRetention(t *testing.T) {
 	/*
 		2592000 seconds in 30 days
 		3600 in one hour
-		2592000 / 3600 = 720 individual query "ranges" should exist, plus one because time ranges can be weird
+		2592000 / 14400 = 180 individual query "ranges" should exist, plus one because time ranges can be weird
 		First order should == "sum"
 		Second order should == "avg"
 		AggTime should == "1m"
@@ -17,8 +17,8 @@ func TestConvertRetention(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Error parsing valid retention string: %v", err)
 	}
-	if len(res.QueryRanges) != 721 {
-		t.Fatalf("Found %v query ranges. Should have found 720", len(res.QueryRanges))
+	if len(res.QueryRanges) != 181 {
+		t.Fatalf("Found %v query ranges. Should have found 181", len(res.QueryRanges))
 	}
 	if res.FirstOrder != "sum" {
 		t.Fatalf("Incorrect first order aggregation %q. Should have been 'sum'", res.FirstOrder)

app/vmctl/opentsdb/testdata/exampleOutput.json

@@ -1,398 +0,0 @@
-{
-  "outputs": [
-    {
-      "id": "a",
-      "alias": "query",
-      "dps": [
-        [
-          1614099600000,
-          0.28
-        ],
-        [
-          1614099660000,
-          0.22
-        ],
-        [
-          1614099720000,
-          0.18
-        ],
-        [
-          1614099780000,
-          0.14
-        ],
-        [
-          1614099840000,
-          0.24
-        ],
-        [
-          1614099900000,
-          0.19
-        ],
-        [
-          1614099960000,
-          0.22
-        ],
-        [
-          1614100020000,
-          0.2
-        ],
-        [
-          1614100080000,
-          0.18
-        ],
-        [
-          1614100140000,
-          0.22
-        ],
-        [
-          1614100200000,
-          0.17
-        ],
-        [
-          1614100260000,
-          0.16
-        ],
-        [
-          1614100320000,
-          0.22
-        ],
-        [
-          1614100380000,
-          0.3
-        ],
-        [
-          1614100440000,
-          0.28
-        ],
-        [
-          1614100500000,
-          0.27
-        ],
-        [
-          1614100560000,
-          0.26
-        ],
-        [
-          1614100620000,
-          0.23
-        ],
-        [
-          1614100680000,
-          0.18
-        ],
-        [
-          1614100740000,
-          0.3
-        ],
-        [
-          1614100800000,
-          0.24
-        ],
-        [
-          1614100860000,
-          0.19
-        ],
-        [
-          1614100920000,
-          0.16
-        ],
-        [
-          1614100980000,
-          0.19
-        ],
-        [
-          1614101040000,
-          0.23
-        ],
-        [
-          1614101100000,
-          0.18
-        ],
-        [
-          1614101160000,
-          0.15
-        ],
-        [
-          1614101220000,
-          0.12
-        ],
-        [
-          1614101280000,
-          0.1
-        ],
-        [
-          1614101340000,
-          0.24
-        ],
-        [
-          1614101400000,
-          0.19
-        ],
-        [
-          1614101460000,
-          0.16
-        ],
-        [
-          1614101520000,
-          0.14
-        ],
-        [
-          1614101580000,
-          0.12
-        ],
-        [
-          1614101640000,
-          0.14
-        ],
-        [
-          1614101700000,
-          0.12
-        ],
-        [
-          1614101760000,
-          0.13
-        ],
-        [
-          1614101820000,
-          0.12
-        ],
-        [
-          1614101880000,
-          0.11
-        ],
-        [
-          1614101940000,
-          0.36
-        ],
-        [
-          1614102000000,
-          0.35
-        ],
-        [
-          1614102060000,
-          0.3
-        ],
-        [
-          1614102120000,
-          0.32
-        ],
-        [
-          1614102180000,
-          0.27
-        ],
-        [
-          1614102240000,
-          0.26
-        ],
-        [
-          1614102300000,
-          0.21
-        ],
-        [
-          1614102360000,
-          0.18
-        ],
-        [
-          1614102420000,
-          0.15
-        ],
-        [
-          1614102480000,
-          0.12
-        ],
-        [
-          1614102540000,
-          0.24
-        ],
-        [
-          1614102600000,
-          0.2
-        ],
-        [
-          1614102660000,
-          0.17
-        ],
-        [
-          1614102720000,
-          0.18
-        ],
-        [
-          1614102780000,
-          0.14
-        ],
-        [
-          1614102840000,
-          0.39
-        ],
-        [
-          1614102900000,
-          0.31
-        ],
-        [
-          1614102960000,
-          0.3
-        ],
-        [
-          1614103020000,
-          0.24
-        ],
-        [
-          1614103080000,
-          0.26
-        ],
-        [
-          1614103140000,
-          0.21
-        ],
-        [
-          1614103200000,
-          0.17
-        ],
-        [
-          1614103260000,
-          0.15
-        ],
-        [
-          1614103320000,
-          0.2
-        ],
-        [
-          1614103380000,
-          0.2
-        ],
-        [
-          1614103440000,
-          0.22
-        ],
-        [
-          1614103500000,
-          0.19
-        ],
-        [
-          1614103560000,
-          0.22
-        ],
-        [
-          1614103620000,
-          0.29
-        ],
-        [
-          1614103680000,
-          0.31
-        ],
-        [
-          1614103740000,
-          0.28
-        ],
-        [
-          1614103800000,
-          0.23
-        ]
-      ],
-      "dpsMeta": {
-        "firstTimestamp": 1614099600000,
-        "lastTimestamp": 1614103800000,
-        "setCount": 71,
-        "series": 1
-      },
-      "meta": [
-        {
-          "index": 0,
-          "metrics": [
-            "timestamp"
-          ]
-        },
-        {
-          "index": 1,
-          "metrics": [
-            "system.load5"
-          ],
-          "commonTags": {
-            "rack": "undef",
-            "host": "use1-mon-metrics-1",
-            "row": "undef",
-            "dc": "us-east-1",
-            "group": "monitoring"
-          },
-          "aggregatedTags": []
-        }
-      ]
-    }
-  ],
-  "query": {
-    "name": null,
-    "time": {
-      "start": "1h-ago",
-      "end": null,
-      "timezone": null,
-      "downsampler": {
-        "interval": "1m",
-        "aggregator": "avg",
-        "fillPolicy": {
-          "policy": "nan",
-          "value": "NaN"
-        }
-      },
-      "aggregator": "sum",
-      "rate": false
-    },
-    "filters": [
-      {
-        "id": "f1",
-        "tags": [
-          {
-            "tagk": "host",
-            "filter": "use1-mon-metrics-1",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "group",
-            "filter": "monitoring",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "dc",
-            "filter": "us-east-1",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "rack",
-            "filter": "undef",
-            "group_by": true,
-            "type": "literal_or"
-          },
-          {
-            "tagk": "row",
-            "filter": "undef",
-            "group_by": true,
-            "type": "literal_or"
-          }
-        ],
-        "explicitTags": false
-      }
-    ],
-    "metrics": [
-      {
-        "metric": "system.load5",
-        "id": "a",
-        "filter": "f1",
-        "aggregator": null,
-        "timeOffset": null,
-        "fillPolicy": {
-          "policy": "nan",
-          "value": "NaN"
-        }
-      }
-    ],
-    "expressions": [],
-    "outputs": [
-      {
-        "id": "a",
-        "alias": "query"
-      }
-    ]
-  }
-}

app/vmctl/opentsdb/testdata/exampleQuery.json

@@ -1,62 +0,0 @@
-{
-   "time": {
-       "start": "1h-ago",
-       "aggregator":"sum",
-       "downsampler": {
-           "interval": "1m",
-           "aggregator": "avg",
-           "fillPolicy": {
-               "policy": "nan"
-           }
-       }
-   },
-   "filters": [
-       {
-           "tags": [
-               {
-                   "type": "literal_or",
-                   "tagk": "host",
-                   "filter": "use1-mon-metrics-1",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "group",
-                   "filter": "monitoring",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "dc",
-                   "filter": "us-east-1",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "rack",
-                   "filter": "undef",
-                   "groupBy": true
-               },
-               {
-                   "type": "literal_or",
-                   "tagk": "row",
-                   "filter": "undef",
-                   "groupBy": true
-               }
-           ],
-           "id": "f1"
-       }
-   ],
-   "metrics": [
-       {
-           "id": "a",
-           "metric": "system.load5",
-           "filter": "f1",
-           "fillPolicy":{"policy":"nan"}
-       }
-   ],
-   "expressions": [],
-   "outputs":[
-     {"id":"a", "alias":"query"}
-   ]
-}

app/vmctl/prometheus.go

@@ -25,7 +25,7 @@ type prometheusProcessor struct {
 	cc int
 }
 
-func (pp *prometheusProcessor) run(silent bool) error {
+func (pp *prometheusProcessor) run(silent, verbose bool) error {
 	blocks, err := pp.cl.Explore()
 	if err != nil {
 		return fmt.Errorf("explore failed: %s", err)
@@ -66,7 +66,7 @@ func (pp *prometheusProcessor) run(silent bool) error {
 			return fmt.Errorf("prometheus error: %s", promErr)
 		case vmErr := <-pp.im.Errors():
 			close(blockReadersCh)
-			return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
 		case blockReadersCh <- br:
 		}
 	}
@@ -77,7 +77,9 @@ func (pp *prometheusProcessor) run(silent bool) error {
 	pp.im.Close()
 	// drain import errors channel
 	for vmErr := range pp.im.Errors() {
-		return fmt.Errorf("Import process failed: \n%s", wrapErr(vmErr))
+		if vmErr.Err != nil {
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
+		}
 	}
 	bar.Finish()
 	log.Println("Import finished!")

app/vmctl/utils.go

@@ -23,11 +23,29 @@ func prompt(question string) bool {
 	return false
 }
 
-func wrapErr(vmErr *vm.ImportError) error {
+func wrapErr(vmErr *vm.ImportError, verbose bool) error {
 	var errTS string
+	var maxTS, minTS int64
 	for _, ts := range vmErr.Batch {
-		errTS += fmt.Sprintf("%s for timestamps range %d - %d\n",
-			ts.String(), ts.Timestamps[0], ts.Timestamps[len(ts.Timestamps)-1])
+		if minTS < ts.Timestamps[0] || minTS == 0 {
+			minTS = ts.Timestamps[0]
+		}
+		if maxTS < ts.Timestamps[len(ts.Timestamps)-1] {
+			maxTS = ts.Timestamps[len(ts.Timestamps)-1]
+		}
+		if verbose {
+			errTS += fmt.Sprintf("%s for timestamps range %d - %d\n",
+				ts.String(), ts.Timestamps[0], ts.Timestamps[len(ts.Timestamps)-1])
+		}
 	}
-	return fmt.Errorf("%s with error: %s", errTS, vmErr.Err)
+	var verboseMsg string
+	if !verbose {
+		verboseMsg = "(enable `--verbose` output to get more details)"
+	}
+	if vmErr.Err == nil {
+		return fmt.Errorf("%s\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
+			vmErr.Err, minTS, maxTS, verboseMsg, errTS)
+	}
+	return fmt.Errorf("%s\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
+		vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 }

app/vmctl/vm/vm.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )
 
@@ -47,6 +48,9 @@ type Config struct {
 	RoundDigits int
 	// ExtraLabels that will be added to all imported series. Must be in label=value format.
 	ExtraLabels []string
+	// RateLimit defines a data transfer speed in bytes per second.
+	// Is applied to each worker (see Concurrency) independently.
+	RateLimit int64
 }
 
 // Importer performs insertion of timeseries
@@ -63,6 +67,8 @@ type Importer struct {
 	input  chan *TimeSeries
 	errors chan *ImportError
 
+	rl *limiter.Limiter
+
 	wg   sync.WaitGroup
 	once sync.Once
 
@@ -123,6 +129,7 @@ func NewImporter(cfg Config) (*Importer, error) {
 		compress:   cfg.Compress,
 		user:       cfg.User,
 		password:   cfg.Password,
+		rl:         limiter.NewLimiter(cfg.RateLimit),
 		close:      make(chan struct{}),
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
@@ -149,9 +156,11 @@ func NewImporter(cfg Config) (*Importer, error) {
 // ImportError is type of error generated
 // in case of unsuccessful import request
 type ImportError struct {
-	// The batch of timeseries that failed
+	// The batch of timeseries processed by importer at the moment
 	Batch []*TimeSeries
 	// The error that appeared during insert
+	// If err is nil - no error happened and Batch
+	// Is the latest delivered Batch.
 	Err error
 }
 
@@ -180,12 +189,13 @@ func (im *Importer) startWorker(batchSize, significantFigures, roundDigits int)
 	for {
 		select {
 		case <-im.close:
-			if err := im.Import(batch); err != nil {
-				im.errors <- &ImportError{
-					Batch: batch,
-					Err:   err,
-				}
+			exitErr := &ImportError{
+				Batch: batch,
 			}
+			if err := im.Import(batch); err != nil {
+				exitErr.Err = err
+			}
+			im.errors <- exitErr
 			return
 		case ts := <-im.input:
 			// init waitForBatch when first
@@ -301,12 +311,13 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 
 	w := io.Writer(pw)
 	if im.compress {
-		zw, err := gzip.NewWriterLevel(pw, 1)
+		zw, err := gzip.NewWriterLevel(w, 1)
 		if err != nil {
 			return fmt.Errorf("unexpected error when creating gzip writer: %s", err)
 		}
 		w = zw
 	}
+	w = limiter.NewWriteLimiter(w, im.rl)
 	bw := bufio.NewWriterSize(w, 16*1024)
 
 	var totalSamples, totalBytes int
@@ -321,8 +332,8 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	if err := bw.Flush(); err != nil {
 		return err
 	}
-	if im.compress {
-		err := w.(*gzip.Writer).Close()
+	if closer, ok := w.(io.Closer); ok {
+		err := closer.Close()
 		if err != nil {
 			return err
 		}

app/vmctl/vm_native.go

@@ -7,12 +7,15 @@ import (
 	"log"
 	"net/http"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/cheggaaa/pb/v3"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 )
 
 type vmNativeProcessor struct {
-	filter filter
+	filter    filter
+	rateLimit int64
 
 	dst *vmNativeClient
 	src *vmNativeClient
@@ -84,7 +87,12 @@ func (p *vmNativeProcessor) run() error {
 	bar := pb.ProgressBarTemplate(barTpl).Start64(0)
 	barReader := bar.NewProxyReader(exportReader)
 
-	_, err = io.Copy(pw, barReader)
+	w := io.Writer(pw)
+	if p.rateLimit > 0 {
+		rl := limiter.NewLimiter(p.rateLimit)
+		w = limiter.NewWriteLimiter(pw, rl)
+	}
+	_, err = io.Copy(w, barReader)
 	if err != nil {
 		return fmt.Errorf("failed to write into %q: %s", p.dst.addr, err)
 	}

app/vmgateway/README.md

@@ -1,6 +1,6 @@
 # vmgateway
 
-***vmgateway is a part of [enterprise package](https://victoriametrics.com/enterprise.html). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
+***vmgateway is a part of [enterprise package](https://victoriametrics.com/products/enterprise/). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
 
 
 <img alt="vmgateway" src="vmgateway-overview.jpeg">
@@ -14,7 +14,7 @@
   * Provides access by tenantID in the Cluster version
   * Allows for separate write/read/admin access to data
 
-`vmgateway` is included in our [enterprise packages](https://victoriametrics.com/enterprise.html).
+`vmgateway` is included in our [enterprise packages](https://victoriametrics.com/products/enterprise/).
 
 
 ## Access Control
@@ -36,6 +36,7 @@ jwt token must be in following format:
          "team": "dev",
          "project": "mobile"
       },
+      "extra_filters": ["{env~=\"prod|dev\",team!=\"test\"}"],
       "mode": 1
   }
 }
@@ -44,7 +45,8 @@ Where:
 - `exp` - required, expire time in unix_timestamp. If the token expires then `vmgateway` rejects the request.
 - `vm_access` - required, dict with claim info, minimum form: `{"vm_access": {"tenand_id": {}}`
 - `tenant_id` - optional, for cluster mode, routes requests to the corresponding tenant.
-- `extra_labels` - optional, key-value pairs for label filters added to the ingested or selected metrics.
+- `extra_labels` - optional, key-value pairs for label filters added to the ingested or selected metrics. Multiple filters are added with `and` operation. If defined, `extra_label` from original request removed.
+- `extra_filters` - optional, [series selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) added to the select query requests. Multiple selectors are added with `or` operation. If defined, `extra_filter` from original request removed.
 - `mode` - optional, access mode for api - read, write, or full. Supported values: 0 - full (default value), 1 - read, 2 - write.
 
 ## QuickStart

app/vminsert/main.go

@@ -42,7 +42,9 @@ var (
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
+	maxLabelValueLen       = flag.Int("maxLabelValueLen", 16*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
 )
 
 var (
@@ -56,6 +58,7 @@ var (
 func Init() {
 	relabel.Init()
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
+	storage.SetMaxLabelValueLen(*maxLabelValueLen)
 	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*graphiteListenAddr) > 0 {
@@ -164,26 +167,26 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{"valid":true}`)
 		return true
 	case "/datadog/api/v1/check_run":
 		datadogCheckRunRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/datadog/intake/":
 		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
 	case "/prometheus/targets", "/targets":
@@ -192,11 +195,19 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/prometheus/config", "/config":
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
 		promscrapeConfigRequests.Inc()
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)

app/vminsert/relabel/relabel.go

@@ -16,6 +16,7 @@ import (
 
 var (
 	relabelConfig = flag.String("relabelConfig", "", "Optional path to a file with relabeling rules, which are applied to all the ingested metrics. "+
+		"The path can point either to local file or to http url. "+
 		"See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal")
 	relabelDebug = flag.Bool("relabelDebug", false, "Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, "+
 		"then the metrics aren't sent to storage. This is useful for debugging the relabeling configs")

app/vmrestore/README.md

@@ -85,12 +85,32 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
     	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -customS3Endpoint string
     	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -enableTCP6
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
     	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
     	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+  -http.connTimeout duration
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+  -http.disableResponseCompression
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+  -http.idleConnTimeout duration
+    	Timeout for incoming idle http connections (default 1m0s)
+  -http.maxGracefulShutdownDuration duration
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+  -http.pathPrefix string
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+  -http.shutdownDelay duration
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+  -httpAuth.password string
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+  -httpAuth.username string
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+  -httpListenAddr string
+    	TCP address for exporting metrics at /metrics page (default ":8421")
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -113,12 +133,24 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
     	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+  -metricsAuthKey string
+    	Auth key for /metrics. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -pprofAuthKey string
+    	Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings
+  -s3ForcePathStyle
+    	Prefixing endpoint with bucket name when set false, true by default. (default true)
   -skipBackupCompleteCheck
     	Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file
   -src string
     	Source path with backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
   -storageDataPath string
     	Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir is synchronized with -src contents, i.e. it works like 'rsync --delete' (default "victoria-metrics-data")
+  -tls
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+  -tlsCertFile string
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+  -tlsKeyFile string
+    	Path to file with TLS key. Used only if -tls is set
   -version
     	Show VictoriaMetrics version
 ```
@@ -131,7 +163,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.
 2. Run `make vmrestore` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmrestore` binary and puts it into the `bin` folder.
 

app/vmrestore/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
@@ -11,11 +12,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 var (
-	src = flag.String("src", "", "Source path with backup on the remote storage. "+
+	httpListenAddr = flag.String("httpListenAddr", ":8421", "TCP address for exporting metrics at /metrics page")
+	src            = flag.String("src", "", "Source path with backup on the remote storage. "+
 		"Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
 	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
 		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir "+
@@ -33,6 +36,8 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
+	go httpserver.Serve(*httpListenAddr, nil)
+
 	srcFS, err := newSrcFS()
 	if err != nil {
 		logger.Fatalf("%s", err)
@@ -52,6 +57,13 @@ func main() {
 	}
 	srcFS.MustStop()
 	dstFS.MustStop()
+
+	startTime := time.Now()
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http server for metrics: %s", err)
+	}
+	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func usage() {

app/vmselect/graphite/metrics_api.go

@@ -456,7 +456,7 @@ const maxRegexpCacheSize = 10000
 
 func getContentType(jsonp string) string {
 	if jsonp == "" {
-		return "application/json; charset=utf-8"
+		return "application/json"
 	}
 	return "text/javascript; charset=utf-8"
 }

app/vmselect/graphite/tags_api.go

@@ -32,7 +32,7 @@ func TagsDelSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 	var row graphiteparser.Row
 	var tagsPool []graphiteparser.Tag
 	ct := startTime.UnixNano() / 1e6
-	etfs, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return fmt.Errorf("cannot setup tag filters: %w", err)
 	}
@@ -53,8 +53,8 @@ func TagsDelSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 				Value: []byte(tag.Value),
 			})
 		}
-		tfs = append(tfs, etfs...)
-		sq := storage.NewSearchQuery(0, ct, [][]storage.TagFilter{tfs})
+		tfss := joinTagFilterss(tfs, etfs)
+		sq := storage.NewSearchQuery(0, ct, tfss)
 		n, err := netstorage.DeleteSeries(sq, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot delete series for %q: %w", sq, err)
@@ -62,7 +62,7 @@ func TagsDelSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 		totalDeleted += n
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	if totalDeleted > 0 {
 		fmt.Fprintf(w, "true")
 	} else {
@@ -141,7 +141,7 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
 	// Return response
 	contentType := "text/plain; charset=utf-8"
 	if isJSONResponse {
-		contentType = "application/json; charset=utf-8"
+		contentType = "application/json"
 	}
 	w.Header().Set("Content-Type", contentType)
 	WriteTagsTagMultiSeriesResponse(w, canonicalPaths, isJSONResponse)
@@ -181,7 +181,7 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r
 	valuePrefix := r.FormValue("valuePrefix")
 	exprs := r.Form["expr"]
 	var tagValues []string
-	etfs, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return fmt.Errorf("cannot setup tag filters: %w", err)
 	}
@@ -266,7 +266,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *
 	tagPrefix := r.FormValue("tagPrefix")
 	exprs := r.Form["expr"]
 	var labels []string
-	etfs, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return fmt.Errorf("cannot setup tag filters: %w", err)
 	}
@@ -345,7 +345,7 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R
 	if len(exprs) == 0 {
 		return fmt.Errorf("expecting at least one `expr` query arg")
 	}
-	etfs, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return fmt.Errorf("cannot setup tag filters: %w", err)
 	}
@@ -362,7 +362,7 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R
 		paths = paths[:limit]
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteTagsFindSeriesResponse(bw, paths)
@@ -418,7 +418,7 @@ func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter
 		return err
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteTagValuesResponse(bw, tagName, tagValues)
@@ -449,7 +449,7 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er
 		return err
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteTagsResponse(bw, labels)
@@ -474,14 +474,14 @@ func getInt(r *http.Request, argName string) (int, error) {
 	return n, nil
 }
 
-func getSearchQueryForExprs(startTime time.Time, etfs []storage.TagFilter, exprs []string) (*storage.SearchQuery, error) {
+func getSearchQueryForExprs(startTime time.Time, etfs [][]storage.TagFilter, exprs []string) (*storage.SearchQuery, error) {
 	tfs, err := exprsToTagFilters(exprs)
 	if err != nil {
 		return nil, err
 	}
 	ct := startTime.UnixNano() / 1e6
-	tfs = append(tfs, etfs...)
-	sq := storage.NewSearchQuery(0, ct, [][]storage.TagFilter{tfs})
+	tfss := joinTagFilterss(tfs, etfs)
+	sq := storage.NewSearchQuery(0, ct, tfss)
 	return sq, nil
 }
 
@@ -524,3 +524,7 @@ func parseFilterExpr(s string) (*storage.TagFilter, error) {
 		IsRegexp:   isRegexp,
 	}, nil
 }
+
+func joinTagFilterss(tfs []storage.TagFilter, extraFilters [][]storage.TagFilter) [][]storage.TagFilter {
+	return searchutils.JoinTagFilterss([][]storage.TagFilter{tfs}, extraFilters)
+}

app/vmselect/main.go

@@ -200,7 +200,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	if strings.HasPrefix(path, "/functions") {
 		graphiteFunctionsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, "%s", `{}`)
 		return true
 	}
@@ -404,26 +404,26 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/rules", "/rules":
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
 		rulesRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
 		return true
 	case "/api/v1/alerts", "/alerts":
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
 		alertsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
 		return true
 	case "/api/v1/metadata":
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
 		metadataRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
 		return true
 	case "/api/v1/query_exemplars":
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars
 		queryExemplarsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
-		fmt.Fprintf(w, "%s", `{"status":"success","data":null}`)
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":[]}`)
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
 		deleteRequests.Inc()
@@ -459,7 +459,7 @@ func isGraphiteTagsPath(path string) bool {
 func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
 	logger.Warnf("error in %q: %s", httpserver.GetRequestURI(r), err)
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	statusCode := http.StatusUnprocessableEntity
 	var esc *httpserver.ErrorWithStatusCode
 	if errors.As(err, &esc) {

app/vmselect/netstorage/netstorage.go

@@ -468,7 +468,8 @@ func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.
 	if firstErr != nil {
 		return firstErr
 	}
-	mergeSortBlocks(dst, sbs)
+	dedupInterval := storage.GetDedupInterval()
+	mergeSortBlocks(dst, sbs, dedupInterval)
 	return nil
 }
 
@@ -489,7 +490,7 @@ var sbPool sync.Pool
 
 var metricRowsSkipped = metrics.NewCounter(`vm_metric_rows_skipped_total{name="vmselect"}`)
 
-func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
+func mergeSortBlocks(dst *Result, sbh sortBlocksHeap, dedupInterval int64) {
 	// Skip empty sort blocks, since they cannot be passed to heap.Init.
 	src := sbh
 	sbh = sbh[:0]
@@ -532,8 +533,7 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
 			putSortBlock(top)
 		}
 	}
-
-	timestamps, values := storage.DeduplicateSamples(dst.Timestamps, dst.Values)
+	timestamps, values := storage.DeduplicateSamples(dst.Timestamps, dst.Values, dedupInterval)
 	dedups := len(dst.Timestamps) - len(timestamps)
 	dedupsDuringSelect.Add(dedups)
 	dst.Timestamps = timestamps

app/vmselect/prometheus/prometheus.go

@@ -129,6 +129,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 	if err != nil {
 		return err
 	}
+	reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
 	deadline := searchutils.GetDeadlineForExport(r, startTime)
 	tagFilterss, err := getTagFilterssFromRequest(r)
 	if err != nil {
@@ -140,30 +141,58 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 	defer bufferedwriter.Put(bw)
 
 	resultsCh := make(chan *quicktemplate.ByteBuffer, cgroup.AvailableCPUs())
-	doneCh := make(chan error)
-	go func() {
-		err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
-			if err := bw.Error(); err != nil {
-				return err
-			}
-			if err := b.UnmarshalData(); err != nil {
-				return fmt.Errorf("cannot unmarshal block during export: %s", err)
-			}
-			xb := exportBlockPool.Get().(*exportBlock)
-			xb.mn = mn
-			xb.timestamps, xb.values = b.AppendRowsWithTimeRangeFilter(xb.timestamps[:0], xb.values[:0], tr)
-			if len(xb.timestamps) > 0 {
-				bb := quicktemplate.AcquireByteBuffer()
-				WriteExportCSVLine(bb, xb, fieldNames)
-				resultsCh <- bb
-			}
-			xb.reset()
-			exportBlockPool.Put(xb)
-			return nil
-		})
-		close(resultsCh)
-		doneCh <- err
-	}()
+	writeCSVLine := func(xb *exportBlock) {
+		if len(xb.timestamps) == 0 {
+			return
+		}
+		bb := quicktemplate.AcquireByteBuffer()
+		WriteExportCSVLine(bb, xb, fieldNames)
+		resultsCh <- bb
+	}
+	doneCh := make(chan error, 1)
+	if !reduceMemUsage {
+		rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
+		}
+		go func() {
+			err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+				if err := bw.Error(); err != nil {
+					return err
+				}
+				xb := exportBlockPool.Get().(*exportBlock)
+				xb.mn = &rs.MetricName
+				xb.timestamps = rs.Timestamps
+				xb.values = rs.Values
+				writeCSVLine(xb)
+				xb.reset()
+				exportBlockPool.Put(xb)
+				return nil
+			})
+			close(resultsCh)
+			doneCh <- err
+		}()
+	} else {
+		go func() {
+			err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
+				if err := bw.Error(); err != nil {
+					return err
+				}
+				if err := b.UnmarshalData(); err != nil {
+					return fmt.Errorf("cannot unmarshal block during export: %s", err)
+				}
+				xb := exportBlockPool.Get().(*exportBlock)
+				xb.mn = mn
+				xb.timestamps, xb.values = b.AppendRowsWithTimeRangeFilter(xb.timestamps[:0], xb.values[:0], tr)
+				writeCSVLine(xb)
+				xb.reset()
+				exportBlockPool.Put(xb)
+				return nil
+			})
+			close(resultsCh)
+			doneCh <- err
+		}()
+	}
 	// Consume all the data from resultsCh.
 	for bb := range resultsCh {
 		// Do not check for error in bw.Write, since this error is checked inside netstorage.ExportBlocks above.
@@ -283,11 +312,11 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	if start >= end {
 		end = start + defaultStep
 	}
-	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return err
 	}
-	if err := exportHandler(w, matches, etf, start, end, format, maxRowsPerLine, reduceMemUsage, deadline); err != nil {
+	if err := exportHandler(w, matches, etfs, start, end, format, maxRowsPerLine, reduceMemUsage, deadline); err != nil {
 		return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %w", matches, start, end, err)
 	}
 	return nil
@@ -295,7 +324,7 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 
 var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export"}`)
 
-func exportHandler(w http.ResponseWriter, matches []string, etf []storage.TagFilter, start, end int64, format string, maxRowsPerLine int, reduceMemUsage bool, deadline searchutils.Deadline) error {
+func exportHandler(w http.ResponseWriter, matches []string, etfs [][]storage.TagFilter, start, end int64, format string, maxRowsPerLine int, reduceMemUsage bool, deadline searchutils.Deadline) error {
 	writeResponseFunc := WriteExportStdResponse
 	writeLineFunc := func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
 		bb := quicktemplate.AcquireByteBuffer()
@@ -352,7 +381,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etf []storage.TagFil
 	if err != nil {
 		return err
 	}
-	tagFilterss = addEnforcedFiltersToTagFilterss(tagFilterss, etf)
+	tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
 
 	sq := storage.NewSearchQuery(start, end, tagFilterss)
 	w.Header().Set("Content-Type", contentType)
@@ -360,7 +389,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etf []storage.TagFil
 	defer bufferedwriter.Put(bw)
 
 	resultsCh := make(chan *quicktemplate.ByteBuffer, cgroup.AvailableCPUs())
-	doneCh := make(chan error)
+	doneCh := make(chan error, 1)
 	if !reduceMemUsage {
 		rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
 		if err != nil {
@@ -478,13 +507,13 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
-	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return err
 	}
 	matches := getMatchesFromRequest(r)
 	var labelValues []string
-	if len(matches) == 0 && len(etf) == 0 {
+	if len(matches) == 0 && len(etfs) == 0 {
 		if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
 			var err error
 			labelValues, err = netstorage.GetLabelValues(labelName, deadline)
@@ -527,13 +556,13 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 		if err != nil {
 			return err
 		}
-		labelValues, err = labelValuesWithMatches(labelName, matches, etf, start, end, deadline)
+		labelValues, err = labelValuesWithMatches(labelName, matches, etfs, start, end, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot obtain label values for %q, match[]=%q, start=%d, end=%d: %w", labelName, matches, start, end, err)
 		}
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteLabelValuesResponse(bw, labelValues)
@@ -543,7 +572,7 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 	return nil
 }
 
-func labelValuesWithMatches(labelName string, matches []string, etf []storage.TagFilter, start, end int64, deadline searchutils.Deadline) ([]string, error) {
+func labelValuesWithMatches(labelName string, matches []string, etfs [][]storage.TagFilter, start, end int64, deadline searchutils.Deadline) ([]string, error) {
 	tagFilterss, err := getTagFilterssFromMatches(matches)
 	if err != nil {
 		return nil, err
@@ -564,7 +593,7 @@ func labelValuesWithMatches(labelName string, matches []string, etf []storage.Ta
 	if start >= end {
 		end = start + defaultStep
 	}
-	tagFilterss = addEnforcedFiltersToTagFilterss(tagFilterss, etf)
+	tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
 	if len(tagFilterss) == 0 {
 		logger.Panicf("BUG: tagFilterss must be non-empty")
 	}
@@ -622,7 +651,7 @@ func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	if err != nil {
 		return fmt.Errorf(`cannot obtain label entries: %w`, err)
 	}
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteLabelsCountResponse(bw, labelEntries)
@@ -648,7 +677,7 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
-	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return err
 	}
@@ -679,18 +708,18 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 		topN = n
 	}
 	var status *storage.TSDBStatus
-	if len(matches) == 0 && len(etf) == 0 {
+	if len(matches) == 0 && len(etfs) == 0 {
 		status, err = netstorage.GetTSDBStatusForDate(deadline, date, topN)
 		if err != nil {
 			return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
 		}
 	} else {
-		status, err = tsdbStatusWithMatches(matches, etf, date, topN, deadline)
+		status, err = tsdbStatusWithMatches(matches, etfs, date, topN, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot obtain tsdb status with matches for date=%d, topN=%d: %w", date, topN, err)
 		}
 	}
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteTSDBStatusResponse(bw, status)
@@ -700,12 +729,12 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	return nil
 }
 
-func tsdbStatusWithMatches(matches []string, etf []storage.TagFilter, date uint64, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
+func tsdbStatusWithMatches(matches []string, etfs [][]storage.TagFilter, date uint64, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
 	tagFilterss, err := getTagFilterssFromMatches(matches)
 	if err != nil {
 		return nil, err
 	}
-	tagFilterss = addEnforcedFiltersToTagFilterss(tagFilterss, etf)
+	tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
 	if len(tagFilterss) == 0 {
 		logger.Panicf("BUG: tagFilterss must be non-empty")
 	}
@@ -731,13 +760,13 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
-	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return err
 	}
 	matches := getMatchesFromRequest(r)
 	var labels []string
-	if len(matches) == 0 && len(etf) == 0 {
+	if len(matches) == 0 && len(etfs) == 0 {
 		if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
 			var err error
 			labels, err = netstorage.GetLabels(deadline)
@@ -778,13 +807,13 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		if err != nil {
 			return err
 		}
-		labels, err = labelsWithMatches(matches, etf, start, end, deadline)
+		labels, err = labelsWithMatches(matches, etfs, start, end, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot obtain labels for match[]=%q, start=%d, end=%d: %w", matches, start, end, err)
 		}
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteLabelsResponse(bw, labels)
@@ -794,7 +823,7 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	return nil
 }
 
-func labelsWithMatches(matches []string, etf []storage.TagFilter, start, end int64, deadline searchutils.Deadline) ([]string, error) {
+func labelsWithMatches(matches []string, etfs [][]storage.TagFilter, start, end int64, deadline searchutils.Deadline) ([]string, error) {
 	tagFilterss, err := getTagFilterssFromMatches(matches)
 	if err != nil {
 		return nil, err
@@ -802,7 +831,7 @@ func labelsWithMatches(matches []string, etf []storage.TagFilter, start, end int
 	if start >= end {
 		end = start + defaultStep
 	}
-	tagFilterss = addEnforcedFiltersToTagFilterss(tagFilterss, etf)
+	tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
 	if len(tagFilterss) == 0 {
 		logger.Panicf("BUG: tagFilterss must be non-empty")
 	}
@@ -860,7 +889,7 @@ func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %w", err)
 	}
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteSeriesCountResponse(bw, n)
@@ -911,7 +940,7 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		if err != nil {
 			return fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Content-Type", "application/json")
 		bw := bufferedwriter.Get(w)
 		defer bufferedwriter.Put(bw)
 		resultsCh := make(chan *quicktemplate.ByteBuffer)
@@ -936,7 +965,7 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	resultsCh := make(chan *quicktemplate.ByteBuffer)
@@ -999,7 +1028,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	if len(query) > maxQueryLen.N {
 		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
 	}
-	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return err
 	}
@@ -1014,7 +1043,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		if end < start {
 			end = start
 		}
-		if err := exportHandler(w, []string{childQuery}, etf, start, end, "promapi", 0, false, deadline); err != nil {
+		if err := exportHandler(w, []string{childQuery}, etfs, start, end, "promapi", 0, false, deadline); err != nil {
 			return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %w", childQuery, start, end, err)
 		}
 		queryDuration.UpdateDuration(startTime)
@@ -1030,7 +1059,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		start -= offset
 		end := start
 		start = end - window
-		if err := queryRangeHandler(startTime, w, childQuery, start, end, step, r, ct, etf); err != nil {
+		if err := queryRangeHandler(startTime, w, childQuery, start, end, step, r, ct, etfs); err != nil {
 			return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", childQuery, start, end, step, err)
 		}
 		queryDuration.UpdateDuration(startTime)
@@ -1048,14 +1077,14 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		queryOffset = 0
 	}
 	ec := promql.EvalConfig{
-		Start:              start,
-		End:                start,
-		Step:               step,
-		QuotedRemoteAddr:   httpserver.GetQuotedRemoteAddr(r),
-		Deadline:           deadline,
-		LookbackDelta:      lookbackDelta,
-		RoundDigits:        getRoundDigits(r),
-		EnforcedTagFilters: etf,
+		Start:               start,
+		End:                 start,
+		Step:                step,
+		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
+		Deadline:            deadline,
+		LookbackDelta:       lookbackDelta,
+		RoundDigits:         getRoundDigits(r),
+		EnforcedTagFilterss: etfs,
 	}
 	result, err := promql.Exec(&ec, query, true)
 	if err != nil {
@@ -1070,7 +1099,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		}
 	}
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteQueryResponse(bw, result)
@@ -1105,17 +1134,17 @@ func QueryRangeHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	if err != nil {
 		return err
 	}
-	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return err
 	}
-	if err := queryRangeHandler(startTime, w, query, start, end, step, r, ct, etf); err != nil {
+	if err := queryRangeHandler(startTime, w, query, start, end, step, r, ct, etfs); err != nil {
 		return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", query, start, end, step, err)
 	}
 	return nil
 }
 
-func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string, start, end, step int64, r *http.Request, ct int64, etf []storage.TagFilter) error {
+func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string, start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	mayCache := !searchutils.GetBool(r, "nocache")
 	lookbackDelta, err := getMaxLookback(r)
@@ -1138,15 +1167,15 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
 	}
 
 	ec := promql.EvalConfig{
-		Start:              start,
-		End:                end,
-		Step:               step,
-		QuotedRemoteAddr:   httpserver.GetQuotedRemoteAddr(r),
-		Deadline:           deadline,
-		MayCache:           mayCache,
-		LookbackDelta:      lookbackDelta,
-		RoundDigits:        getRoundDigits(r),
-		EnforcedTagFilters: etf,
+		Start:               start,
+		End:                 end,
+		Step:                step,
+		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
+		Deadline:            deadline,
+		MayCache:            mayCache,
+		LookbackDelta:       lookbackDelta,
+		RoundDigits:         getRoundDigits(r),
+		EnforcedTagFilterss: etfs,
 	}
 	result, err := promql.Exec(&ec, query, false)
 	if err != nil {
@@ -1163,7 +1192,7 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153
 	result = removeEmptyValuesAndTimeseries(result)
 
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	WriteQueryRangeResponse(bw, result)
@@ -1254,24 +1283,12 @@ func getMaxLookback(r *http.Request) (int64, error) {
 	return searchutils.GetDuration(r, "max_lookback", d)
 }
 
-func addEnforcedFiltersToTagFilterss(dstTfss [][]storage.TagFilter, enforcedFilters []storage.TagFilter) [][]storage.TagFilter {
-	if len(dstTfss) == 0 {
-		return [][]storage.TagFilter{
-			enforcedFilters,
-		}
-	}
-	for i := range dstTfss {
-		dstTfss[i] = append(dstTfss[i], enforcedFilters...)
-	}
-	return dstTfss
-}
-
 func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) {
 	tagFilterss := make([][]storage.TagFilter, 0, len(matches))
 	for _, match := range matches {
-		tagFilters, err := promql.ParseMetricSelector(match)
+		tagFilters, err := searchutils.ParseMetricSelector(match)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse %q: %w", match, err)
+			return nil, fmt.Errorf("cannot parse matches[]=%s: %w", match, err)
 		}
 		tagFilterss = append(tagFilterss, tagFilters)
 	}
@@ -1287,11 +1304,11 @@ func getTagFilterssFromRequest(r *http.Request) ([][]storage.TagFilter, error) {
 	if err != nil {
 		return nil, err
 	}
-	etf, err := searchutils.GetEnforcedTagFiltersFromRequest(r)
+	etfs, err := searchutils.GetExtraTagFilters(r)
 	if err != nil {
 		return nil, err
 	}
-	tagFilterss = addEnforcedFiltersToTagFilterss(tagFilterss, etf)
+	tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
 	return tagFilterss, nil
 }
 
@@ -1343,7 +1360,7 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 		return fmt.Errorf("cannot parse `maxLifetime` arg: %w", err)
 	}
 	maxLifetime := time.Duration(maxLifetimeMsecs) * time.Millisecond
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Content-Type", "application/json")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	querystats.WriteJSONQueryStats(bw, topN, maxLifetime)

app/vmselect/prometheus/prometheus_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 func TestRemoveEmptyValuesAndTimeseries(t *testing.T) {
@@ -196,38 +195,3 @@ func TestAdjustLastPoints(t *testing.T) {
 		},
 	})
 }
-
-// helper for tests
-func tfFromKV(k, v string) storage.TagFilter {
-	return storage.TagFilter{
-		Key:   []byte(k),
-		Value: []byte(v),
-	}
-}
-
-func Test_addEnforcedFiltersToTagFilterss(t *testing.T) {
-	f := func(t *testing.T, dstTfss [][]storage.TagFilter, enforcedFilters []storage.TagFilter, want [][]storage.TagFilter) {
-		t.Helper()
-		got := addEnforcedFiltersToTagFilterss(dstTfss, enforcedFilters)
-		if !reflect.DeepEqual(got, want) {
-			t.Fatalf("unxpected result for addEnforcedFiltersToTagFilterss, \ngot: %v,\n want: %v", want, got)
-		}
-	}
-	f(t, [][]storage.TagFilter{{tfFromKV("label", "value")}},
-		nil,
-		[][]storage.TagFilter{{tfFromKV("label", "value")}})
-
-	f(t, nil,
-		[]storage.TagFilter{tfFromKV("ext-label", "ext-value")},
-		[][]storage.TagFilter{{tfFromKV("ext-label", "ext-value")}})
-
-	f(t, [][]storage.TagFilter{
-		{tfFromKV("l1", "v1")},
-		{tfFromKV("l2", "v2")},
-	},
-		[]storage.TagFilter{tfFromKV("ext-l1", "v2")},
-		[][]storage.TagFilter{
-			{tfFromKV("l1", "v1"), tfFromKV("ext-l1", "v2")},
-			{tfFromKV("l2", "v2"), tfFromKV("ext-l1", "v2")},
-		})
-}

app/vmselect/promql/aggr.go

@@ -15,44 +15,41 @@ import (
 )
 
 var aggrFuncs = map[string]aggrFunc{
-	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#aggregation-operators
-	"sum":          newAggrFunc(aggrFuncSum),
-	"min":          newAggrFunc(aggrFuncMin),
-	"max":          newAggrFunc(aggrFuncMax),
-	"avg":          newAggrFunc(aggrFuncAvg),
-	"stddev":       newAggrFunc(aggrFuncStddev),
-	"stdvar":       newAggrFunc(aggrFuncStdvar),
-	"count":        newAggrFunc(aggrFuncCount),
-	"count_values": aggrFuncCountValues,
-	"bottomk":      newAggrFuncTopK(true),
-	"topk":         newAggrFuncTopK(false),
-	"quantile":     aggrFuncQuantile,
-	"group":        newAggrFunc(aggrFuncGroup),
-
-	// PromQL extension funcs
-	"median":         aggrFuncMedian,
-	"limitk":         aggrFuncLimitK,
-	"distinct":       newAggrFunc(aggrFuncDistinct),
-	"sum2":           newAggrFunc(aggrFuncSum2),
-	"geomean":        newAggrFunc(aggrFuncGeomean),
-	"histogram":      newAggrFunc(aggrFuncHistogram),
-	"topk_min":       newAggrFuncRangeTopK(minValue, false),
-	"topk_max":       newAggrFuncRangeTopK(maxValue, false),
-	"topk_avg":       newAggrFuncRangeTopK(avgValue, false),
-	"topk_median":    newAggrFuncRangeTopK(medianValue, false),
-	"topk_last":      newAggrFuncRangeTopK(lastValue, false),
-	"bottomk_min":    newAggrFuncRangeTopK(minValue, true),
-	"bottomk_max":    newAggrFuncRangeTopK(maxValue, true),
+	"any":            aggrFuncAny,
+	"avg":            newAggrFunc(aggrFuncAvg),
+	"bottomk":        newAggrFuncTopK(true),
 	"bottomk_avg":    newAggrFuncRangeTopK(avgValue, true),
+	"bottomk_max":    newAggrFuncRangeTopK(maxValue, true),
 	"bottomk_median": newAggrFuncRangeTopK(medianValue, true),
 	"bottomk_last":   newAggrFuncRangeTopK(lastValue, true),
-	"any":            aggrFuncAny,
+	"bottomk_min":    newAggrFuncRangeTopK(minValue, true),
+	"count":          newAggrFunc(aggrFuncCount),
+	"count_values":   aggrFuncCountValues,
+	"distinct":       newAggrFunc(aggrFuncDistinct),
+	"geomean":        newAggrFunc(aggrFuncGeomean),
+	"group":          newAggrFunc(aggrFuncGroup),
+	"histogram":      newAggrFunc(aggrFuncHistogram),
+	"limitk":         aggrFuncLimitK,
 	"mad":            newAggrFunc(aggrFuncMAD),
+	"max":            newAggrFunc(aggrFuncMax),
+	"median":         aggrFuncMedian,
+	"min":            newAggrFunc(aggrFuncMin),
+	"mode":           newAggrFunc(aggrFuncMode),
 	"outliers_mad":   aggrFuncOutliersMAD,
 	"outliersk":      aggrFuncOutliersK,
-	"mode":           newAggrFunc(aggrFuncMode),
-	"zscore":         aggrFuncZScore,
+	"quantile":       aggrFuncQuantile,
 	"quantiles":      aggrFuncQuantiles,
+	"stddev":         newAggrFunc(aggrFuncStddev),
+	"stdvar":         newAggrFunc(aggrFuncStdvar),
+	"sum":            newAggrFunc(aggrFuncSum),
+	"sum2":           newAggrFunc(aggrFuncSum2),
+	"topk":           newAggrFuncTopK(false),
+	"topk_avg":       newAggrFuncRangeTopK(avgValue, false),
+	"topk_max":       newAggrFuncRangeTopK(maxValue, false),
+	"topk_median":    newAggrFuncRangeTopK(medianValue, false),
+	"topk_last":      newAggrFuncRangeTopK(lastValue, false),
+	"topk_min":       newAggrFuncRangeTopK(minValue, false),
+	"zscore":         aggrFuncZScore,
 }
 
 type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error)
@@ -999,19 +996,19 @@ func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 	if err := expectTransformArgsNum(args, 2); err != nil {
 		return nil, err
 	}
-	ks, err := getScalar(args[0], 0)
+	limits, err := getScalar(args[0], 0)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot obtain limit arg: %w", err)
 	}
-	maxK := 0
-	for _, kf := range ks {
-		k := int(kf)
-		if k > maxK {
-			maxK = k
-		}
+	limit := 0
+	if len(limits) > 0 {
+		limit = int(limits[0])
+	}
+	if limit < 0 {
+		limit = 0
 	}
 	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
-		// Sort series by metricName in order to get consistent set of output series
+		// Sort series by metricName hash in order to get consistent set of output series
 		// across multiple calls to limitk() function.
 		// Sort series by hash in order to guarantee uniform selection across series.
 		type hashSeries struct {
@@ -1033,17 +1030,8 @@ func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 		for i, hs := range hss {
 			tss[i] = hs.ts
 		}
-		if len(tss) > maxK {
-			tss = tss[:maxK]
-		}
-		for i, kf := range ks {
-			k := int(kf)
-			if k < 0 {
-				k = 0
-			}
-			for j := k; j < len(tss); j++ {
-				tss[j].Values[i] = nan
-			}
+		if limit < len(tss) {
+			tss = tss[:limit]
 		}
 		return tss
 	}

app/vmselect/promql/eval.go

@@ -104,8 +104,8 @@ type EvalConfig struct {
 	// How many decimal digits after the point to leave in response.
 	RoundDigits int
 
-	// EnforcedTagFilters used for apply additional label filters to query.
-	EnforcedTagFilters []storage.TagFilter
+	// EnforcedTagFilterss may contain additional label filters to use in the query.
+	EnforcedTagFilterss [][]storage.TagFilter
 
 	timestamps     []int64
 	timestampsOnce sync.Once
@@ -121,7 +121,7 @@ func newEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.MayCache = src.MayCache
 	ec.LookbackDelta = src.LookbackDelta
 	ec.RoundDigits = src.RoundDigits
-	ec.EnforcedTagFilters = src.EnforcedTagFilters
+	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
 
 	// do not copy src.timestamps - they must be generated again.
 	return &ec
@@ -391,7 +391,7 @@ func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.F
 	if nrf == nil {
 		return nil, nil
 	}
-	rollupArgIdx := getRollupArgIdx(fe)
+	rollupArgIdx := metricsql.GetRollupArgIdx(fe)
 	if rollupArgIdx >= len(fe.Args) {
 		// Incorrect number of args for rollup func.
 		return nil, nil
@@ -431,7 +431,7 @@ func evalExprs(ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
 
 func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
 	var re *metricsql.RollupExpr
-	rollupArgIdx := getRollupArgIdx(fe)
+	rollupArgIdx := metricsql.GetRollupArgIdx(fe)
 	if len(fe.Args) <= rollupArgIdx {
 		return nil, nil, fmt.Errorf("expecting at least %d args to %q; got %d args; expr: %q", rollupArgIdx+1, fe.Name, len(fe.Args), fe.AppendString(nil))
 	}
@@ -479,7 +479,43 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 	return &reNew
 }
 
+// expr may contain:
+// - rollupFunc(m) if iafc is nil
+// - aggrFunc(rollupFunc(m)) if iafc isn't nil
 func evalRollupFunc(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	if re.At == nil {
+		return evalRollupFuncWithoutAt(ec, funcName, rf, expr, re, iafc)
+	}
+	tssAt, err := evalExpr(ec, re.At)
+	if err != nil {
+		return nil, fmt.Errorf("cannot evaluate `@` modifier: %w", err)
+	}
+	if len(tssAt) != 1 {
+		return nil, fmt.Errorf("`@` modifier must return a single series; it returns %d series instead", len(tssAt))
+	}
+	atTimestamp := int64(tssAt[0].Values[0] * 1000)
+	ecNew := newEvalConfig(ec)
+	ecNew.Start = atTimestamp
+	ecNew.End = atTimestamp
+	tss, err := evalRollupFuncWithoutAt(ecNew, funcName, rf, expr, re, iafc)
+	if err != nil {
+		return nil, err
+	}
+	// expand single-point tss to the original time range.
+	timestamps := ec.getSharedTimestamps()
+	for _, ts := range tss {
+		v := ts.Values[0]
+		values := make([]float64, len(timestamps))
+		for i := range timestamps {
+			values[i] = v
+		}
+		ts.Timestamps = timestamps
+		ts.Values = values
+	}
+	return tss, nil
+}
+
+func evalRollupFuncWithoutAt(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
 	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
@@ -565,11 +601,12 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
 	}
 	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
 	var tssLock sync.Mutex
+	keepMetricNames := getKeepMetricNames(expr)
 	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) {
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
 		preFunc(values, timestamps)
 		for _, rc := range rcs {
-			if tsm := newTimeseriesMap(funcName, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
+			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
 				rc.DoTimeseriesMap(tsm, values, timestamps)
 				tssLock.Lock()
 				tss = tsm.AppendTimeseriesTo(tss)
@@ -577,7 +614,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
 				continue
 			}
 			var ts timeseries
-			doRollupForTimeseries(funcName, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps)
+			doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps)
 			tssLock.Lock()
 			tss = append(tss, &ts)
 			tssLock.Unlock()
@@ -587,6 +624,22 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
 	return tss, nil
 }
 
+func getKeepMetricNames(expr metricsql.Expr) bool {
+	if ae, ok := expr.(*metricsql.AggrFuncExpr); ok {
+		// Extract rollupFunc(...) from aggrFunc(rollupFunc(...)).
+		// This case is possible when optimized aggrFunc calculations are used
+		// such as `sum(rate(...))`
+		if len(ae.Args) != 1 {
+			return false
+		}
+		expr = ae.Args[0]
+	}
+	if fe, ok := expr.(*metricsql.FuncExpr); ok {
+		return fe.KeepMetricNames
+	}
+	return false
+}
+
 func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64) ([]float64, []int64)) {
 	concurrency := cgroup.AvailableCPUs()
 	if concurrency > len(tss) {
@@ -672,16 +725,15 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc
 	}
 
 	// Fetch the remaining part of the result.
-	tfs := toTagFilters(me.LabelFilters)
-	// append external filters.
-	tfs = append(tfs, ec.EnforcedTagFilters...)
+	tfs := searchutils.ToTagFilters(me.LabelFilters)
+	tfss := searchutils.JoinTagFilterss([][]storage.TagFilter{tfs}, ec.EnforcedTagFilterss)
 	minTimestamp := start - maxSilenceInterval
 	if window > ec.Step {
 		minTimestamp -= window
 	} else {
 		minTimestamp -= ec.Step
 	}
-	sq := storage.NewSearchQuery(minTimestamp, ec.End, [][]storage.TagFilter{tfs})
+	sq := storage.NewSearchQuery(minTimestamp, ec.End, tfss)
 	rss, err := netstorage.ProcessSearchQuery(sq, true, ec.Deadline)
 	if err != nil {
 		return nil, err
@@ -729,18 +781,20 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc
 		rss.Cancel()
 		return nil, fmt.Errorf("not enough memory for processing %d data points across %d time series with %d points in each time series; "+
 			"total available memory for concurrent requests: %d bytes; "+
+			"requested memory: %d bytes; "+
 			"possible solutions are: reducing the number of matching time series; switching to node with more RAM; "+
 			"increasing -memory.allowedPercent; increasing `step` query arg (%gs)",
-			rollupPoints, timeseriesLen*len(rcs), pointsPerTimeseries, rml.MaxSize, float64(ec.Step)/1e3)
+			rollupPoints, timeseriesLen*len(rcs), pointsPerTimeseries, rml.MaxSize, uint64(rollupMemorySize), float64(ec.Step)/1e3)
 	}
 	defer rml.Put(uint64(rollupMemorySize))
 
 	// Evaluate rollup
+	keepMetricNames := getKeepMetricNames(expr)
 	var tss []*timeseries
 	if iafc != nil {
-		tss, err = evalRollupWithIncrementalAggregate(funcName, iafc, rss, rcs, preFunc, sharedTimestamps)
+		tss, err = evalRollupWithIncrementalAggregate(funcName, keepMetricNames, iafc, rss, rcs, preFunc, sharedTimestamps)
 	} else {
-		tss, err = evalRollupNoIncrementalAggregate(funcName, rss, rcs, preFunc, sharedTimestamps)
+		tss, err = evalRollupNoIncrementalAggregate(funcName, keepMetricNames, rss, rcs, preFunc, sharedTimestamps)
 	}
 	if err != nil {
 		return nil, err
@@ -762,7 +816,7 @@ func getRollupMemoryLimiter() *memoryLimiter {
 	return &rollupMemoryLimiter
 }
 
-func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
+func evalRollupWithIncrementalAggregate(funcName string, keepMetricNames bool, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
@@ -770,7 +824,7 @@ func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFu
 		ts := getTimeseries()
 		defer putTimeseries(ts)
 		for _, rc := range rcs {
-			if tsm := newTimeseriesMap(funcName, sharedTimestamps, &rs.MetricName); tsm != nil {
+			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil {
 				rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
 				for _, ts := range tsm.m {
 					iafc.updateTimeseries(ts, workerID)
@@ -778,7 +832,7 @@ func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFu
 				continue
 			}
 			ts.Reset()
-			doRollupForTimeseries(funcName, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
+			doRollupForTimeseries(funcName, keepMetricNames, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
 			iafc.updateTimeseries(ts, workerID)
 
 			// ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used.
@@ -794,7 +848,7 @@ func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFu
 	return tss, nil
 }
 
-func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results, rcs []*rollupConfig,
+func evalRollupNoIncrementalAggregate(funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
 	var tssLock sync.Mutex
@@ -802,7 +856,7 @@ func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results,
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {
-			if tsm := newTimeseriesMap(funcName, sharedTimestamps, &rs.MetricName); tsm != nil {
+			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil {
 				rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
 				tssLock.Lock()
 				tss = tsm.AppendTimeseriesTo(tss)
@@ -810,7 +864,7 @@ func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results,
 				continue
 			}
 			var ts timeseries
-			doRollupForTimeseries(funcName, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
+			doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
 			tssLock.Lock()
 			tss = append(tss, &ts)
 			tssLock.Unlock()
@@ -823,13 +877,13 @@ func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results,
 	return tss, nil
 }
 
-func doRollupForTimeseries(funcName string, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName, valuesSrc []float64, timestampsSrc []int64,
-	sharedTimestamps []int64) {
+func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName,
+	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) {
 	tsDst.MetricName.CopyFrom(mnSrc)
 	if len(rc.TagValue) > 0 {
 		tsDst.MetricName.AddTag("rollup", rc.TagValue)
 	}
-	if !rollupFuncsKeepMetricGroup[funcName] {
+	if !keepMetricNames && !rollupFuncsKeepMetricName[funcName] {
 		tsDst.MetricName.ResetMetricGroup()
 	}
 	tsDst.Values = rc.Do(tsDst.Values[:0], valuesSrc, timestampsSrc)
@@ -876,30 +930,12 @@ func mulNoOverflow(a, b int64) int64 {
 	return a * b
 }
 
-func toTagFilters(lfs []metricsql.LabelFilter) []storage.TagFilter {
-	tfs := make([]storage.TagFilter, len(lfs))
-	for i := range lfs {
-		toTagFilter(&tfs[i], &lfs[i])
-	}
-	return tfs
-}
-
-func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) {
-	if src.Label != "__name__" {
-		dst.Key = []byte(src.Label)
-	} else {
-		// This is required for storage.Search.
-		dst.Key = nil
-	}
-	dst.Value = []byte(src.Value)
-	dst.IsRegexp = src.IsRegexp
-	dst.IsNegative = src.IsNegative
-}
-
 func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]float64, []int64) {
-	if *noStaleMarkers || funcName == "default_rollup" {
+	if *noStaleMarkers || funcName == "default_rollup" || funcName == "stale_samples_over_time" {
 		// Do not drop Prometheus staleness marks (aka stale NaNs) for default_rollup() function,
 		// since it uses them for Prometheus-style staleness detection.
+		// Do not drop staleness marks for stale_samples_over_time() function, since it needs
+		// to calculate the number of staleness markers.
 		return values, timestamps
 	}
 	// Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values.

app/vmselect/promql/exec_test.go

@@ -593,6 +593,29 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("timestamp(alias(time()>=1600))", func(t *testing.T) {
+		t.Parallel()
+		q := `timestamp(alias(time()>=1600,"foo"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("timestamp_with_name(alias(time()>=1600))", func(t *testing.T) {
+		t.Parallel()
+		q := `timestamp_with_name(alias(time()>=1600,"foo"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foo")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("time()/100", func(t *testing.T) {
 		t.Parallel()
 		q := `time()/100`
@@ -942,7 +965,7 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run("exp(time()/1e3)", func(t *testing.T) {
 		t.Parallel()
-		q := `exp(time()/1e3)`
+		q := `exp(alias(time()/1e3, "foobar"))`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{2.718281828459045, 3.3201169227365472, 4.0551999668446745, 4.953032424395115, 6.0496474644129465, 7.38905609893065},
@@ -951,6 +974,73 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("exp(time()/1e3) keep_metric_names", func(t *testing.T) {
+		t.Parallel()
+		q := `exp(alias(time()/1e3, "foobar")) keep_metric_names`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2.718281828459045, 3.3201169227365472, 4.0551999668446745, 4.953032424395115, 6.0496474644129465, 7.38905609893065},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foobar")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("time() @ 1h", func(t *testing.T) {
+		t.Parallel()
+		q := `time() @ 1h`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3600, 3600, 3600, 3600, 3600, 3600},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("time() @ start()", func(t *testing.T) {
+		t.Parallel()
+		q := `time() @ start()`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1000, 1000, 1000, 1000, 1000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("time() @ end()", func(t *testing.T) {
+		t.Parallel()
+		q := `time() @ end()`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2000, 2000, 2000, 2000, 2000, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("time() @ end() offset 10m", func(t *testing.T) {
+		t.Parallel()
+		q := `time() @ end() offset 10m`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1400, 1400, 1400, 1400, 1400, 1400},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("time() @ (end()-10m)", func(t *testing.T) {
+		t.Parallel()
+		q := `time() @ (end()-10m)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1400, 1400, 1400, 1400, 1400, 1400},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("rand()", func(t *testing.T) {
 		t.Parallel()
 		q := `round(rand()/2)`
@@ -1017,6 +1107,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("now()", func(t *testing.T) {
+		t.Parallel()
+		q := `round(now()/now())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("pi()", func(t *testing.T) {
 		t.Parallel()
 		q := `pi()`
@@ -2012,6 +2113,81 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`label_graphite_group()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(label_graphite_group((
+			alias(1, "foo.bar.baz"),
+			alias(2, "abc"),
+			label_set(alias(3, "a.xx.zz.asd"), "qwe", "rty"),
+	        ), 1, 3))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("bar.")
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte(".")
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("xx.asd")
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("qwe"),
+			Value: []byte("rty"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`limit_offset`, func(t *testing.T) {
+		t.Parallel()
+		q := `limit_offset(1, 1, sort_by_label((
+			label_set(time()*1, "foo", "y"),
+			label_set(time()*2, "foo", "a"),
+			label_set(time()*3, "foo", "x"),
+		), "foo"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3000, 3600, 4200, 4800, 5400, 6000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("x"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`sum(label_graphite_group)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(sum by (__name__) (
+			label_graphite_group((
+				alias(1, "foo.bar.baz"),
+				alias(2, "x.y.z"),
+				alias(3, "qe.bar.qqq"),
+			), 1)
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("y")
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("bar")
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`two_timeseries`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_desc(time() or label_set(2, "xx", "foo"))`
@@ -4974,6 +5150,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`duration_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `duration_over_time((time()<1200)[600s:10s], 20s)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{590, 580, 380, 180, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`share_gt_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `share_gt_over_time(rand(0)[200s:10s], 0.7)`
@@ -6043,12 +6230,48 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rate(time())`, func(t *testing.T) {
 		t.Parallel()
-		q := `rate(time())`
+		q := `rate(label_set(alias(time(), "foo"), "x", "y"))`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{1, 1, 1, 1, 1, 1},
 			Timestamps: timestampsExpected,
 		}
+		r.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("x"),
+				Value: []byte("y"),
+			},
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`rate(time()) keep_metric_names`, func(t *testing.T) {
+		t.Parallel()
+		q := `rate(label_set(alias(time(), "foo"), "x", "y")) keep_metric_names`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foo")
+		r.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("x"),
+				Value: []byte("y"),
+			},
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`sum(rate(time()) keep_metric_names) by (__name__)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sum(rate(label_set(alias(time(), "foo"), "x", "y")) keep_metric_names) by (__name__)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foo")
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -6151,6 +6374,22 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`increase_prometheus(time())`, func(t *testing.T) {
+		t.Parallel()
+		q := `increase_prometheus(time())`
+		f(q, nil)
+	})
+	t.Run(`increase_prometheus(time()[201s])`, func(t *testing.T) {
+		t.Parallel()
+		q := `increase_prometheus(time()[201s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{200, 200, 200, 200, 200, 200},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`running_max(1)`, func(t *testing.T) {
 		t.Parallel()
 		q := `running_max(1)`
@@ -6316,9 +6555,9 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
-	t.Run(`deriv(1)`, func(t *testing.T) {
+	t.Run(`deriv(N)`, func(t *testing.T) {
 		t.Parallel()
-		q := `deriv(1)`
+		q := `deriv(1000)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{0, 0, 0, 0, 0, 0},
@@ -6393,6 +6632,22 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`delta_prometheus(time())`, func(t *testing.T) {
+		t.Parallel()
+		q := `delta_prometheus(time())`
+		f(q, nil)
+	})
+	t.Run(`delta_prometheus(time()[201s])`, func(t *testing.T) {
+		t.Parallel()
+		q := `delta_prometheus(time()[201s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{200, 200, 200, 200, 200, 200},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`median_over_time("foo")`, func(t *testing.T) {
 		t.Parallel()
 		q := `median_over_time("foo")`
@@ -7276,6 +7531,7 @@ func TestExecError(t *testing.T) {
 	f(`label_keep()`)
 	f(`label_match()`)
 	f(`label_mismatch()`)
+	f(`label_graphite_group()`)
 	f(`round()`)
 	f(`round(1,2,3)`)
 	f(`sgn()`)
@@ -7285,6 +7541,7 @@ func TestExecError(t *testing.T) {
 	f(`sort_by_label()`)
 	f(`sort_by_label_desc()`)
 	f(`timestamp()`)
+	f(`timestamp_with_name()`)
 	f(`vector()`)
 	f(`histogram_quantile()`)
 	f(`histogram_quantiles()`)
@@ -7329,6 +7586,7 @@ func TestExecError(t *testing.T) {
 	f(`rand_normal(123, 456)`)
 	f(`rand_exponential(122, 456)`)
 	f(`pi(123)`)
+	f(`now(123)`)
 	f(`label_copy()`)
 	f(`label_move()`)
 	f(`median_over_time()`)
@@ -7368,6 +7626,7 @@ func TestExecError(t *testing.T) {
 	f(`prometheus_buckets()`)
 	f(`buckets_limit()`)
 	f(`buckets_limit(1)`)
+	f(`duration_over_time()`)
 	f(`share_le_over_time()`)
 	f(`share_gt_over_time()`)
 	f(`count_le_over_time()`)
@@ -7379,6 +7638,13 @@ func TestExecError(t *testing.T) {
 	f(`bitmap_or()`)
 	f(`bitmap_xor()`)
 	f(`quantiles()`)
+	f(`limit_offset()`)
+	f(`increase()`)
+	f(`increase_prometheus()`)
+	f(`changes()`)
+	f(`changes_prometheus()`)
+	f(`delta()`)
+	f(`delta_prometheus()`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)
@@ -7391,6 +7657,8 @@ func TestExecError(t *testing.T) {
 	f(`topk(label_set(2, "xx", "foo") or 1, 12)`)
 	f(`topk_avg(label_set(2, "xx", "foo") or 1, 12)`)
 	f(`limitk(label_set(2, "xx", "foo") or 1, 12)`)
+	f(`limit_offet((alias(1,"foo"),alias(2,"bar")), 2, 10)`)
+	f(`limit_offet(1, (alias(1,"foo"),alias(2,"bar")), 10)`)
 	f(`round(1, 1 or label_set(2, "xx", "foo"))`)
 	f(`histogram_quantile(1 or label_set(2, "xx", "foo"), 1)`)
 	f(`label_set(1, 2, 3)`)

app/vmselect/promql/parser.go

@@ -1,9 +1,6 @@
 package promql
 
 import (
-	"fmt"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metricsql"
 )
 
@@ -43,21 +40,3 @@ func IsMetricSelectorWithRollup(s string) (childQuery string, window, offset *me
 	wrappedQuery := me.AppendString(nil)
 	return string(wrappedQuery), re.Window, re.Offset
 }
-
-// ParseMetricSelector parses s containing PromQL metric selector
-// and returns the corresponding LabelFilters.
-func ParseMetricSelector(s string) ([]storage.TagFilter, error) {
-	expr, err := parsePromQLWithCache(s)
-	if err != nil {
-		return nil, err
-	}
-	me, ok := expr.(*metricsql.MetricExpr)
-	if !ok {
-		return nil, fmt.Errorf("expecting metricSelector; got %q", expr.AppendString(nil))
-	}
-	if len(me.LabelFilters) == 0 {
-		return nil, fmt.Errorf("labelFilters cannot be empty")
-	}
-	tfs := toTagFilters(me.LabelFilters)
-	return tfs, nil
-}

app/vmselect/promql/parser_test.go

@@ -1,50 +0,0 @@
-package promql
-
-import (
-	"testing"
-)
-
-func TestParseMetricSelectorSuccess(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		tfs, err := ParseMetricSelector(s)
-		if err != nil {
-			t.Fatalf("unexpected error when parsing %q: %s", s, err)
-		}
-		if tfs == nil {
-			t.Fatalf("expecting non-nil tfs when parsing %q", s)
-		}
-	}
-	f("foo")
-	f(":foo")
-	f("  :fo:bar.baz")
-	f(`a{}`)
-	f(`{foo="bar"}`)
-	f(`{:f:oo=~"bar.+"}`)
-	f(`foo {bar != "baz"}`)
-	f(` foo { bar !~ "^ddd(x+)$", a="ss", __name__="sffd"}  `)
-	f(`(foo)`)
-	f(`\п\р\и\в\е\т{\ы="111"}`)
-}
-
-func TestParseMetricSelectorError(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		tfs, err := ParseMetricSelector(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
-		}
-		if tfs != nil {
-			t.Fatalf("expecting nil tfs when parsing %q", s)
-		}
-	}
-	f("")
-	f(`{}`)
-	f(`foo bar`)
-	f(`foo+bar`)
-	f(`sum(bar)`)
-	f(`x{y}`)
-	f(`x{y+z}`)
-	f(`foo[5m]`)
-	f(`foo offset 5m`)
-}