docs/CHANGELOG.md: cut v1.78.0

alerts: filter out non error log messages for `TooManyLogs`

Info and Warn error levels aren't always a result of malfunctioning
or faulty state. So we filter them out.

Makefile

@@ -283,7 +283,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.46.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.46.2
 
 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd

README.md

@@ -14,20 +14,26 @@ VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and t
 
 VictoriaMetrics is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
 [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap packages](https://snapcraft.io/victoriametrics)
-and [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and follow [these instructions](#how-to-start-victoriametrics).
-Then read [Prometheus setup](#prometheus-setup) and [Grafana setup](#grafana-setup) docs.
+and [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). 
+Just download VictoriaMetrics and follow [these instructions](https://docs.victoriametrics.com/Quick-Start.html).
 
-Cluster version of VictoriaMetrics is available [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+The cluster version of VictoriaMetrics is available [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
 
-[Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics. See [features available in enterprise package](https://victoriametrics.com/products/enterprise/). Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
+Learn more about [key concepts](https://docs.victoriametrics.com/keyConcepts.html) of VictoriaMetrics and follow the 
+[QuickStart guide](https://docs.victoriametrics.com/Quick-Start.html) for a better experience.
+
+[Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics. 
+See [features available in enterprise package](https://victoriametrics.com/products/enterprise/).
+Enterprise binaries can be downloaded and evaluated for free 
+from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 ## Prominent features
 
 VictoriaMetrics has the following prominent features:
 
 * It can be used as long-term storage for Prometheus. See [these docs](#prometheus-setup) for details.
-* It can be used as drop-in replacement for Prometheus in Grafana, because it supports [Prometheus querying API](#prometheus-querying-api-usage).
-* It can be used as drop-in replacement for Graphite in Grafana, because it supports [Graphite API](#graphite-api-usage).
+* It can be used as a drop-in replacement for Prometheus in Grafana, because it supports [Prometheus querying API](#prometheus-querying-api-usage).
+* It can be used as a drop-in replacement for Graphite in Grafana, because it supports [Graphite API](#graphite-api-usage).
 * It features easy setup and operation:
   * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
@@ -53,7 +59,7 @@ VictoriaMetrics has the following prominent features:
   * [JSON line format](#how-to-import-data-in-json-line-format).
   * [Arbitrary CSV data](#how-to-import-csv-data).
   * [Native binary format](#how-to-import-data-in-native-format).
-* It supports metrics' relabeling. See [these docs](#relabeling) for details.
+* It supports metrics [relabeling](#relabeling).
 * It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
 * It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/).
 * It has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
@@ -92,9 +98,10 @@ See also [articles and slides about VictoriaMetrics from our users](https://docs
 
 ## Operation
 
-## How to start VictoriaMetrics
+### How to start VictoriaMetrics
 
 Just download [VictoriaMetrics executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or [Docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and start it with the desired command-line flags.
+See also [QuickStart guide](https://docs.victoriametrics.com/Quick-Start.html) for additional information.
 
 The following command-line flags are used the most:
 
@@ -143,18 +150,26 @@ After changes were made, trigger config re-read with the command `curl 127.0.0.1
 
 Add the following lines to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`) in order to send data to VictoriaMetrics:
 
+<div class="with-copy" markdown="1">
+
 ```yml
 remote_write:
   - url: http://<victoriametrics-addr>:8428/api/v1/write
 ```
 
+</div>
+
 Substitute `<victoriametrics-addr>` with hostname or IP address of VictoriaMetrics.
 Then apply new config via the following command:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 kill -HUP `pidof prometheus`
 ```
 
+</div>
+
 Prometheus writes incoming data to local storage and replicates it to remote storage in parallel.
 This means that data remains available in local storage for `--storage.tsdb.retention.time` duration
 even if remote storage is unavailable.
@@ -174,6 +189,8 @@ across Prometheus instances, so time series could be filtered and grouped by thi
 
 For highly loaded Prometheus instances (200k+ samples per second) the following tuning may be applied:
 
+<div class="with-copy" markdown="1">
+
 ```yaml
 remote_write:
   - url: http://<victoriametrics-addr>:8428/api/v1/write
@@ -183,13 +200,18 @@ remote_write:
       max_shards: 30
 ```
 
+</div>
+
 Using remote write increases memory usage for Prometheus by up to ~25%. If you are experiencing issues with
-too high memory consumption of Prometheus, then try to lower `max_samples_per_send` and `capacity` params. Keep in mind that these two params are tightly connected.
+too high memory consumption of Prometheus, then try to lower `max_samples_per_send` and `capacity` params. 
+Keep in mind that these two params are tightly connected.
 Read more about tuning remote write for Prometheus [here](https://prometheus.io/docs/practices/remote_write).
 
-It is recommended upgrading Prometheus to [v2.12.0](https://github.com/prometheus/prometheus/releases) or newer, since previous versions may have issues with `remote_write`.
+It is recommended upgrading Prometheus to [v2.12.0](https://github.com/prometheus/prometheus/releases) or newer, 
+since previous versions may have issues with `remote_write`.
 
-Take a look also at [vmagent](https://docs.victoriametrics.com/vmagent.html) and [vmalert](https://docs.victoriametrics.com/vmalert.html),
+Take a look also at [vmagent](https://docs.victoriametrics.com/vmagent.html) 
+and [vmalert](https://docs.victoriametrics.com/vmalert.html),
 which can be used as faster and less resource-hungry alternative to Prometheus.
 
 ## Grafana setup
@@ -218,6 +240,48 @@ The following steps must be performed during the upgrade / downgrade procedure:
 
 Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/vmagent.html).
 
+## vmui
+
+VictoriaMetrics provides UI for query troubleshooting and exploration. The UI is available at `http://victoriametrics:8428/vmui`.
+The UI allows exploring query results via graphs and tables. It also provides support for [cardinality explorer](#cardinality-explorer).
+
+Graphs in vmui support scrolling and zooming:
+
+* Drag the graph to the left / right in order to move the displayed time range into the past / future.
+* Hold `Ctrl` (or `Cmd` on MacOS) and scroll up / down in order to zoom in / out the graph.
+
+Query history can be navigated by holding `Ctrl` (or `Cmd` on MacOS) and pressing `up` or `down` arrows on the keyboard while the cursor is located in the query input field.
+
+Multi-line queries can be entered by pressing `Shift-Enter` in query input field.
+
+When querying the [backfilled data](https://docs.victoriametrics.com/#backfilling), it may be useful disabling response cache by clicking `Enable cache` checkbox.
+
+VMUI automatically adjusts the interval between datapoints on the graph depending on the horizontal resolution and on the selected time range. The step value can be customized by clickhing `Override step value` checkbox.
+
+VMUI allows investigating correlations between two queries on the same graph. Just click `+Query` button, enter the second query in the newly appeared input field and press `Ctrl+Enter`. Results for both queries should be displayed simultaneously on the same graph. Every query has its own vertical scale, which is displayed on the left and the right side of the graph. Lines for the second query are dashed.
+
+See the [example VMUI at VictoriaMetrics playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/?g0.expr=100%20*%20sum(rate(process_cpu_seconds_total))%20by%20(job)&g0.range_input=1d).
+
+
+## Cardinality explorer
+
+VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
+
+- To identify metric names with the highest number of series.
+- To identify labels with the highest number of series.
+- To identify values with the highest number of series for the selected label (aka `focusLabel`).
+- To identify label=name pairs with the highest number of series.
+- To identify labels with the highest number of unique values.
+
+By default cardinality explorer analyzes time series for the current date. It provides the ability to select different day at the top right corner.
+By default all the time series for the selected date are analyzed. It is possible to narrow down the analysis to series
+matching the specified [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
+
+Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
+
+See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
+
+
 ## How to apply new config to VictoriaMetrics
 
 VictoriaMetrics is configured via command-line flags, so it must be restarted when new command-line flags should be applied:
@@ -264,7 +328,7 @@ VictoriaMetrics doesn't check `DD_API_KEY` param, so it can be set to arbitrary
 
 Example on how to send data to VictoriaMetrics via DataDog "submit metrics" API from command line:
 
-```bash
+```console
 echo '
 {
   "series": [
@@ -288,19 +352,33 @@ echo '
 
 The imported data can be read via [export API](https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format):
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl http://localhost:8428/api/v1/export -d 'match[]=system.load.1'
 ```
 
+</div>
+
 This command should return the following output if everything is OK:
 
-```
+```json
 {"metric":{"__name__":"system.load.1","environment":"test","host":"test.example.com"},"values":[0.5],"timestamps":[1632833641000]}
 ```
 
 Extra labels may be added to all the written time series by passing `extra_label=name=value` query args.
 For example, `/datadog/api/v1/series?extra_label=foo=bar` would add `{foo="bar"}` label to all the ingested metrics.
 
+DataDog agent sends the [configured tags](https://docs.datadoghq.com/getting_started/tagging/) to
+undocumented endpoint - `/datadog/intake`. This endpoint isn't supported by VictoriaMetrics yet.
+This prevents from adding the configured tags to DataDog agent data sent into VictoriaMetrics.
+The workaround is to run a sidecar [vmagent](https://docs.victoriametrics.com/vmagent.html) alongside every DataDog agent,
+which must run with `DD_DD_URL=http://localhost:8429/datadog` environment variable.
+The sidecar `vmagent` must be configured with the needed tags via `-remoteWrite.label` command-line flag and must forward
+incoming data with the added tags to a centralized VictoriaMetrics specified via `-remoteWrite.url` command-line flag.
+
+See [these docs](https://docs.victoriametrics.com/vmagent.html#adding-labels-to-metrics) for details on how to add labels to metrics at `vmagent`.
+
 ## How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)
 
 Use `http://<victoriametric-addr>:8428` url instead of InfluxDB url in agents' configs.
@@ -316,7 +394,7 @@ and stream plain InfluxDB line protocol data to the configured TCP and/or UDP ad
 
 VictoriaMetrics performs the following transformations to the ingested InfluxDB data:
 
-* [`db` query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value
+* [db query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value
   unless `db` tag exists in the InfluxDB line. The `db` label name can be overriden via `-influxDBLabel` command-line flag.
 * Field names are mapped to time series names prefixed with `{measurement}{separator}` value, where `{separator}` equals to `_` by default. It can be changed with `-influxMeasurementFieldSeparator` command-line flag. See also `-influxSkipSingleField` command-line flag. If `{measurement}` is empty or if `-influxSkipMeasurement` command-line flag is set, then time series names correspond to field names.
 * Field values are mapped to time series values.
@@ -338,20 +416,28 @@ foo_field2{tag1="value1", tag2="value2"} 40
 Example for writing data with [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
 to local VictoriaMetrics using `curl`:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -d 'measurement,tag1=value1,tag2=value2 field1=123,field2=1.23' -X POST 'http://localhost:8428/write'
 ```
 
+</div>
+
 An arbitrary number of lines delimited by '\n' (aka newline char) can be sent in a single request.
 After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"measurement_.*"}'
 ```
 
+</div>
+
 The `/api/v1/export` endpoint should return the following response:
 
-```jsonl
+```json
 {"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560272508147]}
 {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[1.23],"timestamps":[1560272508147]}
 ```
@@ -371,7 +457,7 @@ Comma-separated list of expected databases can be passed to VictoriaMetrics via
 Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,
 the following command will enable Graphite receiver in VictoriaMetrics on TCP and UDP port `2003`:
 
-```bash
+```console
 /path/to/victoria-metrics-prod -graphiteListenAddr=:2003
 ```
 
@@ -380,7 +466,7 @@ to the VictoriaMetrics host in `StatsD` configs.
 
 Example for writing data with Graphite plaintext protocol to local VictoriaMetrics using `nc`:
 
-```bash
+```console
 echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N localhost 2003
 ```
 
@@ -388,16 +474,22 @@ VictoriaMetrics sets the current time if the timestamp is omitted.
 An arbitrary number of lines delimited by `\n` (aka newline char) can be sent in one go.
 After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
 ```
 
+</div>
+
 The `/api/v1/export` endpoint should return the following response:
 
-```bash
+```json
 {"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277406000]}
 ```
 
+[Graphite relabeling](https://docs.victoriametrics.com/vmagent.html#graphite-relabeling) can be used if the imported Graphite data is going to be queried via [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html).
+
 ## Querying Graphite data
 
 Data sent to VictoriaMetrics via `Graphite plaintext protocol` may be read via the following APIs:
@@ -412,6 +504,9 @@ VictoriaMetrics supports `__graphite__` pseudo-label for selecting time series w
 
 The `__graphite__` pseudo-label supports e.g. alternate regexp filters such as `(value1|...|valueN)`. They are transparently converted to `{value1,...,valueN}` syntax [used in Graphite](https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards). This allows using [multi-value template variables in Grafana](https://grafana.com/docs/grafana/latest/variables/formatting-multi-value-variables/) inside `__graphite__` pseudo-label. For example, Grafana expands `{__graphite__=~"foo.($bar).baz"}` into `{__graphite__=~"foo.(x|y).baz"}` if `$bar` template variable contains `x` and `y` values. In this case the query is automatically converted into `{__graphite__=~"foo.{x,y}.baz"}` before execution.
 
+VictoriaMetrics also supports Graphite query language - see [these docs](#graphite-render-api-usage).
+
+
 ## How to send data from OpenTSDB-compatible agents
 
 VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
@@ -423,7 +518,7 @@ The same protocol is used for [ingesting data in KairosDB](https://kairosdb.gith
 Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command line flag. For instance,
 the following command enables OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`:
 
-```bash
+```console
 /path/to/victoria-metrics-prod -opentsdbListenAddr=:4242
 ```
 
@@ -431,20 +526,28 @@ Send data to the given address from OpenTSDB-compatible agents.
 
 Example for writing data with OpenTSDB protocol to local VictoriaMetrics using `nc`:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N localhost 4242
 ```
 
+</div>
+
 An arbitrary number of lines delimited by `\n` (aka newline char) can be sent in one go.
 After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
 ```
 
+</div>
+
 The `/api/v1/export` endpoint should return the following response:
 
-```bash
+```json
 {"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277292000]}
 ```
 
@@ -453,7 +556,7 @@ The `/api/v1/export` endpoint should return the following response:
 Enable HTTP server for OpenTSDB `/api/put` requests by setting `-opentsdbHTTPListenAddr` command line flag. For instance,
 the following command enables OpenTSDB HTTP server on port `4242`:
 
-```bash
+```console
 /path/to/victoria-metrics-prod -opentsdbHTTPListenAddr=:4242
 ```
 
@@ -461,25 +564,37 @@ Send data to the given address from OpenTSDB-compatible agents.
 
 Example for writing a single data point:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -H 'Content-Type: application/json' -d '{"metric":"x.y.z","value":45.34,"tags":{"t1":"v1","t2":"v2"}}' http://localhost:4242/api/put
 ```
 
+</div>
+
 Example for writing multiple data points in a single request:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]' http://localhost:4242/api/put
 ```
 
+</div>
+
 After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -G 'http://localhost:8428/api/v1/export' -d 'match[]=x.y.z' -d 'match[]=foo' -d 'match[]=bar'
 ```
 
+</div>
+
 The `/api/v1/export` endpoint should return the following response:
 
-```bash
+```json
 {"metric":{"__name__":"foo"},"values":[45.34],"timestamps":[1566464846000]}
 {"metric":{"__name__":"bar"},"values":[43],"timestamps":[1566464846000]}
 {"metric":{"__name__":"x.y.z","t1":"v1","t2":"v2"},"values":[45.34],"timestamps":[1566464763000]}
@@ -517,15 +632,16 @@ For example, the following query would return data for the last 30 minutes: `/ap
 
 VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
 
+VictoriaMetrics accepts `limit` query arg for `/api/v1/labels` and `/api/v1/label/<labelName>/values` handlers for limiting the number of returned entries. For example, the query to `/api/v1/labels?limit=5` returns a sample of up to 5 unique labels, while ignoring the rest of labels. If the provided `limit` value exceeds the corresponding `-search.maxTagKeys` / `-search.maxTagValues` command-line flag values, then limits specified in the command-line flags are used.
+
 By default, VictoriaMetrics returns time series for the last 5 minutes from `/api/v1/series`, while the Prometheus API defaults to all time.  Use `start` and `end` to select a different time range.
 
-Additionally VictoriaMetrics provides the following handlers:
+Additionally, VictoriaMetrics provides the following handlers:
 
 * `/vmui` - Basic Web UI. See [these docs](#vmui).
 * `/api/v1/series/count` - returns the total number of time series in the database. Some notes:
   * the handler scans all the inverted index, so it can be slow if the database contains tens of millions of time series;
   * the handler may count [deleted time series](#how-to-delete-time-series) additionally to normal time series due to internal implementation restrictions;
-* `/api/v1/labels/count` - returns a list of `label: values_count` entries. It can be used for determining labels with the maximum number of values.
 * `/api/v1/status/active_queries` - returns a list of currently running queries.
 * `/api/v1/status/top_queries` - returns the following query lists:
   * the most frequently executed queries - `topByCount`
@@ -587,26 +703,6 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/
 * [/tags/autoComplete/values](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support)
 * [/tags/delSeries](https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb)
 
-## vmui
-
-VictoriaMetrics provides UI for query troubleshooting and exploration. The UI is available at `http://victoriametrics:8428/vmui`.
-The UI allows exploring query results via graphs and tables. Graphs support scrolling and zooming:
-
-* Drag the graph to the left / right in order to move the displayed time range into the past / future.
-* Hold `Ctrl` (or `Cmd` on MacOS) and scroll up / down in order to zoom in / out the graph.
-
-Query history can be navigated by holding `Ctrl` (or `Cmd` on MacOS) and pressing `up` or `down` arrows on the keyboard while the cursor is located in the query input field.
-
-Multi-line queries can be entered by pressing `Shift-Enter` in query input field.
-
-When querying the [backfilled data](https://docs.victoriametrics.com/#backfilling), it may be useful disabling response cache by clicking `Enable cache` checkbox.
-
-VMUI automatically adjusts the interval between datapoints on the graph depending on the horizontal resolution and on the selected time range. The step value can be customized by clickhing `Override step value` checkbox.
-
-VMUI allows investigating correlations between two queries on the same graph. Just click `+Query` button, enter the second query in the newly appeared input field and press `Ctrl+Enter`. Results for both queries should be displayed simultaneously on the same graph. Every query has its own vertical scale, which is displayed on the left and the right side of the graph. Lines for the second query are dashed.
-
-See the [example VMUI at VictoriaMetrics playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/?g0.expr=100%20*%20sum(rate(process_cpu_seconds_total))%20by%20(job)&g0.range_input=1d).
-
 ## How to build from sources
 
 We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
@@ -660,7 +756,7 @@ The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is pos
 by setting it via `<ROOT_IMAGE>` environment variable.
 For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
-```bash
+```console
 ROOT_IMAGE=scratch make package-victoria-metrics
 ```
 
@@ -672,8 +768,8 @@ More details may be found [here](https://github.com/VictoriaMetrics/VictoriaMetr
 
 ## Setting up service
 
-Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics as a service in your OS.
-There is also [snap package for Ubuntu](https://snapcraft.io/victoriametrics).
+Read [instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics
+as a service for your OS. A [snap package](https://snapcraft.io/victoriametrics) is available for Ubuntu.
 
 ## How to work with snapshots
 
@@ -766,13 +862,18 @@ for metrics to export. Use `{__name__!=""}` selector for fetching all the time s
 The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
 Each JSON line contains samples for a single time series. An example output:
 
-```jsonl
+```json
 {"metric":{"__name__":"up","job":"node_exporter","instance":"localhost:9100"},"values":[0,0,0],"timestamps":[1549891472010,1549891487724,1549891503438]}
 {"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
 ```
 
 Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
 unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+For example:
+```console
+curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
+curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+```
 
 Optional `max_rows_per_line` arg may be added to the request for limiting the maximum number of rows exported per each JSON line.
 Optional `reduce_mem_usage=1` arg may be added to the request for reducing memory usage when exporting big number of time series.
@@ -781,10 +882,14 @@ In this case the output may contain multiple lines with samples for the same tim
 Pass `Accept-Encoding: gzip` HTTP header in the request to `/api/v1/export` in order to reduce network bandwidth during exporing big amounts
 of time series data. This enables gzip compression for the exported data. Example for exporting gzipped data:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -H 'Accept-Encoding: gzip' http://localhost:8428/api/v1/export -d 'match[]={__name__!=""}' > data.jsonl.gz
 ```
 
+</div>
+
 The maximum duration for each request to `/api/v1/export` is limited by `-search.maxExportDuration` command-line flag.
 
 Exported data can be imported via POST'ing it to [/api/v1/import](#how-to-import-data-in-json-line-format).
@@ -812,6 +917,11 @@ for metrics to export.
 
 Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
 unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+For example:
+```console
+curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
+curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+```
 
 The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
 
@@ -825,8 +935,8 @@ for metrics to export. Use `{__name__=~".*"}` selector for fetching all the time
 
 On large databases you may experience problems with limit on the number of time series, which can be exported. In this case you need to adjust `-search.maxExportSeries` command-line flag:
 
-```bash
-# count unique timeseries in database
+```console
+# count unique time series in database
 wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq '.data[0]'
 
 # relaunch victoriametrics with search.maxExportSeries more than value from previous command
@@ -834,6 +944,11 @@ wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq
 
 Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
 unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+For example:
+```console
+curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
+curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+```
 
 The exported data can be imported to VictoriaMetrics via [/api/v1/import/native](#how-to-import-data-in-native-format).
 The native export format may change in incompatible way between VictoriaMetrics releases, so the data exported from the release X
@@ -862,7 +977,7 @@ Time series data can be imported into VictoriaMetrics via any supported data ing
 
 Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
 
-```bash
+```console
 # Export the data from <source-victoriametrics>:
 curl http://source-victoriametrics:8428/api/v1/export -d 'match={__name__!=""}' > exported_data.jsonl
 
@@ -872,7 +987,7 @@ curl -X POST http://destination-victoriametrics:8428/api/v1/import -T exported_d
 
 Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import` for importing gzipped data:
 
-```bash
+```console
 # Export gzipped data from <source-victoriametrics>:
 curl -H 'Accept-Encoding: gzip' http://source-victoriametrics:8428/api/v1/export -d 'match={__name__!=""}' > exported_data.jsonl.gz
 
@@ -893,7 +1008,7 @@ The specification of VictoriaMetrics' native format may yet change and is not fo
 
 If you have a native format file obtained via [/api/v1/export/native](#how-to-export-data-in-native-format) however this is the most efficient protocol for importing data in.
 
-```bash
+```console
 # Export the data from <source-victoriametrics>:
 curl http://source-victoriametrics:8428/api/v1/export/native -d 'match={__name__!=""}' > exported_data.bin
 
@@ -934,20 +1049,20 @@ Each request to `/api/v1/import/csv` may contain arbitrary number of CSV lines.
 
 Example for importing CSV data via `/api/v1/import/csv`:
 
-```bash
+```console
 curl -d "GOOG,1.23,4.56,NYSE" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
 curl -d "MSFT,3.21,1.67,NASDAQ" 'http://localhost:8428/api/v1/import/csv?format=2:metric:ask,3:metric:bid,1:label:ticker,4:label:market'
 ```
 
 After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
 
-```bash
+```console
 curl -G 'http://localhost:8428/api/v1/export' -d 'match[]={ticker!=""}'
 ```
 
 The following response should be returned:
 
-```bash
+```json
 {"metric":{"__name__":"bid","market":"NASDAQ","ticker":"MSFT"},"values":[1.67],"timestamps":[1583865146520]}
 {"metric":{"__name__":"bid","market":"NYSE","ticker":"GOOG"},"values":[4.56],"timestamps":[1583865146495]}
 {"metric":{"__name__":"ask","market":"NASDAQ","ticker":"MSFT"},"values":[3.21],"timestamps":[1583865146520]}
@@ -965,29 +1080,41 @@ VictoriaMetrics accepts data in [Prometheus exposition format](https://github.co
 and in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md)
 via `/api/v1/import/prometheus` path. For example, the following line imports a single line in Prometheus exposition format into VictoriaMetrics:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -d 'foo{bar="baz"} 123' -X POST 'http://localhost:8428/api/v1/import/prometheus'
 ```
 
+</div>
+
 The following command may be used for verifying the imported data:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__=~"foo"}'
 ```
 
+</div>
+
 It should return something like the following:
 
-```
+```json
 {"metric":{"__name__":"foo","bar":"baz"},"values":[123],"timestamps":[1594370496905]}
 ```
 
 Pass `Content-Encoding: gzip` HTTP request header to `/api/v1/import/prometheus` for importing gzipped data:
 
-```bash
+<div class="with-copy" markdown="1">
+
+```console
 # Import gzipped data to <destination-victoriametrics>:
 curl -X POST -H 'Content-Encoding: gzip' http://destination-victoriametrics:8428/api/v1/import/prometheus -T prometheus_data.gz
 ```
 
+</div>
+
 Extra labels may be added to all the imported metrics by passing `extra_label=name=value` query args.
 For example, `/api/v1/import/prometheus?extra_label=foo=bar` would add `{foo="bar"}` label to all the imported metrics.
 
@@ -1020,7 +1147,9 @@ Example contents for `-relabelConfig` file:
   regex: true
 ```
 
-See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details about relabeling in VictoriaMetrics.
+VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling.
+See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
+
 
 ## Federation
 
@@ -1028,8 +1157,13 @@ VictoriaMetrics exports [Prometheus-compatible federation data](https://promethe
 at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
 
 Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval.
-`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. By default, the last point
-on the interval `[now - max_lookback ... now]` is scraped for each time series. The default value for `max_lookback` is `5m` (5 minutes), but it can be overridden.
+`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+For example:
+```console
+curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
+curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=2022-06-06T19:25:48+00:00' -d 'end=2022-06-06T19:29:07+00:00'
+```
+By default, the last point on the interval `[now - max_lookback ... now]` is scraped for each time series. The default value for `max_lookback` is `5m` (5 minutes), but it can be overridden with `max_lookback` query arg.
 For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
 with scrape intervals exceeding `5m`.
 
@@ -1053,7 +1187,7 @@ It is recommended leaving the following amounts of spare resources:
 
 * 50% of free RAM for reducing the probability of OOM (out of memory) crashes and slowdowns during temporary spikes in workload.
 * 50% of spare CPU for reducing the probability of slowdowns during temporary spikes in workload.
-* At least 30% of free storage space at the directory pointed by `-storageDataPath` command-line flag. See also `-storage.minFreeDiskSpaceBytes` command-line flag description [here](#list-of-command-line-flags).
+* At least [20% of free storage space](#storage) at the directory pointed by `-storageDataPath` command-line flag. See also `-storage.minFreeDiskSpaceBytes` command-line flag description [here](#list-of-command-line-flags).
 
 See also [resource usage limits docs](#resource-usage-limits).
 
@@ -1071,7 +1205,7 @@ By default VictoriaMetrics is tuned for an optimal resource usage under typical
 - `-search.maxTagKeys` limits the number of items, which may be returned from [/api/v1/labels](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names). This endpoint is used mostly by Grafana for auto-completion of label names. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxTagKeys` to quite low value in order to limit CPU and memory usage.
 - `-search.maxTagValues` limits the number of items, which may be returned from [/api/v1/label/.../values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values). This endpoint is used mostly by Grafana for auto-completion of label values. Queries to this endpoint may take big amounts of CPU time and memory when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). In this case it might be useful to set the `-search.maxTagValues` to quite low value in order to limit CPU and memory usage.
 
-See also [capacity planning docs](#capacity-planning).
+See also [cardinality limiter](#cardinality-limiter) and [capacity planning docs](#capacity-planning).
 
 
 ## High availability
@@ -1079,7 +1213,7 @@ See also [capacity planning docs](#capacity-planning).
 * Install multiple VictoriaMetrics instances in distinct datacenters (availability zones).
 * Pass addresses of these instances to [vmagent](https://docs.victoriametrics.com/vmagent.html) via `-remoteWrite.url` command-line flag:
 
-```bash
+```console
 /path/to/vmagent -remoteWrite.url=http://<victoriametrics-addr-1>:8428/api/v1/write -remoteWrite.url=http://<victoriametrics-addr-2>:8428/api/v1/write
 ```
 
@@ -1098,7 +1232,7 @@ remote_write:
 
 * Apply the updated config:
 
-```bash
+```console
 kill -HUP `pidof prometheus`
 ```
 
@@ -1136,23 +1270,33 @@ values and timestamps. These are sorted and compressed raw time series values. A
 index files for searching for specific series in the values and timestamps files.
 
 `Parts` are periodically merged into the bigger parts. The resulting `part` is constructed
-under `<-storageDataPath>/data/{small,big}/YYYY_MM/tmp` subdirectory. When the resulting `part` is complete, it is atomically moved from the `tmp`
+under `<-storageDataPath>/data/{small,big}/YYYY_MM/tmp` subdirectory.
+When the resulting `part` is complete, it is atomically moved from the `tmp`
 to its own subdirectory, while the source parts are atomically removed. The end result is that the source
 parts are substituted by a single resulting bigger `part` in the `<-storageDataPath>/data/{small,big}/YYYY_MM/` directory.
+
+VictoriaMetrics doesn't merge parts if their summary size exceeds free disk space.
+This prevents from potential out of disk space errors during merge.
+The number of parts may significantly increase over time under free disk space shortage.
+This increases overhead during data querying, since VictoriaMetrics needs to read data from
+bigger number of parts per each request. That's why it is recommended to have at least 20%
+of free disk space under directory pointed by `-storageDataPath` command-line flag.
+
 Information about merging process is available in [single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
 and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176) Grafana dashboards.
 See more details in [monitoring docs](#monitoring).
 
-The `merge` process is usually named "compaction", because the resulting `part` size is usually smaller than
-the sum of the source `parts`. There are following benefits of doing the merge process:
+The `merge` process improves compression rate and keeps number of `parts` on disk relatively low.
+Benefits of doing the merge process are the following:
 
-* it improves query performance, since lower number of `parts` are inspected with each query;
-* it reduces the number of data files, since each `part`contains fixed number of files;
-* better compression rate for the resulting part.
+* it improves query performance, since lower number of `parts` are inspected with each query
+* it reduces the number of data files, since each `part` contains fixed number of files
+* various background maintenance tasks such as [de-duplication](#deduplication), [downsampling](#downsampling)
+  and [freeing up disk space for the deleted time series](#how-to-delete-time-series) are performed during the merge.
 
 Newly added `parts` either appear in the storage or fail to appear.
 Storage never contains partially created parts. The same applies to merge process — `parts` are either fully
-merged into a new `part` or fail to merge. There are no partially merged `parts` in MergeTree.
+merged into a new `part` or fail to merge. MergeTree doesn't contain partially merged `parts`.
 `Part` contents in MergeTree never change. Parts are immutable. They may be only deleted after the merge
 to a bigger `part` or when the `part` contents goes outside the configured `-retentionPeriod`.
 
@@ -1208,7 +1352,7 @@ The downsampling can be evaluated for free by downloading and using enterprise b
 
 ## Multi-tenancy
 
-Single-node VictoriaMetrics doesn't support multi-tenancy. Use [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) instead.
+Single-node VictoriaMetrics doesn't support multi-tenancy. Use the [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) instead.
 
 ## Scalability and cluster version
 
@@ -1216,7 +1360,7 @@ Though single-node VictoriaMetrics cannot scale to multiple nodes, it is optimiz
 This means that a single-node VictoriaMetrics may scale vertically and substitute a moderately sized cluster built with competing solutions
 such as Thanos, Uber M3, InfluxDB or TimescaleDB. See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
 
-So try single-node VictoriaMetrics at first and then [switch to cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) if you still need
+So try single-node VictoriaMetrics at first and then [switch to the cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) if you still need
 horizontally scalable long-term remote storage for really large Prometheus deployments.
 [Contact us](mailto:info@victoriametrics.com) for enterprise support.
 
@@ -1232,8 +1376,13 @@ Additionally, alerting can be set up with the following tools:
 
 ## Security
 
-Do not forget protecting sensitive endpoints in VictoriaMetrics when exposing it to untrusted networks such as the internet.
-Consider setting the following command-line flags:
+General security recommendations:
+
+- All the VictoriaMetrics components must run in protected private networks without direct access from untrusted networks such as Internet. The exception is [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html).
+- All the requests from untrusted networks to VictoriaMetrics components must go through auth proxy such as vmauth or vmgateway. The proxy must be set up with proper authentication and authorization.
+- Prefer using lists of allowed API endpoints, while disallowing access to other endpoints when configuring auth proxy in front of VictoriaMetrics components.
+
+VictoriaMetrics provides the following security-related command-line flags:
 
 * `-tls`, `-tlsCertFile` and `-tlsKeyFile` for switching from HTTP to HTTPS.
 * `-httpAuth.username` and `-httpAuth.password` for protecting all the HTTP endpoints
@@ -1243,20 +1392,18 @@ Consider setting the following command-line flags:
 * `-forceMergeAuthKey` for protecting `/internal/force_merge` endpoint. See [force merge docs](#forced-merge).
 * `-search.resetCacheAuthKey` for protecting `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
 * `-configAuthKey` for protecting `/config` endpoint, since it may contain sensitive information such as passwords.
-
-- `-pprofAuthKey` for protecting `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
+* `-flagsAuthKey` for protecting `/flags` endpoint.
+* `-pprofAuthKey` for protecting `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
+* `-denyQueryTracing` for disallowing [query tracing](#query-tracing).
 
 Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
-For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`.
-
-Prefer authorizing all the incoming requests from untrusted networks with [vmauth](https://docs.victoriametrics.com/vmauth.html)
-or similar auth proxy.
+For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`. This protects from unexpected requests from untrusted network interfaces.
 
 ## Tuning
 
-* There is no need for VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
+* No need in tuning for VictoriaMetrics - it uses reasonable defaults for command-line flags,
   which are automatically adjusted for the available CPU and RAM resources.
-* There is no need for Operating System tuning since VictoriaMetrics is optimized for default OS settings.
+* No need in tuning for Operating System - VictoriaMetrics is optimized for default OS settings.
   The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a).
   The recommendation is not specific for VictoriaMetrics only but also for any service which handles many HTTP connections and stores data on disk.
 * VictoriaMetrics is a write-heavy application and its performance depends on disk performance. So be careful with other
@@ -1267,31 +1414,35 @@ or similar auth proxy.
   If you plan to store more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
   then the following options are recommended to pass to `mkfs.ext4`:
 
-```bash
+```console
 mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
 ```
 
 ## Monitoring
 
-VictoriaMetrics exports internal metrics in Prometheus format at `/metrics` page.
-These metrics may be collected by [vmagent](https://docs.victoriametrics.com/vmagent.html)
-or Prometheus by adding the corresponding scrape config to it.
-Alternatively they can be self-scraped by setting `-selfScrapeInterval` command-line flag to duration greater than 0.
-For example, `-selfScrapeInterval=10s` would enable self-scraping of `/metrics` page with 10 seconds interval.
+VictoriaMetrics exports internal metrics in Prometheus exposition format at `/metrics` page.
+These metrics can be scraped via [vmagent](https://docs.victoriametrics.com/vmagent.html) or Prometheus.
+Alternatively, single-node VictoriaMetrics can self-scrape the metrics when `-selfScrapeInterval` command-line flag is 
+set to duration greater than 0. For example, `-selfScrapeInterval=10s` would enable self-scraping of `/metrics` page 
+with 10 seconds interval.
 
-There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176). There is also an [alternative dashboard for clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11831).
+Official Grafana dashboards available for [single-node](https://grafana.com/dashboards/10229) 
+and [clustered](https://grafana.com/grafana/dashboards/11176) VictoriaMetrics. 
+See an [alternative dashboard for clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11831) 
+created by community.
 
-Graphs on these dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
+Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it.
 
-It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.com/vmalert.html) or in Prometheus from [this config](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).
+We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
+via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus.
 
-The most interesting metrics are:
+The most interesting health metrics are the following:
 
 * `vm_cache_entries{type="storage/hour_metric_ids"}` - the number of time series with new data points during the last hour
   aka [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series).
 * `increase(vm_new_timeseries_created_total[1h])` - time series [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) during the previous hour.
 * `sum(vm_rows{type=~"storage/.*"})` - total number of `(timestamp, value)` data points in the database.
-* `sum(rate(vm_rows_inserted_total[5m]))` - ingestion rate, i.e. how many samples are inserted int the database per second.
+* `sum(rate(vm_rows_inserted_total[5m]))` - ingestion rate, i.e. how many samples are inserted in the database per second.
 * `vm_free_disk_space_bytes` - free space left at `-storageDataPath`.
 * `sum(vm_data_size_bytes)` - the total size of data on disk.
 * `increase(vm_slow_row_inserts_total[5m])` - the number of slow inserts during the last 5 minutes.
@@ -1301,19 +1452,83 @@ The most interesting metrics are:
   If this number remains high during extended periods of time, then it is likely more RAM is needed for optimal handling
   of the current number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series).
 
-VictoriaMetrics also exposes currently running queries with their execution times at `/api/v1/status/active_queries` page.
-
-See the example of alerting rules for VM components [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).
+VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page.
 
 ## TSDB stats
 
 VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way similar to Prometheus - see [these Prometheus docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). VictoriaMetrics accepts the following optional query args at `/api/v1/status/tsdb` page:
 
 * `topN=N` where `N` is the number of top entries to return in the response. By default top 10 entries are returned.
-* `date=YYYY-MM-DD` where `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day.
+* `date=YYYY-MM-DD` where `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day. Pass `date=1970-01-01` in order to collect global stats across all the days.
+* `focusLabel=LABEL_NAME` returns label values with the highest number of time series for the given `LABEL_NAME` in the `seriesCountByFocusLabelValue` list.
 * `match[]=SELECTOR` where `SELECTOR` is an arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for series to take into account during stats calculation. By default all the series are taken into account.
 * `extra_label=LABEL=VALUE`. See [these docs](#prometheus-querying-api-enhancements) for more details.
 
+VictoriaMetrics provides an UI on top of `/api/v1/status/tsdb` - see [cardinality explorer docs](#cardinality-explorer).
+
+## Query tracing
+
+VictoriaMetrics supports query tracing, which can be used for determining bottlenecks during query processing.
+
+Query tracing can be enabled for a specific query by passing `trace=1` query arg.
+In this case VictoriaMetrics puts query trace into `trace` field in the output JSON.
+
+For example, the following command:
+
+```console
+curl http://localhost:8428/api/v1/query_range -d 'query=2*rand()' -d 'start=-1h' -d 'step=1m' -d 'trace=1' | jq '.trace'
+```
+
+would return the following trace:
+
+```json
+{
+  "duration_msec": 0.099,
+  "message": "/api/v1/query_range: start=1654034340000, end=1654037880000, step=60000, query=\"2*rand()\": series=1",
+  "children": [
+    {
+      "duration_msec": 0.034,
+      "message": "eval: query=2 * rand(), timeRange=[1654034340000..1654037880000], step=60000, mayCache=true: series=1, points=60, pointsPerSeries=60",
+      "children": [
+        {
+          "duration_msec": 0.032,
+          "message": "binary op \"*\": series=1",
+          "children": [
+            {
+              "duration_msec": 0.009,
+              "message": "eval: query=2, timeRange=[1654034340000..1654037880000], step=60000, mayCache=true: series=1, points=60, pointsPerSeries=60"
+            },
+            {
+              "duration_msec": 0.017,
+              "message": "eval: query=rand(), timeRange=[1654034340000..1654037880000], step=60000, mayCache=true: series=1, points=60, pointsPerSeries=60",
+              "children": [
+                {
+                  "duration_msec": 0.015,
+                  "message": "transform rand(): series=1"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "duration_msec": 0.004,
+      "message": "sort series by metric name and labels"
+    },
+    {
+      "duration_msec": 0.044,
+      "message": "generate /api/v1/query_range response for series=1, points=60"
+    }
+  ]
+}
+```
+
+All the durations and timestamps in traces are in milliseconds.
+
+Query tracing is allowed by default. It can be denied by passing `-denyQueryTracing` command-line flag to VictoriaMetrics.
+
+
 ## Cardinality limiter
 
 By default VictoriaMetrics doesn't limit the number of stored time series. The limit can be enforced by setting the following command-line flags:
@@ -1388,7 +1603,7 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet
   It may be needed in order to suppress default gap filling algorithm used by VictoriaMetrics - by default it assumes
   each time series is continuous instead of discrete, so it fills gaps between real samples with regular intervals.
 
-* Metrics and labels leading to [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) or [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) can be determined at `/api/v1/status/tsdb` page. See [these docs](#tsdb-stats) for details.
+* Metrics and labels leading to [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) or [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) can be determined via [cardinality explorer](#cardinality-explorer) and via [/api/v1/status/tsdb](#tsdb-stats) endpoint.
 
 * New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics.
 
@@ -1423,7 +1638,8 @@ The panel `Cache usage %` in `Troubleshooting` section shows the percentage of u
 from the allowed size by type. If the percentage is below 100%, then no further tuning needed.
 
 Please note, default cache sizes were carefully adjusted accordingly to the most
-practical scenarios and workloads. Change the defaults only if you understand the implications.
+practical scenarios and workloads. Change the defaults only if you understand the implications
+and vmstorage has enough free memory to accommodate new cache sizes.
 
 To override the default values see command-line flags with `-storage.cacheSize` prefix.
 See the full description of flags [here](#list-of-command-line-flags).
@@ -1520,7 +1736,7 @@ VictoriaMetrics provides handlers for collecting the following [Go profiles](htt
 
 <div class="with-copy" markdown="1">
 
-```bash
+```console
 curl http://0.0.0.0:8428/debug/pprof/heap > mem.pprof
 ```
 
@@ -1530,7 +1746,7 @@ curl http://0.0.0.0:8428/debug/pprof/heap > mem.pprof
 
 <div class="with-copy" markdown="1">
 
-```bash
+```console
 curl http://0.0.0.0:8428/debug/pprof/profile > cpu.pprof
 ```
 
@@ -1622,9 +1838,10 @@ Files included in each folder:
 ### We kindly ask
 
 * Please don't use any other font instead of suggested.
-* There should be sufficient clear space around the logo.
+* To keep enough clear space around the logo.
 * Do not change spacing, alignment, or relative locations of the design elements.
-* Do not change the proportions of any of the design elements or the design itself. You    may resize as needed but must retain all proportions.
+* Do not change the proportions for any of the design elements or the design itself. 
+  You may resize as needed but must retain all proportions.
 
 ## List of command-line flags
 
@@ -1646,6 +1863,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries
   -denyQueriesOutsideRetention
      Whether to deny queries outside of the configured -retentionPeriod. When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
+  -denyQueryTracing
+     Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/#query-tracing
   -downsampling.period array
      Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1763,6 +1982,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name (default "0")
   -promscrape.cluster.membersCount int
      The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets
+  -promscrape.cluster.name string
+     Optional name of the cluster. If multiple vmagent clusters scrape the same targets, then each cluster must have unique name in order to properly de-duplicate samples received from these clusters. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2679
   -promscrape.cluster.replicationFactor int
      The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 1, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
   -promscrape.config string
@@ -1831,7 +2052,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.suppressDuplicateScrapeTargetErrors
      Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
-     Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
+     Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay
+  -promscrape.suppressScrapeErrorsDelay duration
+     The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
   -relabelConfig string
      Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
   -relabelDebug
@@ -1839,6 +2062,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -retentionPeriod value
      Data with timestamps outside the retentionPeriod is automatically deleted
      The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1)
+  -retentionTimezoneOffset duration
+     The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
   -search.cacheTimestampOffset duration
      The maximum duration since the current time for response data, which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset (default 5m0s)
   -search.disableAutoCacheReset
@@ -1858,9 +2083,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxExportDuration duration
      The maximum duration for /api/v1/export call (default 720h0m0s)
   -search.maxExportSeries int
-     The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage (default 1000000)
+     The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage (default 10000000)
   -search.maxFederateSeries int
-     The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 300000)
+     The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
   -search.maxGraphiteSeries int
      The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
   -search.maxLookback duration
@@ -1879,7 +2104,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxSamplesPerSeries int
      The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000)
   -search.maxSeries int
-     The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 10000)
+     The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 100000)
   -search.maxStalenessInterval duration
      The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons
   -search.maxStatusRequestDuration duration
@@ -1887,7 +2112,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxStepForPointsAdjustment duration
      The maximum step when /api/v1/query_range handler adjusts points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data (default 1m0s)
   -search.maxTSDBStatusSeries int
-     The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage (default 1000000)
+     The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage (default 10000000)
   -search.maxTagKeys int
      The maximum number of tag keys returned from /api/v1/labels (default 100000)
   -search.maxTagValueSuffixesPerSearch int
@@ -1929,6 +2154,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -storage.cacheSizeIndexDBIndexBlocks size
      Overrides max size for indexdb/indexBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning
      Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+  -storage.cacheSizeIndexDBTagFilters size
+     Overrides max size for indexdb/tagFilters cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning
+     Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -storage.cacheSizeStorageTSID size
      Overrides max size for storage/tsid cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning
      Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)

app/victoria-metrics/main.go

@@ -92,7 +92,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"vmui", "Web UI"},
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},

app/vmagent/README.md

@@ -73,7 +73,7 @@ Pass `-help` to `vmagent` in order to see [the full list of supported command-li
 
 * Sending `SUGHUP` signal to `vmagent` process:
 
-  ```bash
+  ```console
   kill -SIGHUP `pidof vmagent`
   ```
 
@@ -252,12 +252,13 @@ Labels can be added to metrics by the following mechanisms:
 VictoriaMetrics components (including `vmagent`) support Prometheus-compatible relabeling.
 They provide the following additional actions on top of actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config):
 
-* `replace_all`: replaces all of the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`.
-* `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`.
-* `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal.
-* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
-* `keep_metrics`: keeps all the metrics with names matching the given `regex`.
-* `drop_metrics`: drops all the metrics with names matching the given `regex`.
+* `replace_all`: replaces all of the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`
+* `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`
+* `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal
+* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal
+* `keep_metrics`: keeps all the metrics with names matching the given `regex`
+* `drop_metrics`: drops all the metrics with names matching the given `regex`
+* `graphite`: applies Graphite-style relabeling to metric name. See [these docs](#graphite-relabeling)
 
 The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent:
 
@@ -305,6 +306,38 @@ You can read more about relabeling in the following articles:
 * [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
 * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
 
+## Graphite relabeling
+
+VictoriaMetrics components support `action: graphite` relabeling rules, which allow extracting various parts from Graphite-style metrics
+into the configured labels with the syntax similar to [Glob matching in statsd_exporter](https://github.com/prometheus/statsd_exporter#glob-matching).
+Note that the `name` field must be substituted with explicit `__name__` option under `labels` section.
+If `__name__` option is missing under `labels` section, then the original Graphite-style metric name is left unchanged.
+
+For example, the following relabeling rule generates `requests_total{job="app42",instance="host124:8080"}` metric
+from "app42.host123.requests.total" Graphite-style metric:
+
+```yaml
+- action: graphite
+  match: "*.*.*.total"
+  labels:
+    __name__: "${3}_total"
+    job: "$1"
+    instance: "${2}:8080"
+```
+
+Important notes about `action: graphite` relabeling rules:
+
+- The relabeling rule is applied only to metrics, which match the given `match` expression. Other metrics remain unchanged.
+- The `*` matches the maximum possible number of chars until the next dot or until the next part of the `match` expression whichever comes first.
+  It may match zero chars if the next char is `.`.
+  For example, `match: "app*foo.bar"` matches `app42foo.bar` and `42` becomes available to use at `labels` section via `$1` capture group.
+- The `$0` capture group matches the original metric name.
+- The relabeling rules are executed in order defined in the original config.
+
+The `action: graphite` relabeling rules are easier to write and maintain than `action: replace` for labels extraction from Graphite-style metric names.
+Additionally, the `action: graphite` relabeling rules usually work much faster than the equivalent `action: replace` rules.
+
+
 ## Prometheus staleness markers
 
 `vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) to `-remoteWrite.url` in the following cases:
@@ -371,8 +404,12 @@ start a cluster of three `vmagent` instances, where each target is scraped by tw
 ```
 
 If each target is scraped by multiple `vmagent` instances, then data deduplication must be enabled at remote storage pointed by `-remoteWrite.url`.
+The `-dedup.minScrapeInterval` must be set to the `scrape_interval` configured at `-promscrape.config`.
 See [these docs](https://docs.victoriametrics.com/#deduplication) for details.
 
+If multiple `vmagent` clusters scrape the same set of targets, then each cluster must have unique value for the `-promscrape.cluster.name` command-line flag.
+This is needed for proper data de-duplication. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2679) for details.
+
 ## Scraping targets via a proxy
 
 `vmagent` supports scraping targets via http, https and socks5 proxies. Proxy address must be specified in `proxy_url` option. For example, the following scrape config instructs
@@ -556,7 +593,7 @@ Every Kafka message may contain multiple lines in `influx`, `prometheus`, `graph
 
 The following command starts `vmagent`, which reads metrics in InfluxDB line protocol format from Kafka broker at `localhost:9092` from the topic `metrics-by-telegraf` and sends them to remote storage at `http://localhost:8428/api/v1/write`:
 
-```bash
+```console
 ./bin/vmagent -remoteWrite.url=http://localhost:8428/api/v1/write \
        -kafka.consumer.topic.brokers=localhost:9092 \
        -kafka.consumer.topic.format=influx \
@@ -618,13 +655,13 @@ Two types of auth are supported:
 
 * sasl with username and password:
 
-```bash
+```console
 ./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SASL_SSL&sasl.mechanisms=PLAIN -remoteWrite.basicAuth.username=user -remoteWrite.basicAuth.password=password
 ```
 
 * tls certificates:
 
-```bash
+```console
 ./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SSL -remoteWrite.tlsCAFile=/opt/ca.pem -remoteWrite.tlsCertFile=/opt/cert.pem -remoteWrite.tlsKeyFile=/opt/key.pem
 ```
 
@@ -653,7 +690,7 @@ The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
 The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
-```bash
+```console
 ROOT_IMAGE=scratch make package-vmagent
 ```
 
@@ -681,7 +718,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 <div class="with-copy" markdown="1">
 
-```bash
+```console
 curl http://0.0.0.0:8429/debug/pprof/heap > mem.pprof
 ```
 
@@ -691,7 +728,7 @@ curl http://0.0.0.0:8429/debug/pprof/heap > mem.pprof
 
 <div class="with-copy" markdown="1">
 
-```bash
+```console
 curl http://0.0.0.0:8429/debug/pprof/profile > cpu.pprof
 ```
 
@@ -719,6 +756,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -datadog.maxInsertRequestSize size
      The maximum size in bytes of a single DataDog POST request to /api/v1/series
      Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
+  -denyQueryTracing
+     Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/#query-tracing
   -dryRun
      Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
   -enableTCP6
@@ -845,6 +884,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name (default "0")
   -promscrape.cluster.membersCount int
      The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets
+  -promscrape.cluster.name string
+     Optional name of the cluster. If multiple vmagent clusters scrape the same targets, then each cluster must have unique name in order to properly de-duplicate samples received from these clusters. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2679
   -promscrape.cluster.replicationFactor int
      The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 1, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/#deduplication (default 1)
   -promscrape.config string
@@ -913,7 +954,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.suppressDuplicateScrapeTargetErrors
      Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
   -promscrape.suppressScrapeErrors
-     Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
+     Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay
+  -promscrape.suppressScrapeErrorsDelay duration
+     The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
   -remoteWrite.aws.accessKey array
      Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
      Supports an array of values separated by comma or specified via multiple flags.

app/vmagent/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"embed"
 	"flag"
 	"fmt"
 	"io"
@@ -23,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -63,6 +65,12 @@ var (
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
 
+var (
+	//go:embed static
+	staticFiles  embed.FS
+	staticServer = http.FileServer(http.FS(staticFiles))
+)
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -159,7 +167,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
-			{"targets", "discovered targets list"},
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -170,6 +179,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if strings.HasPrefix(path, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		path = strings.TrimSuffix(path, "/")
+	}
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
@@ -254,15 +268,24 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/intake/":
+	case "/datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
+	case "/datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/service-discovery":
+		promscrapeServiceDiscoveryRequests.Inc()
+		promscrape.WriteServiceDiscovery(w, r)
+		return true
 	case "/target_response":
 		promscrapeTargetResponseRequests.Inc()
 		if err := promscrape.WriteTargetResponse(w, r); err != nil {
@@ -284,6 +307,22 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
+	case "/api/v1/status/config":
+		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeStatusConfigRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		var bb bytesutil.ByteBuffer
+		promscrape.WriteConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
+		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -305,11 +344,16 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			w.Write([]byte("OK"))
 		}
 		return true
+	default:
+		if strings.HasPrefix(r.URL.Path, "/static") {
+			staticServer.ServeHTTP(w, r)
+			return true
+		}
+		if remotewrite.MultitenancyEnabled() {
+			return processMultitenantRequest(w, r, path)
+		}
+		return false
 	}
-	if remotewrite.MultitenancyEnabled() {
-		return processMultitenantRequest(w, r, path)
-	}
-	return false
 }
 
 func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
@@ -327,6 +371,11 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
 	}
+	if strings.HasPrefix(p.Suffix, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
+	}
 	switch p.Suffix {
 	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
@@ -410,11 +459,16 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/intake/":
+	case "datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
+	case "datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	default:
 		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
 		return true
@@ -447,15 +501,18 @@ var (
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
-	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
+	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
+	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
 
-	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )

app/vmalert/Makefile

@@ -88,7 +88,7 @@ run-vmalert-sd: vmalert
 		-configCheckInterval=10s
 
 replay-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules-replay-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules-replay-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-remoteWrite.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \

app/vmalert/README.md

@@ -36,7 +36,7 @@ implementation and aims to be compatible with its syntax.
 
 To build `vmalert` from sources:
 
-```bash
+```console
 git clone https://github.com/VictoriaMetrics/VictoriaMetrics
 cd VictoriaMetrics
 make vmalert
@@ -52,12 +52,13 @@ To start using `vmalert` you will need the following things:
   aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
   [config file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go).
 * remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-  compatible storage to persist rules and alerts state info;
+  compatible storage to persist rules and alerts state info. To persist results to multiple destinations use vmagent
+  configured with multiple remote writes as a proxy;
 * remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
 
 Then configure `vmalert` accordingly:
 
-```bash
+```console
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard
     -datasource.url=http://localhost:8428 \  # PromQL compatible datasource
     -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
@@ -68,7 +69,7 @@ Then configure `vmalert` accordingly:
     -external.label=replica=a                # Multiple external labels may be set
 ```
 
-Note there's a separate `remoteRead.url` to allow writing results of
+Note there's a separate `remoteWrite.url` to allow writing results of
 alerting/recording rules into a different storage than the initial data that's
 queried. This allows using `vmalert` to aggregate data from a short-term,
 high-frequency, high-cardinality storage into a long-term storage with
@@ -101,6 +102,10 @@ name: <string>
 # How often rules in the group are evaluated.
 [ interval: <duration> | default = -evaluationInterval flag ]
 
+# Limit the number of alerts an alerting rule and series a recording
+# rule can produce. 0 is no limit.
+[ limit: <int> | default = 0 ]
+
 # How many rules execute at once within a group. Increasing concurrency may speed
 # up round execution speed.
 [ concurrency: <integer> | default = 1 ]
@@ -420,6 +425,21 @@ Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only rec
 
 See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
 
+#### Multiple remote writes
+
+For persisting recording or alerting rule results `vmalert` requires `-remoteWrite.url` to be set.
+But this flag supports only one destination. To persist rule results to multiple destinations
+we recommend using [vmagent](https://docs.victoriametrics.com/vmagent.html) as fan-out proxy:
+
+<img alt="vmalert multiple remote write destinations" src="vmalert_multiple_rw.png">
+
+In this topology, `vmalert` is configured to persist rule results to `vmagent`. And `vmagent`
+is configured to fan-out received data to two or more destinations.
+Using `vmagent` as a proxy provides additional benefits such as 
+[data persisting when storage is unreachable](https://docs.victoriametrics.com/vmagent.html#replication-and-high-availability),
+or time series modification via [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling).
+
+
 ### Web
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
@@ -490,7 +510,7 @@ max range per request:  8h20m0s
 
 In `replay` mode all groups are executed sequentially one-by-one. Rules within the group are
 executed sequentially as well (`concurrency` setting is ignored). Vmalert sends rule's expression
-to [/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) endpoint
+to [/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) endpoint
 of the configured `-datasource.url`. Returned data is then processed according to the rule type and
 backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
 Vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
@@ -525,7 +545,7 @@ There are following non-required `replay` flags:
   (rules which depend on each other) rules. It is expected, that remote storage will be able to persist
   previously accepted data during the delay, so data will be available for the subsequent queries.
   Keep it equal or bigger than `-remoteWrite.flushInterval`.
-* `replay.disableProgressBar` - whether to disable progress bar which shows progress work.
+* `-replay.disableProgressBar` - whether to disable progress bar which shows progress work.
   Progress bar may generate a lot of log records, which is not formatted as standard VictoriaMetrics logger.
   It could break logs parsing by external system and generate additional load on it.
 
@@ -535,6 +555,7 @@ See full description for these flags in `./vmalert --help`.
 
 * Graphite engine isn't supported yet;
 * `query` template function is disabled for performance reasons (might be changed in future);
+* `limit` group's param has no effect during replay (might be changed in future);
 
 ## Monitoring
 
@@ -1013,6 +1034,25 @@ It is recommended using
 
 * `vmalert` is located in `vmutils-*` archives there.
 
+### Docker image
+
+You can build `vmalert` docker image from source and push it to your own docker repository.
+Run the following commands from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics):
+```console
+make package-vmalert
+docker tag victoria-metrics/vmalert:version my-repo:my-version-name
+docker push my-repo:my-version-name
+```
+
+To run the built image in `victoria-metrics-k8s-stack` or `VMAlert` CR object apply the following config change:
+```yaml
+kind: VMAlert
+spec:
+  image:
+    repository: my-repo
+    tag: my-version-name
+```
+
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.17.

app/vmalert/alerting.go

@@ -240,7 +240,7 @@ const resolvedRetention = 15 * time.Minute
 
 // Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
 	start := time.Now()
 	qMetrics, err := ar.q.Query(ctx, ar.Expr, ts)
 	ar.mu.Lock()
@@ -307,7 +307,7 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal
 		a.ActiveAt = ts
 		ar.alerts[h] = a
 	}
-
+	var numActivePending int
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
@@ -324,12 +324,17 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal
 			}
 			continue
 		}
+		numActivePending++
 		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
 			a.State = notifier.StateFiring
 			a.Start = ts
 			alertsFired.Inc()
 		}
 	}
+	if limit > 0 && numActivePending > limit {
+		ar.alerts = map[uint64]*notifier.Alert{}
+		return nil, fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
+	}
 	return ar.toTimeSeries(ts.Unix()), nil
 }
 

app/vmalert/alerting_test.go

@@ -304,7 +304,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			for _, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
-				if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
+				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
 				// artificial delay between applying steps
@@ -624,14 +624,14 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	// successful attempt
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	_, err := ar.Exec(context.TODO(), time.Now())
+	_, err := ar.Exec(context.TODO(), time.Now(), 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// label `job` will collide with rule extra label and will make both time series equal
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err = ar.Exec(context.TODO(), time.Now())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
@@ -640,7 +640,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
-	_, err = ar.Exec(context.TODO(), time.Now())
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -649,6 +649,50 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 	}
 }
 
+func TestAlertingRuleLimit(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+	ar.For = time.Minute
+	testCases := []struct {
+		limit  int
+		err    string
+		tssNum int
+	}{
+		{
+			limit:  0,
+			tssNum: 4,
+		},
+		{
+			limit:  -1,
+			tssNum: 4,
+		},
+		{
+			limit:  1,
+			err:    "exec exceeded limit of 1 with 2 alerts",
+			tssNum: 0,
+		},
+		{
+			limit:  4,
+			tssNum: 4,
+		},
+	}
+	var (
+		err       error
+		timestamp = time.Now()
+	)
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
+	for _, testCase := range testCases {
+		_, err = ar.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+	fq.reset()
+}
+
 func TestAlertingRule_Template(t *testing.T) {
 	testCases := []struct {
 		rule      *AlertingRule
@@ -761,7 +805,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
 			fq.add(tc.metrics...)
-			if _, err := tc.rule.Exec(context.TODO(), time.Now()); err != nil {
+			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {

app/vmalert/config/config.go

@@ -27,6 +27,7 @@ type Group struct {
 	File        string
 	Name        string              `yaml:"name"`
 	Interval    *promutils.Duration `yaml:"interval,omitempty"`
+	Limit       int                 `yaml:"limit,omitempty"`
 	Rules       []Rule              `yaml:"rules"`
 	Concurrency int                 `yaml:"concurrency"`
 	// ExtraFilterLabels is a list label filters applied to every rule

app/vmalert/config/config_test.go

@@ -489,6 +489,22 @@ rules:
 name: TestGroup
 params:
     nocache: ["0"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`limit` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+limit: 5
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+limit: 10
 rules:
   - alert: foo
     expr: sum by(job) (up == 1)

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -2,6 +2,7 @@ groups:
   - name: ReplayGroup
     interval: 1m
     concurrency: 1
+    limit: 1000
     rules:
       - record: type:vm_cache_entries:rate5m
         expr: sum(rate(vm_cache_entries[5m])) by (type)

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -2,6 +2,7 @@ groups:
   - name: TestGroup
     interval: 2s
     concurrency: 2
+    limit: 1000
     params:
       denyPartialResponse: ["true"]
       extra_label: ["env=dev"]

app/vmalert/group.go

@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -18,7 +20,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 // Group is an entity for grouping rules
@@ -29,6 +30,7 @@ type Group struct {
 	Rules          []Rule
 	Type           datasource.Type
 	Interval       time.Duration
+	Limit          int
 	Concurrency    int
 	Checksum       string
 	LastEvaluation time.Time
@@ -90,6 +92,7 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Name:        cfg.Name,
 		File:        cfg.File,
 		Interval:    cfg.Interval.Duration(),
+		Limit:       cfg.Limit,
 		Concurrency: cfg.Concurrency,
 		Checksum:    cfg.Checksum,
 		Params:      cfg.Params,
@@ -215,6 +218,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Concurrency = newGroup.Concurrency
 	g.Params = newGroup.Params
 	g.Labels = newGroup.Labels
+	g.Limit = newGroup.Limit
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
@@ -282,7 +286,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		}
 
 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
-		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration)
+		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration, g.Limit)
 		for err := range errs {
 			if err != nil {
 				logger.Errorf("group %q: %s", g.Name, err)
@@ -360,12 +364,12 @@ type executor struct {
 	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }
 
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration) chan error {
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, ts, resolveDuration)
+			res <- e.exec(ctx, rule, ts, resolveDuration, limit)
 		}
 		close(res)
 		return res
@@ -378,7 +382,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.T
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, ts, resolveDuration)
+				res <- e.exec(ctx, r, ts, resolveDuration, limit)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -399,10 +403,10 @@ var (
 	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration, limit int) error {
 	execTotal.Inc()
 
-	tss, err := rule.Exec(ctx, ts)
+	tss, err := rule.Exec(ctx, ts, limit)
 	if err != nil {
 		execErrors.Inc()
 		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
@@ -434,11 +438,17 @@ func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDur
 		return nil
 	}
 
+	wg := sync.WaitGroup{}
 	for _, nt := range e.notifiers() {
-		if err := nt.Send(ctx, alerts); err != nil {
-			errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
-		}
+		wg.Add(1)
+		go func(nt notifier.Notifier) {
+			if err := nt.Send(ctx, alerts); err != nil {
+				errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
+			}
+			wg.Done()
+		}(nt)
 	}
+	wg.Wait()
 	return errGr.Err()
 }
 

app/vmalert/group_test.go

@@ -413,3 +413,42 @@ func TestPurgeStaleSeries(t *testing.T) {
 		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
 	)
 }
+
+func TestFaultyNotifier(t *testing.T) {
+	fq := &fakeQuerier{}
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+
+	r := newTestAlertingRule("instant", 0)
+	r.q = fq
+
+	fn := &fakeNotifier{}
+	e := &executor{
+		notifiers: func() []notifier.Notifier {
+			return []notifier.Notifier{
+				&faultyNotifier{},
+				fn,
+			}
+		},
+	}
+	delay := 5 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), delay)
+	defer cancel()
+
+	go func() {
+		_ = e.exec(ctx, r, time.Now(), 0, 10)
+	}()
+
+	tn := time.Now()
+	deadline := tn.Add(delay / 2)
+	for {
+		if fn.getCounter() > 0 {
+			return
+		}
+		if tn.After(deadline) {
+			break
+		}
+		tn = time.Now()
+		time.Sleep(time.Millisecond * 100)
+	}
+	t.Fatalf("alive notifier didn't receive notification by %v", deadline)
+}

app/vmalert/helpers_test.go

@@ -87,6 +87,18 @@ func (fn *fakeNotifier) getAlerts() []notifier.Alert {
 	return fn.alerts
 }
 
+type faultyNotifier struct {
+	fakeNotifier
+}
+
+func (fn *faultyNotifier) Send(ctx context.Context, _ []notifier.Alert) error {
+	d, ok := ctx.Deadline()
+	if ok {
+		time.Sleep(time.Until(d))
+	}
+	return fmt.Errorf("send failed")
+}
+
 func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
 	return metricWithValuesAndLabels(t, []float64{value}, labels...)
 }

app/vmalert/notifier/alert.go

@@ -163,9 +163,6 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
 	if !execute {
 		return nil
 	}
-	if !execute {
-		return nil
-	}
 	if err = tpl.Execute(dst, data); err != nil {
 		return fmt.Errorf("error evaluating annotation template: %w", err)
 	}

app/vmalert/notifier/config_watcher.go

@@ -166,7 +166,7 @@ func (cw *configWatcher) start() error {
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedRelabelConfigs, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}

app/vmalert/notifier/init.go

@@ -145,7 +145,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 		}
 
 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Minute)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Second*10)
 		if err != nil {
 			return nil, err
 		}

app/vmalert/recording.go

@@ -124,7 +124,7 @@ func (rr *RecordingRule) ExecRange(ctx context.Context, start, end time.Time) ([
 }
 
 // Exec executes RecordingRule expression via the given Querier.
-func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error) {
+func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
 	qMetrics, err := rr.q.Query(ctx, rr.Expr, ts)
 	rr.mu.Lock()
 	defer rr.mu.Unlock()
@@ -137,6 +137,11 @@ func (rr *RecordingRule) Exec(ctx context.Context, ts time.Time) ([]prompbmarsha
 		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
 	}
 
+	numSeries := len(qMetrics)
+	if limit > 0 && numSeries > limit {
+		return nil, fmt.Errorf("exec exceeded limit of %d with %d series", limit, numSeries)
+	}
+
 	duplicates := make(map[string]struct{}, len(qMetrics))
 	var tss []prompbmarshal.TimeSeries
 	for _, r := range qMetrics {

app/vmalert/recording_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func TestRecoridngRule_Exec(t *testing.T) {
+func TestRecordingRule_Exec(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -77,7 +77,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
 			tc.rule.q = fq
-			tss, err := tc.rule.Exec(context.TODO(), time.Now())
+			tss, err := tc.rule.Exec(context.TODO(), time.Now(), 0)
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
 			}
@@ -88,7 +88,7 @@ func TestRecoridngRule_Exec(t *testing.T) {
 	}
 }
 
-func TestRecoridngRule_ExecRange(t *testing.T) {
+func TestRecordingRule_ExecRange(t *testing.T) {
 	timestamp := time.Now()
 	testCases := []struct {
 		rule    *RecordingRule
@@ -169,7 +169,48 @@ func TestRecoridngRule_ExecRange(t *testing.T) {
 	}
 }
 
-func TestRecoridngRule_ExecNegative(t *testing.T) {
+func TestRecordingRuleLimit(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		limit int
+		err   string
+	}{
+		{
+			limit: 0,
+		},
+		{
+			limit: -1,
+		},
+		{
+			limit: 1,
+			err:   "exec exceeded limit of 1 with 3 series",
+		},
+		{
+			limit: 2,
+			err:   "exec exceeded limit of 2 with 3 series",
+		},
+	}
+	testMetrics := []datasource.Metric{
+		metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+	}
+	rule := &RecordingRule{Name: "job:foo", Labels: map[string]string{
+		"source": "test_limit",
+	}}
+	var err error
+	for _, testCase := range testCases {
+		fq := &fakeQuerier{}
+		fq.add(testMetrics...)
+		rule.q = fq
+		_, err = rule.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestRecordingRule_ExecNegative(t *testing.T) {
 	rr := &RecordingRule{Name: "job:foo", Labels: map[string]string{
 		"job": "test",
 	}}
@@ -178,7 +219,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	expErr := "connection reset by peer"
 	fq.setErr(errors.New(expErr))
 	rr.q = fq
-	_, err := rr.Exec(context.TODO(), time.Now())
+	_, err := rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -193,7 +234,7 @@ func TestRecoridngRule_ExecNegative(t *testing.T) {
 	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"))
 	fq.add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
-	_, err = rr.Exec(context.TODO(), time.Now())
+	_, err = rr.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}

app/vmalert/remotewrite/remotewrite.go

@@ -236,6 +236,14 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
+
+	// RFC standard compliant headers
+	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("Content-Type", "application/x-protobuf")
+
+	// Prometheus compliant headers
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+
 	if c.authCfg != nil {
 		if auth := c.authCfg.GetAuthHeader(); auth != "" {
 			req.Header.Set("Authorization", auth)

app/vmalert/remotewrite/remotewrite_test.go

@@ -80,6 +80,22 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("bad method %q", r.Method))
 		return
 	}
+
+	h := r.Header.Get("Content-Encoding")
+	if h != "snappy" {
+		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not snappy (%q)", h))
+	}
+
+	h = r.Header.Get("Content-Type")
+	if h != "application/x-protobuf" {
+		rw.err(w, fmt.Errorf("header read error: Content-Type is not x-protobuf (%q)", h))
+	}
+
+	h = r.Header.Get("X-Prometheus-Remote-Write-Version")
+	if h != "0.1.0" {
+		rw.err(w, fmt.Errorf("header read error: X-Prometheus-Remote-Write-Version is not 0.1.0 (%q)", h))
+	}
+
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		rw.err(w, fmt.Errorf("body read err: %w", err))

app/vmalert/replay.go

@@ -7,12 +7,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dmitryk-dk/pb/v3"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/dmitryk-dk/pb/v3"
 )
 
 var (
@@ -87,6 +88,10 @@ func (g *Group) replay(start, end time.Time, rw *remotewrite.Client) int {
 		"\nrequests to make: \t%d"+
 		"\nmax range per request: \t%v\n",
 		g.Name, g.Interval, iterations, step)
+	if g.Limit > 0 {
+		fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
+			g.Limit)
+	}
 	for _, rule := range g.Rules {
 		fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
 		var bar *pb.ProgressBar

app/vmalert/rule.go

@@ -15,9 +15,10 @@ type Rule interface {
 	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
-	// Exec executes the rule with given context at the given timestamp
-	Exec(ctx context.Context, ts time.Time) ([]prompbmarshal.TimeSeries, error)
-	// ExecRange executes the rule on the given time range
+	// Exec executes the rule with given context at the given timestamp and limit.
+	// returns an err if number of resulting time series exceeds the limit.
+	Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error)
+	// ExecRange executes the rule on the given time range.
 	ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error)
 	// UpdateWith performs modification of current Rule
 	// with fields of the given Rule.

app/vmalert/tpl/footer.qtpl

@@ -1,7 +1,7 @@
 {% func Footer() %}
         </main>
-        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
-        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script src="static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
+        <script src="static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
         <script type="text/javascript">
             function expandAll() {
                $('.collapse').addClass('show');
@@ -18,14 +18,14 @@
 
               $(".group-heading").click(function(e) {
                  let target = $(this).attr('data-bs-target');
-                 let el = $('#'+target);
+                 let el = $("#"+target);
                   new bootstrap.Collapse(el, {
                     toggle: true
                   });
               });
 
               var hash = window.location.hash.substr(1);
-              let group = $('#'+hash);
+              let group = $("#"+hash);
               if (group.length > 0) {
                 group.click();
               }

app/vmalert/tpl/footer.qtpl.go

@@ -22,8 +22,8 @@ func StreamFooter(qw422016 *qt422016.Writer) {
 //line app/vmalert/tpl/footer.qtpl:1
 	qw422016.N().S(`
         </main>
-        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
-        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script src="static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
+        <script src="static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
         <script type="text/javascript">
             function expandAll() {
                $('.collapse').addClass('show');
@@ -40,14 +40,14 @@ func StreamFooter(qw422016 *qt422016.Writer) {
 
               $(".group-heading").click(function(e) {
                  let target = $(this).attr('data-bs-target');
-                 let el = $('#'+target);
+                 let el = $("#"+target);
                   new bootstrap.Collapse(el, {
                     toggle: true
                   });
               });
 
               var hash = window.location.hash.substr(1);
-              let group = $('#'+hash);
+              let group = $("#"+hash);
               if (group.length > 0) {
                 group.click();
               }

app/vmalert/tpl/header.qtpl

@@ -3,7 +3,7 @@
 <html lang="en">
 <head>
     <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <link href="static/css/bootstrap.min.css" rel="stylesheet" />
     <style>
         body{
           min-height: 75rem;

app/vmalert/tpl/header.qtpl.go

@@ -35,7 +35,7 @@ func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
 	}
 //line app/vmalert/tpl/header.qtpl:5
 	qw422016.N().S(`</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <link href="static/css/bootstrap.min.css" rel="stylesheet" />
     <style>
         body{
           min-height: 75rem;

app/vmalert/utils/err_group.go

@@ -3,24 +3,34 @@ package utils
 import (
 	"fmt"
 	"strings"
+	"sync"
 )
 
 // ErrGroup accumulates multiple errors
 // and produces single error message.
 type ErrGroup struct {
+	mu   sync.Mutex
 	errs []error
 }
 
 // Add adds a new error to group.
-// Isn't thread-safe.
+// Is thread-safe.
 func (eg *ErrGroup) Add(err error) {
+	eg.mu.Lock()
 	eg.errs = append(eg.errs, err)
+	eg.mu.Unlock()
 }
 
 // Err checks if group contains at least
 // one error.
 func (eg *ErrGroup) Err() error {
-	if eg == nil || len(eg.errs) == 0 {
+	if eg == nil {
+		return nil
+	}
+
+	eg.mu.Lock()
+	defer eg.mu.Unlock()
+	if len(eg.errs) == 0 {
 		return nil
 	}
 	return eg
@@ -28,6 +38,9 @@ func (eg *ErrGroup) Err() error {
 
 // Error satisfies Error interface
 func (eg *ErrGroup) Error() string {
+	eg.mu.Lock()
+	defer eg.mu.Unlock()
+
 	if len(eg.errs) == 0 {
 		return ""
 	}

app/vmalert/utils/err_group_test.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 )
 
@@ -36,3 +37,29 @@ func TestErrGroup(t *testing.T) {
 		}
 	}
 }
+
+// TestErrGroupConcurrent supposed to test concurrent
+// use of error group.
+// Should be executed with -race flag
+func TestErrGroupConcurrent(t *testing.T) {
+	eg := new(ErrGroup)
+
+	const writersN = 4
+	payload := make(chan error, writersN)
+	for i := 0; i < writersN; i++ {
+		go func() {
+			for err := range payload {
+				eg.Add(err)
+			}
+		}()
+	}
+
+	const iterations = 500
+	for i := 0; i < iterations; i++ {
+		payload <- fmt.Errorf("error %d", i)
+		if i%10 == 0 {
+			_ = eg.Err()
+		}
+	}
+	close(payload)
+}

app/vmalert/vmalert_multiple_rw.excalidraw

@@ -0,0 +1,711 @@
+{
+  "type": "excalidraw",
+  "version": 2,
+  "source": "https://excalidraw.com",
+  "elements": [
+    {
+      "type": "rectangle",
+      "version": 797,
+      "versionNonce": 1977657992,
+      "isDeleted": false,
+      "id": "VgBUzo0blGR-Ijd2mQEEf",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 289.6802978515625,
+      "y": 399.3895568847656,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1194011660,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "miEbzHxOPXe4PEYvXiJp5"
+        },
+        {
+          "type": "arrow",
+          "id": "rcmiQfIWtfbTTlwxqr1sl"
+        },
+        {
+          "type": "arrow",
+          "id": "P-dpWlSTtnsux-zr5oqgF"
+        },
+        {
+          "type": "arrow",
+          "id": "oAToSPttH7aWoD_AqXGFX"
+        },
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "sxEhnxlbT7ldlSsmHDUHp"
+        },
+        {
+          "type": "arrow",
+          "id": "pD9DcILMxa6GaR1U5YyMO"
+        },
+        {
+          "type": "arrow",
+          "id": "HPEwr85wL4IedW0AgdArp"
+        },
+        {
+          "type": "arrow",
+          "id": "EyecK0YM9Cc8T6ju-nTOc"
+        },
+        {
+          "id": "xpdAlCCGgIMAgSaqQ4K65",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372487772,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 671,
+      "versionNonce": 1438327288,
+      "isDeleted": false,
+      "id": "e9TDm09y-GhPm84XWt0Jv",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 311.22686767578125,
+      "y": 420.4738006591797,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 83,
+      "height": 24,
+      "seed": 327273100,
+      "groupIds": [
+        "iBaXgbpyifSwPplm_GO5b"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372487772,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmagent",
+      "baseline": 20,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": null,
+      "originalText": "vmagent"
+    },
+    {
+      "type": "rectangle",
+      "version": 1247,
+      "versionNonce": 1809504904,
+      "isDeleted": false,
+      "id": "Sa4OBd1ZjD6itohm7Ll8z",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 542.2673645019531,
+      "y": 308.46409606933594,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 219.1235961914062,
+      "height": 44.74725341796875,
+      "seed": 126267060,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "he-SpFjCxEQEWpWny2kKP"
+        },
+        {
+          "type": "arrow",
+          "id": "-pjrKo16rOsasM8viZPJ-"
+        },
+        {
+          "id": "HPEwr85wL4IedW0AgdArp",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372610014,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1149,
+      "versionNonce": 1939391880,
+      "isDeleted": false,
+      "id": "we766A079lfGYu2_aC4Pl",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 629.1559448242188,
+      "y": 318.8975372314453,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 48,
+      "height": 24,
+      "seed": 478660236,
+      "groupIds": [
+        "ek-pq3umtz1yN-J_-preq"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372621140,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vm-1",
+      "baseline": 20,
+      "textAlign": "center",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "vm-1"
+    },
+    {
+      "type": "arrow",
+      "version": 337,
+      "versionNonce": 1739475336,
+      "isDeleted": false,
+      "id": "HPEwr85wL4IedW0AgdArp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 423.70701599121094,
+      "y": 431.0309448437124,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 107.82342529296875,
+      "height": 100.61778190120276,
+      "seed": 389863732,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElements": [],
+      "updated": 1655372610015,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": 0.6700023593531782,
+        "gap": 10.266586303710938
+      },
+      "endBinding": {
+        "elementId": "Sa4OBd1ZjD6itohm7Ll8z",
+        "focus": 0.9042666945544442,
+        "gap": 10.736923217773438
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          107.82342529296875,
+          -100.61778190120276
+        ]
+      ]
+    },
+    {
+      "type": "arrow",
+      "version": 429,
+      "versionNonce": 252631288,
+      "isDeleted": false,
+      "id": "EyecK0YM9Cc8T6ju-nTOc",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 424.7585906982422,
+      "y": 432.4328003132737,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 119.94342041015625,
+      "height": 83.58206327156176,
+      "seed": 981082124,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "boundElements": [],
+      "updated": 1655372623571,
+      "link": null,
+      "locked": false,
+      "startBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.6826568395144794,
+        "gap": 11.318161010742188
+      },
+      "endBinding": {
+        "elementId": "lXpACjXQqK7SZF_vrACjJ",
+        "focus": -0.8650156795513397,
+        "gap": 3.6341629028320312
+      },
+      "lastCommittedPoint": null,
+      "startArrowhead": null,
+      "endArrowhead": "arrow",
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          119.94342041015625,
+          83.58206327156176
+        ]
+      ]
+    },
+    {
+      "type": "rectangle",
+      "version": 979,
+      "versionNonce": 896077192,
+      "isDeleted": false,
+      "id": "X08ptHmEm7tCgoFbQntAR",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": -4.634010314941406,
+      "y": 402.69072341918945,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 123.7601318359375,
+      "height": 72.13211059570312,
+      "seed": 1000953848,
+      "groupIds": [
+        "IAd7y_6yDxq13U11FuJvH"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "miEbzHxOPXe4PEYvXiJp5"
+        },
+        {
+          "type": "arrow",
+          "id": "rcmiQfIWtfbTTlwxqr1sl"
+        },
+        {
+          "type": "arrow",
+          "id": "P-dpWlSTtnsux-zr5oqgF"
+        },
+        {
+          "type": "arrow",
+          "id": "oAToSPttH7aWoD_AqXGFX"
+        },
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "sxEhnxlbT7ldlSsmHDUHp"
+        },
+        {
+          "type": "arrow",
+          "id": "pD9DcILMxa6GaR1U5YyMO"
+        },
+        {
+          "type": "arrow",
+          "id": "HPEwr85wL4IedW0AgdArp"
+        },
+        {
+          "type": "arrow",
+          "id": "EyecK0YM9Cc8T6ju-nTOc"
+        },
+        {
+          "id": "xpdAlCCGgIMAgSaqQ4K65",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 844,
+      "versionNonce": 2073980664,
+      "isDeleted": false,
+      "id": "4lz3UmUePrjYOJGyMEsNo",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 16.912559509277344,
+      "y": 423.7749671936035,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 82,
+      "height": 24,
+      "seed": 808600456,
+      "groupIds": [
+        "IAd7y_6yDxq13U11FuJvH"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vmalert",
+      "baseline": 19,
+      "textAlign": "center",
+      "verticalAlign": "middle",
+      "containerId": null,
+      "originalText": "vmalert"
+    },
+    {
+      "id": "xpdAlCCGgIMAgSaqQ4K65",
+      "type": "arrow",
+      "x": 127.58199310302739,
+      "y": 437.3415815729096,
+      "width": 154.43469238281244,
+      "height": 0.2578931190849971,
+      "angle": 0,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "round",
+      "seed": 1769759112,
+      "version": 140,
+      "versionNonce": 1727929480,
+      "isDeleted": false,
+      "boundElements": null,
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false,
+      "points": [
+        [
+          0,
+          0
+        ],
+        [
+          154.43469238281244,
+          0.2578931190849971
+        ]
+      ],
+      "lastCommittedPoint": null,
+      "startBinding": {
+        "elementId": "X08ptHmEm7tCgoFbQntAR",
+        "focus": -0.042373209435744755,
+        "gap": 8.45587158203125
+      },
+      "endBinding": {
+        "elementId": "VgBUzo0blGR-Ijd2mQEEf",
+        "focus": -0.062483627408895646,
+        "gap": 7.663612365722656
+      },
+      "startArrowhead": null,
+      "endArrowhead": "arrow"
+    },
+    {
+      "type": "text",
+      "version": 896,
+      "versionNonce": 619040760,
+      "isDeleted": false,
+      "id": "d_hJkkcPArQGdFiPDbjtp",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 129.2102279663086,
+      "y": 404.1378517150879,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 141,
+      "height": 19,
+      "seed": 2108447992,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        }
+      ],
+      "updated": 1655372487773,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "persist results",
+      "baseline": 15,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "persist results"
+    },
+    {
+      "id": "P35cFQroIm2nrmm3Jlqgp",
+      "type": "text",
+      "x": -7.461128234863281,
+      "y": 483.3255958557129,
+      "width": 301,
+      "height": 20,
+      "angle": 0,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1314060792,
+      "version": 179,
+      "versionNonce": 139280376,
+      "isDeleted": false,
+      "boundElements": null,
+      "updated": 1655372636346,
+      "link": null,
+      "locked": false,
+      "text": " -remoteWrite.url=http://vmagent",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 16,
+      "containerId": null,
+      "originalText": " -remoteWrite.url=http://vmagent"
+    },
+    {
+      "type": "rectangle",
+      "version": 1339,
+      "versionNonce": 812947448,
+      "isDeleted": false,
+      "id": "lXpACjXQqK7SZF_vrACjJ",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 548.3361740112305,
+      "y": 487.1258888244629,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 219.1235961914062,
+      "height": 44.74725341796875,
+      "seed": 333549960,
+      "groupIds": [
+        "vuLTnxw8A0DXtmDYT1F4r"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        },
+        {
+          "type": "arrow",
+          "id": "he-SpFjCxEQEWpWny2kKP"
+        },
+        {
+          "type": "arrow",
+          "id": "-pjrKo16rOsasM8viZPJ-"
+        },
+        {
+          "id": "EyecK0YM9Cc8T6ju-nTOc",
+          "type": "arrow"
+        }
+      ],
+      "updated": 1655372623571,
+      "link": null,
+      "locked": false
+    },
+    {
+      "type": "text",
+      "version": 1244,
+      "versionNonce": 666803448,
+      "isDeleted": false,
+      "id": "v9qzZSsHdJ_ETRlP4Msn5",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 635.2247543334961,
+      "y": 497.55932998657227,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 48,
+      "height": 24,
+      "seed": 1105210104,
+      "groupIds": [
+        "vuLTnxw8A0DXtmDYT1F4r"
+      ],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372625794,
+      "link": null,
+      "locked": false,
+      "fontSize": 20,
+      "fontFamily": 3,
+      "text": "vm-2",
+      "baseline": 20,
+      "textAlign": "center",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "vm-2"
+    },
+    {
+      "id": "yb3B2pFN0OZOd4yLmSU2m",
+      "type": "text",
+      "x": 449.79036712646484,
+      "y": 406.616886138916,
+      "width": 442,
+      "height": 20,
+      "angle": 0,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "seed": 1374332808,
+      "version": 196,
+      "versionNonce": 526480264,
+      "isDeleted": false,
+      "boundElements": null,
+      "updated": 1655372596999,
+      "link": null,
+      "locked": false,
+      "text": "-remoteWrite.url=https://vm-1:8428/api/v1/write",
+      "fontSize": 16,
+      "fontFamily": 3,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "baseline": 16,
+      "containerId": null,
+      "originalText": "-remoteWrite.url=https://vm-1:8428/api/v1/write"
+    },
+    {
+      "type": "text",
+      "version": 242,
+      "versionNonce": 1304477832,
+      "isDeleted": false,
+      "id": "8CXNdrDePIAAwgJB2b8YT",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 450.0511703491211,
+      "y": 432.6653480529785,
+      "strokeColor": "black",
+      "backgroundColor": "transparent",
+      "width": 442,
+      "height": 20,
+      "seed": 1349606392,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [],
+      "updated": 1655372600292,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "-remoteWrite.url=https://vm-2:8428/api/v1/write",
+      "baseline": 16,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "-remoteWrite.url=https://vm-2:8428/api/v1/write"
+    },
+    {
+      "type": "text",
+      "version": 1195,
+      "versionNonce": 1912405496,
+      "isDeleted": false,
+      "id": "Ev-VujoFglVNIh5GIhsba",
+      "fillStyle": "hachure",
+      "strokeWidth": 1,
+      "strokeStyle": "solid",
+      "roughness": 0,
+      "opacity": 100,
+      "angle": 0,
+      "x": 357.2894821166992,
+      "y": 370.6587562561035,
+      "strokeColor": "#000000",
+      "backgroundColor": "transparent",
+      "width": 114,
+      "height": 20,
+      "seed": 1289300104,
+      "groupIds": [],
+      "strokeSharpness": "sharp",
+      "boundElements": [
+        {
+          "type": "arrow",
+          "id": "wRO0q9xKPHc8e8XPPsQWh"
+        }
+      ],
+      "updated": 1655372703770,
+      "link": null,
+      "locked": false,
+      "fontSize": 16,
+      "fontFamily": 3,
+      "text": "fan-out data",
+      "baseline": 16,
+      "textAlign": "left",
+      "verticalAlign": "top",
+      "containerId": null,
+      "originalText": "fan-out data"
+    }
+  ],
+  "appState": {
+    "gridSize": null,
+    "viewBackgroundColor": "#ffffff"
+  },
+  "files": {}
+}

app/vmalert/web.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"embed"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -23,6 +24,12 @@ var (
 	navItems []tpl.NavItem
 )
 
+var (
+	//go:embed static
+	staticFiles  embed.FS
+	staticServer = http.FileServer(http.FS(staticFiles))
+)
+
 func initLinks() {
 	pathPrefix := httpserver.GetPathPrefix()
 	if pathPrefix == "" {
@@ -99,6 +106,11 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	default:
+		if strings.HasPrefix(r.URL.Path, "/static") {
+			staticServer.ServeHTTP(w, r)
+			return true
+		}
+
 		if !strings.HasSuffix(r.URL.Path, "/status") {
 			return false
 		}

app/vmauth/README.md

@@ -10,7 +10,7 @@ The `-auth.config` can point to either local file or to http url.
 Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
 and pass the following flag to `vmauth` binary in order to start authorizing and routing requests:
 
-```bash
+```console
 /path/to/vmauth -auth.config=/path/to/auth/config.yml
 ```
 
@@ -129,7 +129,7 @@ It is expected that all the backend services protected by `vmauth` are located i
 
 Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable https. This can be done by passing the following `-tls*` command-line flags to `vmauth`:
 
-```bash
+```console
   -tls
      Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
@@ -140,7 +140,11 @@ Do not transfer Basic Auth headers in plaintext over untrusted networks. Enable
 
 Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termination_proxy) may be put in front of `vmauth`.
 
-It is recommended protecting `/-/reload` endpoint with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
+It is recommended protecting  following endpoints with authKeys:
+* `/-/reload` with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
+* `/flags` with `-flagsAuthkey` command-line flag, so unauthorized users couldn't get application command-line flags.
+* `/metrics` with `metricsAuthkey` command-line flag, so unauthorized users couldn't get access to [vmauth metrics](#monitoring).
+* `/debug/pprof` with `pprofAuthKey` command-line flag, so unauthorized users couldn't get access to [profiling information](#profiling).
 
 ## Monitoring
 
@@ -181,7 +185,7 @@ The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
 The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
-```bash
+```console
 ROOT_IMAGE=scratch make package-vmauth
 ```
 
@@ -193,7 +197,7 @@ ROOT_IMAGE=scratch make package-vmauth
 
 <div class="with-copy" markdown="1">
 
-```bash
+```console
 curl http://0.0.0.0:8427/debug/pprof/heap > mem.pprof
 ```
 
@@ -203,7 +207,7 @@ curl http://0.0.0.0:8427/debug/pprof/heap > mem.pprof
 
 <div class="with-copy" markdown="1">
 
-```bash
+```console
 curl http://0.0.0.0:8427/debug/pprof/profile > cpu.pprof
 ```
 
@@ -217,7 +221,7 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
 
 Pass `-help` command-line arg to `vmauth` in order to see all the configuration options:
 
-```bash
+```console
 ./vmauth -help
 
 vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.

app/vmbackup/README.md

@@ -28,7 +28,7 @@ creation of hourly, daily, weekly and monthly backups.
 
 Regular backup can be performed with the following command:
 
-```bash
+```console
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup>
 ```
 
@@ -43,7 +43,7 @@ vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=h
 If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
 with the following command:
 
-```bash
+```console
 ./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
 ```
 
@@ -54,7 +54,7 @@ It saves time and network bandwidth costs by performing server-side copy for the
 Incremental backups are performed if `-dst` points to an already existing backup. In this case only new data is uploaded to remote storage.
 It saves time and network bandwidth costs when working with big backups:
 
-```bash
+```console
 ./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/existing/backup>
 ```
 
@@ -64,7 +64,7 @@ Smart backups mean storing full daily backups into `YYYYMMDD` folders and creati
 
 * Run the following command every hour:
 
-```bash
+```console
 ./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/latest
 ```
 
@@ -73,7 +73,7 @@ The command will upload only changed data to `gs://<bucket>/latest`.
 
 * Run the following command once a day:
 
-```bash
+```console
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
 ```
 
@@ -129,7 +129,7 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
     for s3 (aws, minio or other s3 compatible storages):
 
-     ```bash
+     ```console
      [default]
      aws_access_key_id=theaccesskey
      aws_secret_access_key=thesecretaccesskeyvalue
@@ -155,7 +155,7 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 * Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
   You have to add a custom url endpoint via flag:
 
-```bash
+```console
   # for minio
   -customS3Endpoint=http://localhost:9000
 
@@ -165,7 +165,7 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 * Run `vmbackup -help` in order to see all the available options:
 
-```bash
+```console
   -concurrency int
      The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
   -configFilePath string
@@ -280,6 +280,6 @@ The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
 The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
-```bash
+```console
 ROOT_IMAGE=scratch make package-vmbackup
 ```

app/vmctl/README.md

@@ -15,7 +15,7 @@ Features:
 To see the full list of supported modes
 run the following command:
 
-```bash
+```console
 $ ./vmctl --help                                        
 NAME:
    vmctl - VictoriaMetrics command-line tool
@@ -197,7 +197,8 @@ One important note for OpenTSDB migration: Queries/HBase scans can "get stuck" w
 
 ## Migrating data from InfluxDB (1.x)
 
-`vmctl` supports the `influx` mode to migrate data from InfluxDB to VictoriaMetrics time-series database.
+`vmctl` supports the `influx` mode for [migrating data from InfluxDB to VictoriaMetrics](https://docs.victoriametrics.com/guides/migrate-from-influx.html)
+time-series database.
 
 See `./vmctl influx --help` for details and full list of flags.
 
@@ -526,7 +527,7 @@ and specify `accountID` param.
 In this mode, `vmctl` allows verifying correctness and integrity of data exported via [native format](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-export-data-in-native-format) from VictoriaMetrics. 
 You can verify exported data at disk before uploading it by `vmctl verify-block` command:
 
-```bash
+```console
 # export blocks from VictoriaMetrics
 curl localhost:8428/api/v1/export/native -g -d 'match[]={__name__!=""}' -o exported_data_block
 # verify block content
@@ -649,7 +650,7 @@ The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmctl`.
 The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
-```bash
+```console
 ROOT_IMAGE=scratch make package-vmctl
 ```
 

app/vmgateway/README.md

@@ -54,7 +54,7 @@ Where:
 
 Start the single version of VictoriaMetrics
 
-```bash
+```console
 # single
 # start node
 ./bin/victoria-metrics --selfScrapeInterval=10s
@@ -62,19 +62,19 @@ Start the single version of VictoriaMetrics
 
 Start vmgateway
 
-```bash
+```console
 ./bin/vmgateway -eula -enable.auth -read.url http://localhost:8428 --write.url http://localhost:8428
 ```
 
 Retrieve data from the database
 
-```bash
+```console
 curl 'http://localhost:8431/api/v1/series/count' -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ2bV9hY2Nlc3MiOnsidGVuYW50X2lkIjp7fSwicm9sZSI6MX0sImV4cCI6MTkzOTM0NjIxMH0.5WUxEfdcV9hKo4CtQdtuZYOGpGXWwaqM9VuVivMMrVg'
 ```
 
 A request with an incorrect token or without any token will be rejected:
 
-```bash
+```console
 curl 'http://localhost:8431/api/v1/series/count'
 
 curl 'http://localhost:8431/api/v1/series/count' -H 'Authorization: Bearer incorrect-token'
@@ -124,7 +124,7 @@ limits:
 
 cluster version of VictoriaMetrics is required for rate limiting.
 
-```bash
+```console
 # start datasource for cluster metrics
 
 cat << EOF > cluster.yaml

app/vminsert/main.go

@@ -1,6 +1,7 @@
 package vminsert
 
 import (
+	"embed"
 	"flag"
 	"fmt"
 	"net/http"
@@ -20,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
@@ -54,6 +56,11 @@ var (
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
 
+//go:embed static
+var staticFiles embed.FS
+
+var staticServer = http.FileServer(http.FS(staticFiles))
+
 // Init initializes vminsert.
 func Init() {
 	relabel.Init()
@@ -100,6 +107,20 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	defer requestDuration.UpdateDuration(startTime)
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if strings.HasPrefix(path, "/static") {
+		staticServer.ServeHTTP(w, r)
+		return true
+	}
+	if strings.HasPrefix(path, "/prometheus/static") {
+		r.URL.Path = strings.TrimPrefix(path, "/prometheus")
+		staticServer.ServeHTTP(w, r)
+		return true
+	}
+	if strings.HasPrefix(path, "/datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		path = strings.TrimSuffix(path, "/")
+	}
 	switch path {
 	case "/prometheus/api/v1/write", "/api/v1/write":
 		prometheusWriteRequests.Inc()
@@ -186,15 +207,24 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/intake/":
+	case "/datadog/intake":
 		datadogIntakeRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
+	case "/datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/prometheus/targets", "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/prometheus/service-discovery", "/service-discovery":
+		promscrapeServiceDiscoveryRequests.Inc()
+		promscrape.WriteServiceDiscovery(w, r)
+		return true
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -222,6 +252,22 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
+	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
+		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		promscrapeStatusConfigRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		var bb bytesutil.ByteBuffer
+		promscrape.WriteConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
+		return true
 	case "/prometheus/-/reload", "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
@@ -277,15 +323,18 @@ var (
 
 	datadogValidateRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
-	datadogIntakeRequests   = metrics.NewCounter(`vm_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vm_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
+	datadogMetadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	promscrapeTargetsRequests      = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
+	promscrapeTargetsRequests          = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
+	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vm_http_requests_total{path="/service-discovery"}`)
+	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vm_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/target_response"}`)
 
-	promscrapeConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/config"}`)
+	promscrapeConfigRequests       = metrics.NewCounter(`vm_http_requests_total{path="/config"}`)
+	promscrapeStatusConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/config"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
 

app/vmrestore/README.md

@@ -10,7 +10,7 @@ when restarting `vmrestore` with the same args.
 
 VictoriaMetrics must be stopped during the restore process.
 
-```bash
+```console
 vmrestore -src=gs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
 
 ```
@@ -36,7 +36,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 
     for s3 (aws, minio or other s3 compatible storages):
 
-     ```bash
+     ```console
      [default]
      aws_access_key_id=theaccesskey
      aws_secret_access_key=thesecretaccesskeyvalue
@@ -62,7 +62,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 * Usage with s3 custom url endpoint.  It is possible to use `vmrestore` with s3 api compatible storages, like  minio, cloudian and other.
   You have to add custom url endpoint with a flag:
 
-```bash
+```console
   # for minio:
   -customS3Endpoint=http://localhost:9000
 
@@ -72,7 +72,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 
 * Run `vmrestore -help` in order to see all the available options:
 
-```bash
+```console
   -concurrency int
      The number of concurrent workers. Higher concurrency may reduce restore duration (default 10)
   -configFilePath string
@@ -180,6 +180,6 @@ The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.
 The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
-```bash
+```console
 ROOT_IMAGE=scratch make package-vmrestore
 ```

app/vmselect/graphite/metrics_api.go

@@ -22,9 +22,6 @@ import (
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
 func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
 	format := r.FormValue("format")
 	if format == "" {
 		format = "treejson"
@@ -119,9 +116,6 @@ func deduplicatePaths(paths []string, delimiter string) []string {
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
 func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
 	queries := r.Form["query"]
 	if len(queries) == 0 {
 		return fmt.Errorf("missing `query` arg")
@@ -202,11 +196,9 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json
 func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
 	jsonp := r.FormValue("jsonp")
-	metricNames, err := netstorage.GetLabelValues("__name__", deadline)
+	sq := storage.NewSearchQuery(0, 0, nil, 0)
+	metricNames, err := netstorage.GetLabelValues(nil, "__name__", sq, 0, deadline)
 	if err != nil {
 		return fmt.Errorf(`cannot obtain metric names: %w`, err)
 	}
@@ -227,7 +219,7 @@ func metricsFind(tr storage.TimeRange, label, qHead, qTail string, delimiter byt
 	n := strings.IndexAny(qTail, "*{[")
 	if n < 0 {
 		query := qHead + qTail
-		suffixes, err := netstorage.GetTagValueSuffixes(tr, label, query, delimiter, deadline)
+		suffixes, err := netstorage.GetTagValueSuffixes(nil, tr, label, query, delimiter, deadline)
 		if err != nil {
 			return nil, err
 		}
@@ -247,7 +239,7 @@ func metricsFind(tr storage.TimeRange, label, qHead, qTail string, delimiter byt
 	}
 	if n == len(qTail)-1 && strings.HasSuffix(qTail, "*") {
 		query := qHead + qTail[:len(qTail)-1]
-		suffixes, err := netstorage.GetTagValueSuffixes(tr, label, query, delimiter, deadline)
+		suffixes, err := netstorage.GetTagValueSuffixes(nil, tr, label, query, delimiter, deadline)
 		if err != nil {
 			return nil, err
 		}

app/vmselect/graphite/tags_api.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"regexp"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
@@ -24,9 +23,6 @@ import (
 // See https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb
 func TagsDelSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
 	paths := r.Form["path"]
 	totalDeleted := 0
 	var row graphiteparser.Row
@@ -55,7 +51,7 @@ func TagsDelSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 		}
 		tfss := joinTagFilterss(tfs, etfs)
 		sq := storage.NewSearchQuery(0, ct, tfss, 0)
-		n, err := netstorage.DeleteSeries(sq, deadline)
+		n, err := netstorage.DeleteSeries(nil, sq, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot delete series for %q: %w", sq, err)
 		}
@@ -86,9 +82,8 @@ func TagsTagMultiSeriesHandler(startTime time.Time, w http.ResponseWriter, r *ht
 }
 
 func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request, isJSONResponse bool) error {
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	_ = deadline // TODO: use the deadline as in the cluster branch
 	paths := r.Form["path"]
 	var row graphiteparser.Row
 	var labels []prompb.Label
@@ -163,10 +158,7 @@ var (
 // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
 func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
-	limit, err := getInt(r, "limit")
+	limit, err := searchutils.GetInt(r, "limit")
 	if err != nil {
 		return err
 	}
@@ -190,7 +182,7 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r
 		// Escape special chars in tagPrefix as Graphite does.
 		// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228
 		filter := regexp.QuoteMeta(valuePrefix)
-		tagValues, err = netstorage.GetGraphiteTagValues(tag, filter, limit, deadline)
+		tagValues, err = netstorage.GetGraphiteTagValues(nil, tag, filter, limit, deadline)
 		if err != nil {
 			return err
 		}
@@ -200,7 +192,7 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r
 		if err != nil {
 			return err
 		}
-		mns, err := netstorage.SearchMetricNames(sq, deadline)
+		mns, err := netstorage.SearchMetricNames(nil, sq, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
 		}
@@ -252,10 +244,7 @@ var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_sec
 // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
 func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
-	limit, err := getInt(r, "limit")
+	limit, err := searchutils.GetInt(r, "limit")
 	if err != nil {
 		return err
 	}
@@ -276,7 +265,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *
 		// Escape special chars in tagPrefix as Graphite does.
 		// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181
 		filter := regexp.QuoteMeta(tagPrefix)
-		labels, err = netstorage.GetGraphiteTags(filter, limit, deadline)
+		labels, err = netstorage.GetGraphiteTags(nil, filter, limit, deadline)
 		if err != nil {
 			return err
 		}
@@ -286,7 +275,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *
 		if err != nil {
 			return err
 		}
-		mns, err := netstorage.SearchMetricNames(sq, deadline)
+		mns, err := netstorage.SearchMetricNames(nil, sq, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
 		}
@@ -334,10 +323,7 @@ var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_secon
 // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
 func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
-	limit, err := getInt(r, "limit")
+	limit, err := searchutils.GetInt(r, "limit")
 	if err != nil {
 		return err
 	}
@@ -353,7 +339,7 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R
 	if err != nil {
 		return err
 	}
-	mns, err := netstorage.SearchMetricNames(sq, deadline)
+	mns, err := netstorage.SearchMetricNames(nil, sq, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
 	}
@@ -405,15 +391,12 @@ var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{pat
 // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
 func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
-	limit, err := getInt(r, "limit")
+	limit, err := searchutils.GetInt(r, "limit")
 	if err != nil {
 		return err
 	}
 	filter := r.FormValue("filter")
-	tagValues, err := netstorage.GetGraphiteTagValues(tagName, filter, limit, deadline)
+	tagValues, err := netstorage.GetGraphiteTagValues(nil, tagName, filter, limit, deadline)
 	if err != nil {
 		return err
 	}
@@ -436,15 +419,12 @@ var tagValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/t
 // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
 func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
-	if err := r.ParseForm(); err != nil {
-		return fmt.Errorf("cannot parse form values: %w", err)
-	}
-	limit, err := getInt(r, "limit")
+	limit, err := searchutils.GetInt(r, "limit")
 	if err != nil {
 		return err
 	}
 	filter := r.FormValue("filter")
-	labels, err := netstorage.GetGraphiteTags(filter, limit, deadline)
+	labels, err := netstorage.GetGraphiteTags(nil, filter, limit, deadline)
 	if err != nil {
 		return err
 	}
@@ -462,18 +442,6 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er
 
 var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`)
 
-func getInt(r *http.Request, argName string) (int, error) {
-	argValue := r.FormValue(argName)
-	if len(argValue) == 0 {
-		return 0, nil
-	}
-	n, err := strconv.Atoi(argValue)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse %q=%q: %w", argName, argValue, err)
-	}
-	return n, nil
-}
-
 func getSearchQueryForExprs(startTime time.Time, etfs [][]storage.TagFilter, exprs []string, maxMetrics int) (*storage.SearchQuery, error) {
 	tfs, err := exprsToTagFilters(exprs)
 	if err != nil {

app/vmselect/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -91,6 +92,8 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	startTime := time.Now()
 	defer requestDuration.UpdateDuration(startTime)
+	tracerEnabled := searchutils.GetBool(r, "trace")
+	qt := querytracer.New(tracerEnabled, r.URL.Path)
 
 	// Limit the number of concurrent queries.
 	select {
@@ -106,6 +109,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		t := timerpool.Get(d)
 		select {
 		case concurrencyCh <- struct{}{}:
+			qt.Printf("wait in queue because -search.maxConcurrentRequests=%d concurrent requests are executed", *maxConcurrentRequests)
 			timerpool.Put(t)
 			defer func() { <-concurrencyCh }()
 		case <-t.C:
@@ -177,14 +181,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
 	}
-
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := path[len("/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {
 			labelValuesRequests.Inc()
 			labelName := s[:len(s)-len("/values")]
 			httpserver.EnableCORS(w, r)
-			if err := prometheus.LabelValuesHandler(startTime, labelName, w, r); err != nil {
+			if err := prometheus.LabelValuesHandler(qt, startTime, labelName, w, r); err != nil {
 				labelValuesErrors.Inc()
 				sendPrometheusError(w, r, err)
 				return true
@@ -212,7 +215,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/query":
 		queryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.QueryHandler(startTime, w, r); err != nil {
+		if err := prometheus.QueryHandler(qt, startTime, w, r); err != nil {
 			queryErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -221,7 +224,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/query_range":
 		queryRangeRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.QueryRangeHandler(startTime, w, r); err != nil {
+		if err := prometheus.QueryRangeHandler(qt, startTime, w, r); err != nil {
 			queryRangeErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -230,7 +233,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/series":
 		seriesRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.SeriesHandler(startTime, w, r); err != nil {
+		if err := prometheus.SeriesHandler(qt, startTime, w, r); err != nil {
 			seriesErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -248,24 +251,16 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/labels":
 		labelsRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.LabelsHandler(startTime, w, r); err != nil {
+		if err := prometheus.LabelsHandler(qt, startTime, w, r); err != nil {
 			labelsErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
 		}
 		return true
-	case "/api/v1/labels/count":
-		labelsCountRequests.Inc()
-		httpserver.EnableCORS(w, r)
-		if err := prometheus.LabelsCountHandler(startTime, w, r); err != nil {
-			labelsCountErrors.Inc()
-			sendPrometheusError(w, r, err)
-			return true
-		}
-		return true
 	case "/api/v1/status/tsdb":
 		statusTSDBRequests.Inc()
-		if err := prometheus.TSDBStatusHandler(startTime, w, r); err != nil {
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.TSDBStatusHandler(qt, startTime, w, r); err != nil {
 			statusTSDBErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -277,6 +272,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/status/top_queries":
 		topQueriesRequests.Inc()
+		httpserver.EnableCORS(w, r)
 		if err := prometheus.QueryStatsHandler(startTime, w, r); err != nil {
 			topQueriesErrors.Inc()
 			sendPrometheusError(w, r, fmt.Errorf("cannot query status endpoint: %w", err))
@@ -497,9 +493,6 @@ var (
 	labelsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/labels"}`)
 	labelsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/labels"}`)
 
-	labelsCountRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/labels/count"}`)
-	labelsCountErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/labels/count"}`)
-
 	statusTSDBRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/tsdb"}`)
 	statusTSDBErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/status/tsdb"}`)
 

app/vmselect/netstorage/netstorage.go

@@ -17,6 +17,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastrand"
@@ -192,7 +193,8 @@ var resultPool sync.Pool
 // Data processing is immediately stopped if f returns non-nil error.
 //
 // rss becomes unusable after the call to RunParallel.
-func (rss *Results) RunParallel(f func(rs *Result, workerID uint) error) error {
+func (rss *Results) RunParallel(qt *querytracer.Tracer, f func(rs *Result, workerID uint) error) error {
+	qt = qt.NewChild("parallel process of fetched data")
 	defer rss.mustClose()
 
 	// Spin up local workers.
@@ -254,6 +256,7 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint) error) error {
 		close(workCh)
 	}
 	workChsWG.Wait()
+	qt.Donef("series=%d, samples=%d", seriesProcessedTotal, rowsProcessedTotal)
 
 	return firstErr
 }
@@ -594,7 +597,9 @@ func (sbh *sortBlocksHeap) Pop() interface{} {
 }
 
 // DeleteSeries deletes time series matching the given tagFilterss.
-func DeleteSeries(sq *storage.SearchQuery, deadline searchutils.Deadline) (int, error) {
+func DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline searchutils.Deadline) (int, error) {
+	qt = qt.NewChild("delete series: %s", sq)
+	defer qt.Done()
 	tr := storage.TimeRange{
 		MinTimestamp: sq.MinTimestamp,
 		MaxTimestamp: sq.MaxTimestamp,
@@ -606,32 +611,43 @@ func DeleteSeries(sq *storage.SearchQuery, deadline searchutils.Deadline) (int,
 	return vmstorage.DeleteMetrics(tfss)
 }
 
-// GetLabelsOnTimeRange returns labels for the given tr until the given deadline.
-func GetLabelsOnTimeRange(tr storage.TimeRange, deadline searchutils.Deadline) ([]string, error) {
+// GetLabelNames returns label names matching the given sq until the given deadline.
+func GetLabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline searchutils.Deadline) ([]string, error) {
+	qt = qt.NewChild("get labels: %s", sq)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	labels, err := vmstorage.SearchTagKeysOnTimeRange(tr, *maxTagKeysPerSearch, deadline.Deadline())
+	if maxLabelNames > *maxTagKeysPerSearch || maxLabelNames <= 0 {
+		maxLabelNames = *maxTagKeysPerSearch
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: sq.MinTimestamp,
+		MaxTimestamp: sq.MaxTimestamp,
+	}
+	tfss, err := setupTfss(tr, sq.TagFilterss, sq.MaxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	labels, err := vmstorage.SearchLabelNamesWithFiltersOnTimeRange(qt, tfss, tr, maxLabelNames, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during labels search on time range: %w", err)
 	}
-	// Substitute "" with "__name__"
-	for i := range labels {
-		if labels[i] == "" {
-			labels[i] = "__name__"
-		}
-	}
 	// Sort labels like Prometheus does
 	sort.Strings(labels)
+	qt.Printf("sort %d labels", len(labels))
 	return labels, nil
 }
 
 // GetGraphiteTags returns Graphite tags until the given deadline.
-func GetGraphiteTags(filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
+func GetGraphiteTags(qt *querytracer.Tracer, filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
+	qt = qt.NewChild("get graphite tags: filter=%s, limit=%d", filter, limit)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	labels, err := GetLabels(deadline)
+	sq := storage.NewSearchQuery(0, 0, nil, 0)
+	labels, err := GetLabelNames(qt, sq, 0, deadline)
 	if err != nil {
 		return nil, err
 	}
@@ -671,54 +687,46 @@ func hasString(a []string, s string) bool {
 	return false
 }
 
-// GetLabels returns labels until the given deadline.
-func GetLabels(deadline searchutils.Deadline) ([]string, error) {
+// GetLabelValues returns label values matching the given labelName and sq until the given deadline.
+func GetLabelValues(qt *querytracer.Tracer, labelName string, sq *storage.SearchQuery, maxLabelValues int, deadline searchutils.Deadline) ([]string, error) {
+	qt = qt.NewChild("get values for label %s: %s", labelName, sq)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
-	labels, err := vmstorage.SearchTagKeys(*maxTagKeysPerSearch, deadline.Deadline())
+	if maxLabelValues > *maxTagValuesPerSearch || maxLabelValues <= 0 {
+		maxLabelValues = *maxTagValuesPerSearch
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: sq.MinTimestamp,
+		MaxTimestamp: sq.MaxTimestamp,
+	}
+	tfss, err := setupTfss(tr, sq.TagFilterss, sq.MaxMetrics, deadline)
 	if err != nil {
-		return nil, fmt.Errorf("error during labels search: %w", err)
+		return nil, err
 	}
-	// Substitute "" with "__name__"
-	for i := range labels {
-		if labels[i] == "" {
-			labels[i] = "__name__"
-		}
-	}
-	// Sort labels like Prometheus does
-	sort.Strings(labels)
-	return labels, nil
-}
-
-// GetLabelValuesOnTimeRange returns label values for the given labelName on the given tr
-// until the given deadline.
-func GetLabelValuesOnTimeRange(labelName string, tr storage.TimeRange, deadline searchutils.Deadline) ([]string, error) {
-	if deadline.Exceeded() {
-		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
-	}
-	if labelName == "__name__" {
-		labelName = ""
-	}
-	// Search for tag values
-	labelValues, err := vmstorage.SearchTagValuesOnTimeRange([]byte(labelName), tr, *maxTagValuesPerSearch, deadline.Deadline())
+	labelValues, err := vmstorage.SearchLabelValuesWithFiltersOnTimeRange(qt, labelName, tfss, tr, maxLabelValues, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during label values search on time range for labelName=%q: %w", labelName, err)
 	}
 	// Sort labelValues like Prometheus does
 	sort.Strings(labelValues)
+	qt.Printf("sort %d label values", len(labelValues))
 	return labelValues, nil
 }
 
 // GetGraphiteTagValues returns tag values for the given tagName until the given deadline.
-func GetGraphiteTagValues(tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
+func GetGraphiteTagValues(qt *querytracer.Tracer, tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
+	qt = qt.NewChild("get graphite tag values for tagName=%s, filter=%s, limit=%d", tagName, filter, limit)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
 	if tagName == "name" {
 		tagName = ""
 	}
-	tagValues, err := GetLabelValues(tagName, deadline)
+	sq := storage.NewSearchQuery(0, 0, nil, 0)
+	tagValues, err := GetLabelValues(qt, tagName, sq, 0, deadline)
 	if err != nil {
 		return nil, err
 	}
@@ -734,29 +742,12 @@ func GetGraphiteTagValues(tagName, filter string, limit int, deadline searchutil
 	return tagValues, nil
 }
 
-// GetLabelValues returns label values for the given labelName
-// until the given deadline.
-func GetLabelValues(labelName string, deadline searchutils.Deadline) ([]string, error) {
-	if deadline.Exceeded() {
-		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
-	}
-	if labelName == "__name__" {
-		labelName = ""
-	}
-	// Search for tag values
-	labelValues, err := vmstorage.SearchTagValues([]byte(labelName), *maxTagValuesPerSearch, deadline.Deadline())
-	if err != nil {
-		return nil, fmt.Errorf("error during label values search for labelName=%q: %w", labelName, err)
-	}
-	// Sort labelValues like Prometheus does
-	sort.Strings(labelValues)
-	return labelValues, nil
-}
-
 // GetTagValueSuffixes returns tag value suffixes for the given tagKey and the given tagValuePrefix.
 //
 // It can be used for implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
-func GetTagValueSuffixes(tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, deadline searchutils.Deadline) ([]string, error) {
+func GetTagValueSuffixes(qt *querytracer.Tracer, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, deadline searchutils.Deadline) ([]string, error) {
+	qt = qt.NewChild("get tag value suffixes for tagKey=%s, tagValuePrefix=%s, timeRange=%s", tagKey, tagValuePrefix, &tr)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
@@ -773,52 +764,12 @@ func GetTagValueSuffixes(tr storage.TimeRange, tagKey, tagValuePrefix string, de
 	return suffixes, nil
 }
 
-// GetLabelEntries returns all the label entries until the given deadline.
-func GetLabelEntries(deadline searchutils.Deadline) ([]storage.TagEntry, error) {
-	if deadline.Exceeded() {
-		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
-	}
-	labelEntries, err := vmstorage.SearchTagEntries(*maxTagKeysPerSearch, *maxTagValuesPerSearch, deadline.Deadline())
-	if err != nil {
-		return nil, fmt.Errorf("error during label entries request: %w", err)
-	}
-
-	// Substitute "" with "__name__"
-	for i := range labelEntries {
-		e := &labelEntries[i]
-		if e.Key == "" {
-			e.Key = "__name__"
-		}
-	}
-
-	// Sort labelEntries by the number of label values in each entry.
-	sort.Slice(labelEntries, func(i, j int) bool {
-		a, b := labelEntries[i].Values, labelEntries[j].Values
-		if len(a) != len(b) {
-			return len(a) > len(b)
-		}
-		return labelEntries[i].Key > labelEntries[j].Key
-	})
-
-	return labelEntries, nil
-}
-
-// GetTSDBStatusForDate returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
-func GetTSDBStatusForDate(deadline searchutils.Deadline, date uint64, topN, maxMetrics int) (*storage.TSDBStatus, error) {
-	if deadline.Exceeded() {
-		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
-	}
-	status, err := vmstorage.GetTSDBStatusForDate(date, topN, maxMetrics, deadline.Deadline())
-	if err != nil {
-		return nil, fmt.Errorf("error during tsdb status request: %w", err)
-	}
-	return status, nil
-}
-
-// GetTSDBStatusWithFilters returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
+// GetTSDBStatus returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
 //
 // It accepts aribtrary filters on time series in sq.
-func GetTSDBStatusWithFilters(deadline searchutils.Deadline, sq *storage.SearchQuery, topN int) (*storage.TSDBStatus, error) {
+func GetTSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
+	qt = qt.NewChild("get tsdb stats: %s, focusLabel=%q, topN=%d", sq, focusLabel, topN)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
@@ -831,15 +782,17 @@ func GetTSDBStatusWithFilters(deadline searchutils.Deadline, sq *storage.SearchQ
 		return nil, err
 	}
 	date := uint64(tr.MinTimestamp) / (3600 * 24 * 1000)
-	status, err := vmstorage.GetTSDBStatusWithFiltersForDate(tfss, date, topN, sq.MaxMetrics, deadline.Deadline())
+	status, err := vmstorage.GetTSDBStatus(qt, tfss, date, focusLabel, topN, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
-		return nil, fmt.Errorf("error during tsdb status with filters request: %w", err)
+		return nil, fmt.Errorf("error during tsdb status request: %w", err)
 	}
 	return status, nil
 }
 
 // GetSeriesCount returns the number of unique series.
-func GetSeriesCount(deadline searchutils.Deadline) (uint64, error) {
+func GetSeriesCount(qt *querytracer.Tracer, deadline searchutils.Deadline) (uint64, error) {
+	qt = qt.NewChild("get series count")
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return 0, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
@@ -871,7 +824,9 @@ var ssPool sync.Pool
 // Data processing is immediately stopped if f returns non-nil error.
 // It is the responsibility of f to call b.UnmarshalData before reading timestamps and values from the block.
 // It is the responsibility of f to filter blocks according to the given tr.
-func ExportBlocks(sq *storage.SearchQuery, deadline searchutils.Deadline, f func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error) error {
+func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline searchutils.Deadline, f func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error) error {
+	qt = qt.NewChild("export blocks: %s", sq)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
 	}
@@ -893,7 +848,7 @@ func ExportBlocks(sq *storage.SearchQuery, deadline searchutils.Deadline, f func
 	sr := getStorageSearch()
 	defer putStorageSearch(sr)
 	startTime := time.Now()
-	sr.Init(vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
+	sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	indexSearchDuration.UpdateDuration(startTime)
 
 	// Start workers that call f in parallel on available CPU cores.
@@ -926,6 +881,7 @@ func ExportBlocks(sq *storage.SearchQuery, deadline searchutils.Deadline, f func
 
 	// Feed workers with work
 	blocksRead := 0
+	samples := 0
 	for sr.NextMetricBlock() {
 		blocksRead++
 		if deadline.Exceeded() {
@@ -938,13 +894,16 @@ func ExportBlocks(sq *storage.SearchQuery, deadline searchutils.Deadline, f func
 		if err := xw.mn.Unmarshal(sr.MetricBlockRef.MetricName); err != nil {
 			return fmt.Errorf("cannot unmarshal metricName for block #%d: %w", blocksRead, err)
 		}
-		sr.MetricBlockRef.BlockRef.MustReadBlock(&xw.b, true)
+		br := sr.MetricBlockRef.BlockRef
+		br.MustReadBlock(&xw.b, true)
+		samples += br.RowsCount()
 		workCh <- xw
 	}
 	close(workCh)
 
 	// Wait for workers to finish.
 	wg.Wait()
+	qt.Printf("export blocks=%d, samples=%d", blocksRead, samples)
 
 	// Check errors.
 	err = sr.Error()
@@ -977,7 +936,9 @@ var exportWorkPool = &sync.Pool{
 }
 
 // SearchMetricNames returns all the metric names matching sq until the given deadline.
-func SearchMetricNames(sq *storage.SearchQuery, deadline searchutils.Deadline) ([]storage.MetricName, error) {
+func SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline searchutils.Deadline) ([]storage.MetricName, error) {
+	qt = qt.NewChild("fetch metric names: %s", sq)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting to search metric names: %s", deadline.String())
 	}
@@ -995,7 +956,7 @@ func SearchMetricNames(sq *storage.SearchQuery, deadline searchutils.Deadline) (
 		return nil, err
 	}
 
-	mns, err := vmstorage.SearchMetricNames(tfss, tr, sq.MaxMetrics, deadline.Deadline())
+	mns, err := vmstorage.SearchMetricNames(qt, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("cannot find metric names: %w", err)
 	}
@@ -1005,7 +966,9 @@ func SearchMetricNames(sq *storage.SearchQuery, deadline searchutils.Deadline) (
 // ProcessSearchQuery performs sq until the given deadline.
 //
 // Results.RunParallel or Results.Cancel must be called on the returned Results.
-func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline searchutils.Deadline) (*Results, error) {
+func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, fetchData bool, deadline searchutils.Deadline) (*Results, error) {
+	qt = qt.NewChild("fetch matching series: %s, fetchData=%v", sq, fetchData)
+	defer qt.Done()
 	if deadline.Exceeded() {
 		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
 	}
@@ -1028,7 +991,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 
 	sr := getStorageSearch()
 	startTime := time.Now()
-	maxSeriesCount := sr.Init(vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
+	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	indexSearchDuration.UpdateDuration(startTime)
 	m := make(map[string][]blockRef, maxSeriesCount)
 	orderedMetricNames := make([]string, 0, maxSeriesCount)
@@ -1085,6 +1048,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 		putStorageSearch(sr)
 		return nil, fmt.Errorf("cannot finalize temporary file: %w", err)
 	}
+	qt.Printf("fetch unique series=%d, blocks=%d, samples=%d, bytes=%d", len(m), blocksRead, samples, tbf.Len())
 
 	var rss Results
 	rss.tr = tr

app/vmselect/netstorage/tmp_blocks_file.go

@@ -124,6 +124,11 @@ func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	return addr, nil
 }
 
+// Len() returnt tbf size in bytes.
+func (tbf *tmpBlocksFile) Len() uint64 {
+	return tbf.offset
+}
+
 func (tbf *tmpBlocksFile) Finalize() error {
 	if tbf.f == nil {
 		return nil

app/vmselect/prometheus/export.qtpl

@@ -5,6 +5,7 @@
 	"time"
 
 	"github.com/valyala/quicktemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 ) %}
 
@@ -125,8 +126,12 @@
 }
 {% endfunc %}
 
-{% func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) %}
+{% func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) %}
 {
+	{% code
+		lines := 0
+		bytesTotal := 0
+	%}
 	"status":"success",
 	"data":{
 		"resultType":"matrix",
@@ -134,18 +139,30 @@
 			{% code bb, ok := <-resultsCh %}
 			{% if ok %}
 				{%z= bb.B %}
-				{% code quicktemplate.ReleaseByteBuffer(bb) %}
+				{% code
+					lines++
+					bytesTotal += len(bb.B)
+					quicktemplate.ReleaseByteBuffer(bb)
+				%}
 				{% for bb := range resultsCh %}
 					,{%z= bb.B %}
-					{% code quicktemplate.ReleaseByteBuffer(bb) %}
+					{% code
+						lines++
+						bytesTotal += len(bb.B)
+						quicktemplate.ReleaseByteBuffer(bb)
+					%}
 				{% endfor %}
 			{% endif %}
 		]
 	}
+	{% code
+		qt.Donef("export format=promapi: lines=%d, bytes=%d", lines, bytesTotal)
+	%}
+	{%= dumpQueryTrace(qt) %}
 }
 {% endfunc %}
 
-{% func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) %}
+{% func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) %}
 	{% for bb := range resultsCh %}
 		{%z= bb.B %}
 		{% code quicktemplate.ReleaseByteBuffer(bb) %}

app/vmselect/prometheus/export.qtpl.go

@@ -11,570 +11,588 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/export.qtpl:13
+//line app/vmselect/prometheus/export.qtpl:14
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/export.qtpl:13
+//line app/vmselect/prometheus/export.qtpl:14
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/export.qtpl:13
+//line app/vmselect/prometheus/export.qtpl:14
 func StreamExportCSVLine(qw422016 *qt422016.Writer, xb *exportBlock, fieldNames []string) {
-//line app/vmselect/prometheus/export.qtpl:14
-	if len(xb.timestamps) == 0 || len(fieldNames) == 0 {
-//line app/vmselect/prometheus/export.qtpl:14
-		return
-//line app/vmselect/prometheus/export.qtpl:14
-	}
 //line app/vmselect/prometheus/export.qtpl:15
-	for i, timestamp := range xb.timestamps {
+	if len(xb.timestamps) == 0 || len(fieldNames) == 0 {
+//line app/vmselect/prometheus/export.qtpl:15
+		return
+//line app/vmselect/prometheus/export.qtpl:15
+	}
 //line app/vmselect/prometheus/export.qtpl:16
+	for i, timestamp := range xb.timestamps {
+//line app/vmselect/prometheus/export.qtpl:17
 		value := xb.values[i]
 
-//line app/vmselect/prometheus/export.qtpl:17
+//line app/vmselect/prometheus/export.qtpl:18
 		streamexportCSVField(qw422016, xb.mn, fieldNames[0], timestamp, value)
-//line app/vmselect/prometheus/export.qtpl:18
+//line app/vmselect/prometheus/export.qtpl:19
 		for _, fieldName := range fieldNames[1:] {
-//line app/vmselect/prometheus/export.qtpl:18
+//line app/vmselect/prometheus/export.qtpl:19
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/export.qtpl:20
-			streamexportCSVField(qw422016, xb.mn, fieldName, timestamp, value)
 //line app/vmselect/prometheus/export.qtpl:21
-		}
+			streamexportCSVField(qw422016, xb.mn, fieldName, timestamp, value)
 //line app/vmselect/prometheus/export.qtpl:22
+		}
+//line app/vmselect/prometheus/export.qtpl:23
 		qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/export.qtpl:23
-	}
 //line app/vmselect/prometheus/export.qtpl:24
+	}
+//line app/vmselect/prometheus/export.qtpl:25
 }
 
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 func WriteExportCSVLine(qq422016 qtio422016.Writer, xb *exportBlock, fieldNames []string) {
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	StreamExportCSVLine(qw422016, xb, fieldNames)
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 }
 
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 func ExportCSVLine(xb *exportBlock, fieldNames []string) string {
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	WriteExportCSVLine(qb422016, xb, fieldNames)
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:24
+//line app/vmselect/prometheus/export.qtpl:25
 }
 
-//line app/vmselect/prometheus/export.qtpl:26
-func streamexportCSVField(qw422016 *qt422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
 //line app/vmselect/prometheus/export.qtpl:27
-	if fieldName == "__value__" {
+func streamexportCSVField(qw422016 *qt422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
 //line app/vmselect/prometheus/export.qtpl:28
-		qw422016.N().F(value)
+	if fieldName == "__value__" {
 //line app/vmselect/prometheus/export.qtpl:29
-		return
+		qw422016.N().F(value)
 //line app/vmselect/prometheus/export.qtpl:30
-	}
-//line app/vmselect/prometheus/export.qtpl:31
-	if fieldName == "__timestamp__" {
-//line app/vmselect/prometheus/export.qtpl:32
-		qw422016.N().DL(timestamp)
-//line app/vmselect/prometheus/export.qtpl:33
 		return
-//line app/vmselect/prometheus/export.qtpl:34
+//line app/vmselect/prometheus/export.qtpl:31
 	}
+//line app/vmselect/prometheus/export.qtpl:32
+	if fieldName == "__timestamp__" {
+//line app/vmselect/prometheus/export.qtpl:33
+		qw422016.N().DL(timestamp)
+//line app/vmselect/prometheus/export.qtpl:34
+		return
 //line app/vmselect/prometheus/export.qtpl:35
-	if strings.HasPrefix(fieldName, "__timestamp__:") {
+	}
 //line app/vmselect/prometheus/export.qtpl:36
+	if strings.HasPrefix(fieldName, "__timestamp__:") {
+//line app/vmselect/prometheus/export.qtpl:37
 		timeFormat := fieldName[len("__timestamp__:"):]
 
-//line app/vmselect/prometheus/export.qtpl:37
-		switch timeFormat {
 //line app/vmselect/prometheus/export.qtpl:38
-		case "unix_s":
+		switch timeFormat {
 //line app/vmselect/prometheus/export.qtpl:39
-			qw422016.N().DL(timestamp / 1000)
+		case "unix_s":
 //line app/vmselect/prometheus/export.qtpl:40
-		case "unix_ms":
+			qw422016.N().DL(timestamp / 1000)
 //line app/vmselect/prometheus/export.qtpl:41
-			qw422016.N().DL(timestamp)
+		case "unix_ms":
 //line app/vmselect/prometheus/export.qtpl:42
-		case "unix_ns":
+			qw422016.N().DL(timestamp)
 //line app/vmselect/prometheus/export.qtpl:43
-			qw422016.N().DL(timestamp * 1e6)
+		case "unix_ns":
 //line app/vmselect/prometheus/export.qtpl:44
+			qw422016.N().DL(timestamp * 1e6)
+//line app/vmselect/prometheus/export.qtpl:45
 		case "rfc3339":
-//line app/vmselect/prometheus/export.qtpl:46
+//line app/vmselect/prometheus/export.qtpl:47
 			bb := quicktemplate.AcquireByteBuffer()
 			bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], time.RFC3339)
 
-//line app/vmselect/prometheus/export.qtpl:49
+//line app/vmselect/prometheus/export.qtpl:50
 			qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:51
+//line app/vmselect/prometheus/export.qtpl:52
 			quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:53
-		default:
 //line app/vmselect/prometheus/export.qtpl:54
+		default:
+//line app/vmselect/prometheus/export.qtpl:55
 			if strings.HasPrefix(timeFormat, "custom:") {
-//line app/vmselect/prometheus/export.qtpl:56
+//line app/vmselect/prometheus/export.qtpl:57
 				layout := timeFormat[len("custom:"):]
 				bb := quicktemplate.AcquireByteBuffer()
 				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], layout)
 
-//line app/vmselect/prometheus/export.qtpl:60
-				if bytes.ContainsAny(bb.B, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:61
-					qw422016.E().QZ(bb.B)
+				if bytes.ContainsAny(bb.B, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:62
-				} else {
+					qw422016.E().QZ(bb.B)
 //line app/vmselect/prometheus/export.qtpl:63
-					qw422016.N().Z(bb.B)
+				} else {
 //line app/vmselect/prometheus/export.qtpl:64
+					qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:65
 				}
-//line app/vmselect/prometheus/export.qtpl:66
+//line app/vmselect/prometheus/export.qtpl:67
 				quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:68
-			} else {
-//line app/vmselect/prometheus/export.qtpl:68
-				qw422016.N().S(`Unsupported timeFormat=`)
 //line app/vmselect/prometheus/export.qtpl:69
-				qw422016.N().S(timeFormat)
+			} else {
+//line app/vmselect/prometheus/export.qtpl:69
+				qw422016.N().S(`Unsupported timeFormat=`)
 //line app/vmselect/prometheus/export.qtpl:70
-			}
+				qw422016.N().S(timeFormat)
 //line app/vmselect/prometheus/export.qtpl:71
-		}
+			}
 //line app/vmselect/prometheus/export.qtpl:72
-		return
+		}
 //line app/vmselect/prometheus/export.qtpl:73
-	}
+		return
 //line app/vmselect/prometheus/export.qtpl:74
+	}
+//line app/vmselect/prometheus/export.qtpl:75
 	v := mn.GetTagValue(fieldName)
 
-//line app/vmselect/prometheus/export.qtpl:75
-	if bytes.ContainsAny(v, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:76
-		qw422016.N().QZ(v)
+	if bytes.ContainsAny(v, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:77
-	} else {
+		qw422016.N().QZ(v)
 //line app/vmselect/prometheus/export.qtpl:78
-		qw422016.N().Z(v)
+	} else {
 //line app/vmselect/prometheus/export.qtpl:79
-	}
+		qw422016.N().Z(v)
 //line app/vmselect/prometheus/export.qtpl:80
+	}
+//line app/vmselect/prometheus/export.qtpl:81
 }
 
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 func writeexportCSVField(qq422016 qtio422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	streamexportCSVField(qw422016, mn, fieldName, timestamp, value)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 }
 
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 func exportCSVField(mn *storage.MetricName, fieldName string, timestamp int64, value float64) string {
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	writeexportCSVField(qb422016, mn, fieldName, timestamp, value)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:81
 }
 
-//line app/vmselect/prometheus/export.qtpl:82
+//line app/vmselect/prometheus/export.qtpl:83
 func StreamExportPrometheusLine(qw422016 *qt422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:83
-	if len(xb.timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:83
-		return
-//line app/vmselect/prometheus/export.qtpl:83
-	}
 //line app/vmselect/prometheus/export.qtpl:84
+	if len(xb.timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:84
+		return
+//line app/vmselect/prometheus/export.qtpl:84
+	}
+//line app/vmselect/prometheus/export.qtpl:85
 	bb := quicktemplate.AcquireByteBuffer()
 
-//line app/vmselect/prometheus/export.qtpl:85
+//line app/vmselect/prometheus/export.qtpl:86
 	writeprometheusMetricName(bb, xb.mn)
 
-//line app/vmselect/prometheus/export.qtpl:86
+//line app/vmselect/prometheus/export.qtpl:87
 	for i, ts := range xb.timestamps {
-//line app/vmselect/prometheus/export.qtpl:87
+//line app/vmselect/prometheus/export.qtpl:88
 		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:87
-		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:88
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/export.qtpl:89
 		qw422016.N().F(xb.values[i])
-//line app/vmselect/prometheus/export.qtpl:88
+//line app/vmselect/prometheus/export.qtpl:89
 		qw422016.N().S(` `)
-//line app/vmselect/prometheus/export.qtpl:89
+//line app/vmselect/prometheus/export.qtpl:90
 		qw422016.N().DL(ts)
-//line app/vmselect/prometheus/export.qtpl:89
+//line app/vmselect/prometheus/export.qtpl:90
 		qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/export.qtpl:90
-	}
 //line app/vmselect/prometheus/export.qtpl:91
+	}
+//line app/vmselect/prometheus/export.qtpl:92
 	quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 }
 
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 func WriteExportPrometheusLine(qq422016 qtio422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	StreamExportPrometheusLine(qw422016, xb)
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 }
 
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 func ExportPrometheusLine(xb *exportBlock) string {
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	WriteExportPrometheusLine(qb422016, xb)
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:92
+//line app/vmselect/prometheus/export.qtpl:93
 }
 
-//line app/vmselect/prometheus/export.qtpl:94
+//line app/vmselect/prometheus/export.qtpl:95
 func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:96
 	if len(xb.timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:96
 		return
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:96
 	}
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:96
 	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:97
+//line app/vmselect/prometheus/export.qtpl:98
 	streammetricNameObject(qw422016, xb.mn)
-//line app/vmselect/prometheus/export.qtpl:97
+//line app/vmselect/prometheus/export.qtpl:98
 	qw422016.N().S(`,"values":[`)
-//line app/vmselect/prometheus/export.qtpl:99
-	if len(xb.values) > 0 {
 //line app/vmselect/prometheus/export.qtpl:100
+	if len(xb.values) > 0 {
+//line app/vmselect/prometheus/export.qtpl:101
 		values := xb.values
 
-//line app/vmselect/prometheus/export.qtpl:101
-		qw422016.N().F(values[0])
 //line app/vmselect/prometheus/export.qtpl:102
+		qw422016.N().F(values[0])
+//line app/vmselect/prometheus/export.qtpl:103
 		values = values[1:]
 
-//line app/vmselect/prometheus/export.qtpl:103
+//line app/vmselect/prometheus/export.qtpl:104
 		for _, v := range values {
-//line app/vmselect/prometheus/export.qtpl:103
+//line app/vmselect/prometheus/export.qtpl:104
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/export.qtpl:104
-			if math.IsNaN(v) {
-//line app/vmselect/prometheus/export.qtpl:104
-				qw422016.N().S(`null`)
-//line app/vmselect/prometheus/export.qtpl:104
-			} else {
-//line app/vmselect/prometheus/export.qtpl:104
-				qw422016.N().F(v)
-//line app/vmselect/prometheus/export.qtpl:104
-			}
 //line app/vmselect/prometheus/export.qtpl:105
+			if math.IsNaN(v) {
+//line app/vmselect/prometheus/export.qtpl:105
+				qw422016.N().S(`null`)
+//line app/vmselect/prometheus/export.qtpl:105
+			} else {
+//line app/vmselect/prometheus/export.qtpl:105
+				qw422016.N().F(v)
+//line app/vmselect/prometheus/export.qtpl:105
+			}
+//line app/vmselect/prometheus/export.qtpl:106
 		}
-//line app/vmselect/prometheus/export.qtpl:106
+//line app/vmselect/prometheus/export.qtpl:107
 	}
-//line app/vmselect/prometheus/export.qtpl:106
+//line app/vmselect/prometheus/export.qtpl:107
 	qw422016.N().S(`],"timestamps":[`)
-//line app/vmselect/prometheus/export.qtpl:109
-	if len(xb.timestamps) > 0 {
 //line app/vmselect/prometheus/export.qtpl:110
+	if len(xb.timestamps) > 0 {
+//line app/vmselect/prometheus/export.qtpl:111
 		timestamps := xb.timestamps
 
-//line app/vmselect/prometheus/export.qtpl:111
-		qw422016.N().DL(timestamps[0])
 //line app/vmselect/prometheus/export.qtpl:112
+		qw422016.N().DL(timestamps[0])
+//line app/vmselect/prometheus/export.qtpl:113
 		timestamps = timestamps[1:]
 
-//line app/vmselect/prometheus/export.qtpl:113
-		for _, ts := range timestamps {
-//line app/vmselect/prometheus/export.qtpl:113
-			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:114
-			qw422016.N().DL(ts)
+		for _, ts := range timestamps {
+//line app/vmselect/prometheus/export.qtpl:114
+			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:115
+			qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:116
 		}
-//line app/vmselect/prometheus/export.qtpl:116
+//line app/vmselect/prometheus/export.qtpl:117
 	}
-//line app/vmselect/prometheus/export.qtpl:116
+//line app/vmselect/prometheus/export.qtpl:117
 	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/export.qtpl:118
+//line app/vmselect/prometheus/export.qtpl:119
 	qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 }
 
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 func WriteExportJSONLine(qq422016 qtio422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	StreamExportJSONLine(qw422016, xb)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 }
 
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 func ExportJSONLine(xb *exportBlock) string {
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	WriteExportJSONLine(qb422016, xb)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:119
+//line app/vmselect/prometheus/export.qtpl:120
 }
 
-//line app/vmselect/prometheus/export.qtpl:121
+//line app/vmselect/prometheus/export.qtpl:122
 func StreamExportPromAPILine(qw422016 *qt422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:121
+//line app/vmselect/prometheus/export.qtpl:122
 	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:123
+//line app/vmselect/prometheus/export.qtpl:124
 	streammetricNameObject(qw422016, xb.mn)
-//line app/vmselect/prometheus/export.qtpl:123
+//line app/vmselect/prometheus/export.qtpl:124
 	qw422016.N().S(`,"values":`)
-//line app/vmselect/prometheus/export.qtpl:124
+//line app/vmselect/prometheus/export.qtpl:125
 	streamvaluesWithTimestamps(qw422016, xb.values, xb.timestamps)
-//line app/vmselect/prometheus/export.qtpl:124
+//line app/vmselect/prometheus/export.qtpl:125
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 }
 
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 func WriteExportPromAPILine(qq422016 qtio422016.Writer, xb *exportBlock) {
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	StreamExportPromAPILine(qw422016, xb)
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 }
 
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 func ExportPromAPILine(xb *exportBlock) string {
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	WriteExportPromAPILine(qb422016, xb)
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:126
+//line app/vmselect/prometheus/export.qtpl:127
 }
 
-//line app/vmselect/prometheus/export.qtpl:128
-func StreamExportPromAPIResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:128
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vmselect/prometheus/export.qtpl:129
+func StreamExportPromAPIResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/export.qtpl:129
+	qw422016.N().S(`{`)
+//line app/vmselect/prometheus/export.qtpl:132
+	lines := 0
+	bytesTotal := 0
+
 //line app/vmselect/prometheus/export.qtpl:134
+	qw422016.N().S(`"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vmselect/prometheus/export.qtpl:139
 	bb, ok := <-resultsCh
 
-//line app/vmselect/prometheus/export.qtpl:135
+//line app/vmselect/prometheus/export.qtpl:140
 	if ok {
-//line app/vmselect/prometheus/export.qtpl:136
+//line app/vmselect/prometheus/export.qtpl:141
 		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:137
+//line app/vmselect/prometheus/export.qtpl:143
+		lines++
+		bytesTotal += len(bb.B)
 		quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:138
+//line app/vmselect/prometheus/export.qtpl:147
 		for bb := range resultsCh {
-//line app/vmselect/prometheus/export.qtpl:138
+//line app/vmselect/prometheus/export.qtpl:147
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/export.qtpl:139
+//line app/vmselect/prometheus/export.qtpl:148
 			qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:140
+//line app/vmselect/prometheus/export.qtpl:150
+			lines++
+			bytesTotal += len(bb.B)
 			quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:141
+//line app/vmselect/prometheus/export.qtpl:154
 		}
-//line app/vmselect/prometheus/export.qtpl:142
+//line app/vmselect/prometheus/export.qtpl:155
 	}
-//line app/vmselect/prometheus/export.qtpl:142
-	qw422016.N().S(`]}}`)
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:155
+	qw422016.N().S(`]}`)
+//line app/vmselect/prometheus/export.qtpl:159
+	qt.Donef("export format=promapi: lines=%d, bytes=%d", lines, bytesTotal)
+
+//line app/vmselect/prometheus/export.qtpl:161
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/export.qtpl:161
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/export.qtpl:163
 }
 
-//line app/vmselect/prometheus/export.qtpl:146
-func WriteExportPromAPIResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
+func WriteExportPromAPIResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/export.qtpl:163
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:146
-	StreamExportPromAPIResponse(qw422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
+	StreamExportPromAPIResponse(qw422016, resultsCh, qt)
+//line app/vmselect/prometheus/export.qtpl:163
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
 }
 
-//line app/vmselect/prometheus/export.qtpl:146
-func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
+func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/export.qtpl:163
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:146
-	WriteExportPromAPIResponse(qb422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
+	WriteExportPromAPIResponse(qb422016, resultsCh, qt)
+//line app/vmselect/prometheus/export.qtpl:163
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:146
+//line app/vmselect/prometheus/export.qtpl:163
 }
 
-//line app/vmselect/prometheus/export.qtpl:148
-func StreamExportStdResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:149
+//line app/vmselect/prometheus/export.qtpl:165
+func StreamExportStdResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/export.qtpl:166
 	for bb := range resultsCh {
-//line app/vmselect/prometheus/export.qtpl:150
+//line app/vmselect/prometheus/export.qtpl:167
 		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:151
+//line app/vmselect/prometheus/export.qtpl:168
 		quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:152
+//line app/vmselect/prometheus/export.qtpl:169
 	}
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
 }
 
-//line app/vmselect/prometheus/export.qtpl:153
-func WriteExportStdResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
+func WriteExportStdResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/export.qtpl:170
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:153
-	StreamExportStdResponse(qw422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
+	StreamExportStdResponse(qw422016, resultsCh, qt)
+//line app/vmselect/prometheus/export.qtpl:170
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
 }
 
-//line app/vmselect/prometheus/export.qtpl:153
-func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
+func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/export.qtpl:170
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:153
-	WriteExportStdResponse(qb422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
+	WriteExportStdResponse(qb422016, resultsCh, qt)
+//line app/vmselect/prometheus/export.qtpl:170
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:153
+//line app/vmselect/prometheus/export.qtpl:170
 }
 
-//line app/vmselect/prometheus/export.qtpl:155
+//line app/vmselect/prometheus/export.qtpl:172
 func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
-//line app/vmselect/prometheus/export.qtpl:156
+//line app/vmselect/prometheus/export.qtpl:173
 	qw422016.N().Z(mn.MetricGroup)
-//line app/vmselect/prometheus/export.qtpl:157
+//line app/vmselect/prometheus/export.qtpl:174
 	if len(mn.Tags) > 0 {
-//line app/vmselect/prometheus/export.qtpl:157
+//line app/vmselect/prometheus/export.qtpl:174
 		qw422016.N().S(`{`)
-//line app/vmselect/prometheus/export.qtpl:159
+//line app/vmselect/prometheus/export.qtpl:176
 		tags := mn.Tags
 
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:177
 		qw422016.N().Z(tags[0].Key)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:177
 		qw422016.N().S(`=`)
-//line app/vmselect/prometheus/export.qtpl:160
+//line app/vmselect/prometheus/export.qtpl:177
 		qw422016.N().QZ(tags[0].Value)
-//line app/vmselect/prometheus/export.qtpl:161
+//line app/vmselect/prometheus/export.qtpl:178
 		tags = tags[1:]
 
-//line app/vmselect/prometheus/export.qtpl:162
+//line app/vmselect/prometheus/export.qtpl:179
 		for i := range tags {
-//line app/vmselect/prometheus/export.qtpl:163
+//line app/vmselect/prometheus/export.qtpl:180
 			tag := &tags[i]
 
-//line app/vmselect/prometheus/export.qtpl:163
+//line app/vmselect/prometheus/export.qtpl:180
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/export.qtpl:164
+//line app/vmselect/prometheus/export.qtpl:181
 			qw422016.N().Z(tag.Key)
-//line app/vmselect/prometheus/export.qtpl:164
+//line app/vmselect/prometheus/export.qtpl:181
 			qw422016.N().S(`=`)
-//line app/vmselect/prometheus/export.qtpl:164
+//line app/vmselect/prometheus/export.qtpl:181
 			qw422016.N().QZ(tag.Value)
-//line app/vmselect/prometheus/export.qtpl:165
+//line app/vmselect/prometheus/export.qtpl:182
 		}
-//line app/vmselect/prometheus/export.qtpl:165
+//line app/vmselect/prometheus/export.qtpl:182
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/export.qtpl:167
+//line app/vmselect/prometheus/export.qtpl:184
 	}
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 }
 
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 func writeprometheusMetricName(qq422016 qtio422016.Writer, mn *storage.MetricName) {
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	streamprometheusMetricName(qw422016, mn)
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 }
 
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 func prometheusMetricName(mn *storage.MetricName) string {
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	writeprometheusMetricName(qb422016, mn)
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:168
+//line app/vmselect/prometheus/export.qtpl:185
 }

app/vmselect/prometheus/label_values_response.qtpl

@@ -1,7 +1,12 @@
 {% stripspace %}
+
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+) %}
+
 LabelValuesResponse generates response for /api/v1/label/<labelName>/values .
 See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
-{% func LabelValuesResponse(labelValues []string) %}
+{% func LabelValuesResponse(labelValues []string, qt *querytracer.Tracer) %}
 {
 	"status":"success",
 	"data":[
@@ -10,6 +15,11 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-va
 			{% if i+1 < len(labelValues) %},{% endif %}
 		{% endfor %}
 	]
+	{% code
+		qt.Printf("generate response for %d label values", len(labelValues))
+		qt.Done()
+	%}
+	{%= dumpQueryTrace(qt) %}
 }
 {% endfunc %}
 {% endstripspace %}

app/vmselect/prometheus/label_values_response.qtpl.go

@@ -1,67 +1,80 @@
 // Code generated by qtc from "label_values_response.qtpl". DO NOT EDIT.
 // See https://github.com/valyala/quicktemplate for details.
 
-// LabelValuesResponse generates response for /api/v1/label/<labelName>/values .See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
-
-//line app/vmselect/prometheus/label_values_response.qtpl:4
+//line app/vmselect/prometheus/label_values_response.qtpl:3
 package prometheus
 
-//line app/vmselect/prometheus/label_values_response.qtpl:4
+//line app/vmselect/prometheus/label_values_response.qtpl:3
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+)
+
+// LabelValuesResponse generates response for /api/v1/label/<labelName>/values .See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
+
+//line app/vmselect/prometheus/label_values_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/label_values_response.qtpl:4
+//line app/vmselect/prometheus/label_values_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/label_values_response.qtpl:4
-func StreamLabelValuesResponse(qw422016 *qt422016.Writer, labelValues []string) {
-//line app/vmselect/prometheus/label_values_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":[`)
-//line app/vmselect/prometheus/label_values_response.qtpl:8
-	for i, labelValue := range labelValues {
 //line app/vmselect/prometheus/label_values_response.qtpl:9
+func StreamLabelValuesResponse(qw422016 *qt422016.Writer, labelValues []string, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/label_values_response.qtpl:9
+	qw422016.N().S(`{"status":"success","data":[`)
+//line app/vmselect/prometheus/label_values_response.qtpl:13
+	for i, labelValue := range labelValues {
+//line app/vmselect/prometheus/label_values_response.qtpl:14
 		qw422016.N().Q(labelValue)
-//line app/vmselect/prometheus/label_values_response.qtpl:10
+//line app/vmselect/prometheus/label_values_response.qtpl:15
 		if i+1 < len(labelValues) {
-//line app/vmselect/prometheus/label_values_response.qtpl:10
+//line app/vmselect/prometheus/label_values_response.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/label_values_response.qtpl:10
+//line app/vmselect/prometheus/label_values_response.qtpl:15
 		}
-//line app/vmselect/prometheus/label_values_response.qtpl:11
+//line app/vmselect/prometheus/label_values_response.qtpl:16
 	}
-//line app/vmselect/prometheus/label_values_response.qtpl:11
-	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:16
+	qw422016.N().S(`]`)
+//line app/vmselect/prometheus/label_values_response.qtpl:19
+	qt.Printf("generate response for %d label values", len(labelValues))
+	qt.Done()
+
+//line app/vmselect/prometheus/label_values_response.qtpl:22
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/label_values_response.qtpl:22
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 }
 
-//line app/vmselect/prometheus/label_values_response.qtpl:14
-func WriteLabelValuesResponse(qq422016 qtio422016.Writer, labelValues []string) {
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
+func WriteLabelValuesResponse(qq422016 qtio422016.Writer, labelValues []string, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/label_values_response.qtpl:14
-	StreamLabelValuesResponse(qw422016, labelValues)
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
+	StreamLabelValuesResponse(qw422016, labelValues, qt)
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 }
 
-//line app/vmselect/prometheus/label_values_response.qtpl:14
-func LabelValuesResponse(labelValues []string) string {
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
+func LabelValuesResponse(labelValues []string, qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/label_values_response.qtpl:14
-	WriteLabelValuesResponse(qb422016, labelValues)
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
+	WriteLabelValuesResponse(qb422016, labelValues, qt)
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 	return qs422016
-//line app/vmselect/prometheus/label_values_response.qtpl:14
+//line app/vmselect/prometheus/label_values_response.qtpl:24
 }

app/vmselect/prometheus/labels_count_response.qtpl

@@ -1,17 +0,0 @@
-{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" %}
-
-{% stripspace %}
-LabelsCountResponse generates response for /api/v1/labels/count .
-{% func LabelsCountResponse(labelEntries []storage.TagEntry) %}
-{
-	"status":"success",
-	"data":{
-		{% for i, e := range labelEntries %}
-			{%q= e.Key %}:{%d= len(e.Values) %}
-			{% if i+1 < len(labelEntries) %},{% endif %}
-		{% endfor %}
-	}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vmselect/prometheus/labels_count_response.qtpl.go

@@ -1,74 +0,0 @@
-// Code generated by qtc from "labels_count_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vmselect/prometheus/labels_count_response.qtpl:1
-package prometheus
-
-//line app/vmselect/prometheus/labels_count_response.qtpl:1
-import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-
-// LabelsCountResponse generates response for /api/v1/labels/count .
-
-//line app/vmselect/prometheus/labels_count_response.qtpl:5
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vmselect/prometheus/labels_count_response.qtpl:5
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vmselect/prometheus/labels_count_response.qtpl:5
-func StreamLabelsCountResponse(qw422016 *qt422016.Writer, labelEntries []storage.TagEntry) {
-//line app/vmselect/prometheus/labels_count_response.qtpl:5
-	qw422016.N().S(`{"status":"success","data":{`)
-//line app/vmselect/prometheus/labels_count_response.qtpl:9
-	for i, e := range labelEntries {
-//line app/vmselect/prometheus/labels_count_response.qtpl:10
-		qw422016.N().Q(e.Key)
-//line app/vmselect/prometheus/labels_count_response.qtpl:10
-		qw422016.N().S(`:`)
-//line app/vmselect/prometheus/labels_count_response.qtpl:10
-		qw422016.N().D(len(e.Values))
-//line app/vmselect/prometheus/labels_count_response.qtpl:11
-		if i+1 < len(labelEntries) {
-//line app/vmselect/prometheus/labels_count_response.qtpl:11
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/labels_count_response.qtpl:11
-		}
-//line app/vmselect/prometheus/labels_count_response.qtpl:12
-	}
-//line app/vmselect/prometheus/labels_count_response.qtpl:12
-	qw422016.N().S(`}}`)
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-}
-
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-func WriteLabelsCountResponse(qq422016 qtio422016.Writer, labelEntries []storage.TagEntry) {
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	StreamLabelsCountResponse(qw422016, labelEntries)
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-}
-
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-func LabelsCountResponse(labelEntries []storage.TagEntry) string {
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	WriteLabelsCountResponse(qb422016, labelEntries)
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-	return qs422016
-//line app/vmselect/prometheus/labels_count_response.qtpl:15
-}

app/vmselect/prometheus/labels_response.qtpl

@@ -1,7 +1,12 @@
 {% stripspace %}
+
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+) %}
+
 LabelsResponse generates response for /api/v1/labels .
 See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
-{% func LabelsResponse(labels []string) %}
+{% func LabelsResponse(labels []string, qt *querytracer.Tracer) %}
 {
 	"status":"success",
 	"data":[
@@ -10,6 +15,11 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-nam
 			{% if i+1 < len(labels) %},{% endif %}
 		{% endfor %}
 	]
+	{% code
+		qt.Printf("generate response for %d labels", len(labels))
+		qt.Done()
+	%}
+	{%= dumpQueryTrace(qt) %}
 }
 {% endfunc %}
 {% endstripspace %}

app/vmselect/prometheus/labels_response.qtpl.go

@@ -1,67 +1,80 @@
 // Code generated by qtc from "labels_response.qtpl". DO NOT EDIT.
 // See https://github.com/valyala/quicktemplate for details.
 
-// LabelsResponse generates response for /api/v1/labels .See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
-
-//line app/vmselect/prometheus/labels_response.qtpl:4
+//line app/vmselect/prometheus/labels_response.qtpl:3
 package prometheus
 
-//line app/vmselect/prometheus/labels_response.qtpl:4
+//line app/vmselect/prometheus/labels_response.qtpl:3
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+)
+
+// LabelsResponse generates response for /api/v1/labels .See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
+
+//line app/vmselect/prometheus/labels_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/labels_response.qtpl:4
+//line app/vmselect/prometheus/labels_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/labels_response.qtpl:4
-func StreamLabelsResponse(qw422016 *qt422016.Writer, labels []string) {
-//line app/vmselect/prometheus/labels_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":[`)
-//line app/vmselect/prometheus/labels_response.qtpl:8
-	for i, label := range labels {
 //line app/vmselect/prometheus/labels_response.qtpl:9
+func StreamLabelsResponse(qw422016 *qt422016.Writer, labels []string, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/labels_response.qtpl:9
+	qw422016.N().S(`{"status":"success","data":[`)
+//line app/vmselect/prometheus/labels_response.qtpl:13
+	for i, label := range labels {
+//line app/vmselect/prometheus/labels_response.qtpl:14
 		qw422016.N().Q(label)
-//line app/vmselect/prometheus/labels_response.qtpl:10
+//line app/vmselect/prometheus/labels_response.qtpl:15
 		if i+1 < len(labels) {
-//line app/vmselect/prometheus/labels_response.qtpl:10
+//line app/vmselect/prometheus/labels_response.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/labels_response.qtpl:10
+//line app/vmselect/prometheus/labels_response.qtpl:15
 		}
-//line app/vmselect/prometheus/labels_response.qtpl:11
+//line app/vmselect/prometheus/labels_response.qtpl:16
 	}
-//line app/vmselect/prometheus/labels_response.qtpl:11
-	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:16
+	qw422016.N().S(`]`)
+//line app/vmselect/prometheus/labels_response.qtpl:19
+	qt.Printf("generate response for %d labels", len(labels))
+	qt.Done()
+
+//line app/vmselect/prometheus/labels_response.qtpl:22
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/labels_response.qtpl:22
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/labels_response.qtpl:24
 }
 
-//line app/vmselect/prometheus/labels_response.qtpl:14
-func WriteLabelsResponse(qq422016 qtio422016.Writer, labels []string) {
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
+func WriteLabelsResponse(qq422016 qtio422016.Writer, labels []string, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/labels_response.qtpl:24
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/labels_response.qtpl:14
-	StreamLabelsResponse(qw422016, labels)
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
+	StreamLabelsResponse(qw422016, labels, qt)
+//line app/vmselect/prometheus/labels_response.qtpl:24
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
 }
 
-//line app/vmselect/prometheus/labels_response.qtpl:14
-func LabelsResponse(labels []string) string {
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
+func LabelsResponse(labels []string, qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/labels_response.qtpl:24
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/labels_response.qtpl:14
-	WriteLabelsResponse(qb422016, labels)
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
+	WriteLabelsResponse(qb422016, labels, qt)
+//line app/vmselect/prometheus/labels_response.qtpl:24
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
 	return qs422016
-//line app/vmselect/prometheus/labels_response.qtpl:14
+//line app/vmselect/prometheus/labels_response.qtpl:24
 }

app/vmselect/prometheus/query_range_response.qtpl

@@ -1,25 +1,37 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
 {% stripspace %}
 QueryRangeResponse generates response for /api/v1/query_range.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
-{% func QueryRangeResponse(rs []netstorage.Result) %}
+{% func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %}
 {
+	{% code
+		seriesCount := len(rs)
+		pointsCount := 0
+	%}
 	"status":"success",
 	"data":{
 		"resultType":"matrix",
 		"result":[
 			{% if len(rs) > 0 %}
 				{%= queryRangeLine(&rs[0]) %}
+				{% code pointsCount += len(rs[0].Values) %}
 				{% code rs = rs[1:] %}
 				{% for i := range rs %}
 					,{%= queryRangeLine(&rs[i]) %}
+					{% code pointsCount += len(rs[i].Values) %}
 				{% endfor %}
 			{% endif %}
 		]
 	}
+	{% code
+		qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)
+		qtDone()
+	%}
+	{%= dumpQueryTrace(qt) %}
 }
 {% endfunc %}
 

app/vmselect/prometheus/query_range_response.qtpl.go

@@ -7,112 +7,133 @@ package prometheus
 //line app/vmselect/prometheus/query_range_response.qtpl:1
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 )
 
 // QueryRangeResponse generates response for /api/v1/query_range.See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 
-//line app/vmselect/prometheus/query_range_response.qtpl:8
+//line app/vmselect/prometheus/query_range_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/query_range_response.qtpl:8
+//line app/vmselect/prometheus/query_range_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/query_range_response.qtpl:8
-func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:8
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vmselect/prometheus/query_range_response.qtpl:9
+func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
+//line app/vmselect/prometheus/query_range_response.qtpl:9
+	qw422016.N().S(`{`)
+//line app/vmselect/prometheus/query_range_response.qtpl:12
+	seriesCount := len(rs)
+	pointsCount := 0
+
 //line app/vmselect/prometheus/query_range_response.qtpl:14
+	qw422016.N().S(`"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vmselect/prometheus/query_range_response.qtpl:19
 	if len(rs) > 0 {
-//line app/vmselect/prometheus/query_range_response.qtpl:15
+//line app/vmselect/prometheus/query_range_response.qtpl:20
 		streamqueryRangeLine(qw422016, &rs[0])
-//line app/vmselect/prometheus/query_range_response.qtpl:16
+//line app/vmselect/prometheus/query_range_response.qtpl:21
+		pointsCount += len(rs[0].Values)
+
+//line app/vmselect/prometheus/query_range_response.qtpl:22
 		rs = rs[1:]
 
-//line app/vmselect/prometheus/query_range_response.qtpl:17
+//line app/vmselect/prometheus/query_range_response.qtpl:23
 		for i := range rs {
-//line app/vmselect/prometheus/query_range_response.qtpl:17
+//line app/vmselect/prometheus/query_range_response.qtpl:23
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/query_range_response.qtpl:18
+//line app/vmselect/prometheus/query_range_response.qtpl:24
 			streamqueryRangeLine(qw422016, &rs[i])
-//line app/vmselect/prometheus/query_range_response.qtpl:19
+//line app/vmselect/prometheus/query_range_response.qtpl:25
+			pointsCount += len(rs[i].Values)
+
+//line app/vmselect/prometheus/query_range_response.qtpl:26
 		}
-//line app/vmselect/prometheus/query_range_response.qtpl:20
+//line app/vmselect/prometheus/query_range_response.qtpl:27
 	}
-//line app/vmselect/prometheus/query_range_response.qtpl:20
-	qw422016.N().S(`]}}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-}
+//line app/vmselect/prometheus/query_range_response.qtpl:27
+	qw422016.N().S(`]}`)
+//line app/vmselect/prometheus/query_range_response.qtpl:31
+	qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)
+	qtDone()
 
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-func WriteQueryRangeResponse(qq422016 qtio422016.Writer, rs []netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	StreamQueryRangeResponse(qw422016, rs)
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-}
-
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-func QueryRangeResponse(rs []netstorage.Result) string {
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	WriteQueryRangeResponse(qb422016, rs)
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-	return qs422016
-//line app/vmselect/prometheus/query_range_response.qtpl:24
-}
-
-//line app/vmselect/prometheus/query_range_response.qtpl:26
-func streamqueryRangeLine(qw422016 *qt422016.Writer, r *netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:26
-	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:28
-	streammetricNameObject(qw422016, &r.MetricName)
-//line app/vmselect/prometheus/query_range_response.qtpl:28
-	qw422016.N().S(`,"values":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:29
-	streamvaluesWithTimestamps(qw422016, r.Values, r.Timestamps)
-//line app/vmselect/prometheus/query_range_response.qtpl:29
+//line app/vmselect/prometheus/query_range_response.qtpl:34
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/query_range_response.qtpl:34
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:31
-func writequeryRangeLine(qq422016 qtio422016.Writer, r *netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
+func WriteQueryRangeResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
-	streamqueryRangeLine(qw422016, r)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
+	StreamQueryRangeResponse(qw422016, rs, qt, qtDone)
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:31
-func queryRangeLine(r *netstorage.Result) string {
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
+func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string {
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_range_response.qtpl:31
-	writequeryRangeLine(qb422016, r)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
+	WriteQueryRangeResponse(qb422016, rs, qt, qtDone)
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
 	return qs422016
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:36
+}
+
+//line app/vmselect/prometheus/query_range_response.qtpl:38
+func streamqueryRangeLine(qw422016 *qt422016.Writer, r *netstorage.Result) {
+//line app/vmselect/prometheus/query_range_response.qtpl:38
+	qw422016.N().S(`{"metric":`)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	streammetricNameObject(qw422016, &r.MetricName)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	qw422016.N().S(`,"values":`)
+//line app/vmselect/prometheus/query_range_response.qtpl:41
+	streamvaluesWithTimestamps(qw422016, r.Values, r.Timestamps)
+//line app/vmselect/prometheus/query_range_response.qtpl:41
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+}
+
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+func writequeryRangeLine(qq422016 qtio422016.Writer, r *netstorage.Result) {
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	streamqueryRangeLine(qw422016, r)
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+}
+
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+func queryRangeLine(r *netstorage.Result) string {
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	writequeryRangeLine(qb422016, r)
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/query_range_response.qtpl:43
+	return qs422016
+//line app/vmselect/prometheus/query_range_response.qtpl:43
 }

app/vmselect/prometheus/query_response.qtpl

@@ -1,12 +1,14 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
 {% stripspace %}
 QueryResponse generates response for /api/v1/query.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
-{% func QueryResponse(rs []netstorage.Result) %}
+{% func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %}
 {
+	{% code seriesCount := len(rs) %}
 	"status":"success",
 	"data":{
 		"resultType":"vector",
@@ -27,6 +29,11 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 			{% endif %}
 		]
 	}
+	{% code
+		qt.Printf("generate /api/v1/query response for series=%d", seriesCount)
+		qtDone()
+	%}
+	{%= dumpQueryTrace(qt) %}
 }
 {% endfunc %}
 {% endstripspace %}

app/vmselect/prometheus/query_response.qtpl.go

@@ -7,88 +7,102 @@ package prometheus
 //line app/vmselect/prometheus/query_response.qtpl:1
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 )
 
 // QueryResponse generates response for /api/v1/query.See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 
-//line app/vmselect/prometheus/query_response.qtpl:8
+//line app/vmselect/prometheus/query_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/query_response.qtpl:8
+//line app/vmselect/prometheus/query_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/query_response.qtpl:8
-func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result) {
-//line app/vmselect/prometheus/query_response.qtpl:8
-	qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
-//line app/vmselect/prometheus/query_response.qtpl:14
+//line app/vmselect/prometheus/query_response.qtpl:9
+func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
+//line app/vmselect/prometheus/query_response.qtpl:9
+	qw422016.N().S(`{`)
+//line app/vmselect/prometheus/query_response.qtpl:11
+	seriesCount := len(rs)
+
+//line app/vmselect/prometheus/query_response.qtpl:11
+	qw422016.N().S(`"status":"success","data":{"resultType":"vector","result":[`)
+//line app/vmselect/prometheus/query_response.qtpl:16
 	if len(rs) > 0 {
-//line app/vmselect/prometheus/query_response.qtpl:14
+//line app/vmselect/prometheus/query_response.qtpl:16
 		qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/query_response.qtpl:16
+//line app/vmselect/prometheus/query_response.qtpl:18
 		streammetricNameObject(qw422016, &rs[0].MetricName)
-//line app/vmselect/prometheus/query_response.qtpl:16
+//line app/vmselect/prometheus/query_response.qtpl:18
 		qw422016.N().S(`,"value":`)
-//line app/vmselect/prometheus/query_response.qtpl:17
-		streammetricRow(qw422016, rs[0].Timestamps[0], rs[0].Values[0])
-//line app/vmselect/prometheus/query_response.qtpl:17
-		qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_response.qtpl:19
+		streammetricRow(qw422016, rs[0].Timestamps[0], rs[0].Values[0])
+//line app/vmselect/prometheus/query_response.qtpl:19
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/query_response.qtpl:21
 		rs = rs[1:]
 
-//line app/vmselect/prometheus/query_response.qtpl:20
+//line app/vmselect/prometheus/query_response.qtpl:22
 		for i := range rs {
-//line app/vmselect/prometheus/query_response.qtpl:21
+//line app/vmselect/prometheus/query_response.qtpl:23
 			r := &rs[i]
 
-//line app/vmselect/prometheus/query_response.qtpl:21
+//line app/vmselect/prometheus/query_response.qtpl:23
 			qw422016.N().S(`,{"metric":`)
-//line app/vmselect/prometheus/query_response.qtpl:23
+//line app/vmselect/prometheus/query_response.qtpl:25
 			streammetricNameObject(qw422016, &r.MetricName)
-//line app/vmselect/prometheus/query_response.qtpl:23
+//line app/vmselect/prometheus/query_response.qtpl:25
 			qw422016.N().S(`,"value":`)
-//line app/vmselect/prometheus/query_response.qtpl:24
-			streammetricRow(qw422016, r.Timestamps[0], r.Values[0])
-//line app/vmselect/prometheus/query_response.qtpl:24
-			qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_response.qtpl:26
+			streammetricRow(qw422016, r.Timestamps[0], r.Values[0])
+//line app/vmselect/prometheus/query_response.qtpl:26
+			qw422016.N().S(`}`)
+//line app/vmselect/prometheus/query_response.qtpl:28
 		}
-//line app/vmselect/prometheus/query_response.qtpl:27
+//line app/vmselect/prometheus/query_response.qtpl:29
 	}
-//line app/vmselect/prometheus/query_response.qtpl:27
-	qw422016.N().S(`]}}`)
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:29
+	qw422016.N().S(`]}`)
+//line app/vmselect/prometheus/query_response.qtpl:33
+	qt.Printf("generate /api/v1/query response for series=%d", seriesCount)
+	qtDone()
+
+//line app/vmselect/prometheus/query_response.qtpl:36
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/query_response.qtpl:36
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/query_response.qtpl:38
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:31
-func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result) {
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
+func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
+//line app/vmselect/prometheus/query_response.qtpl:38
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_response.qtpl:31
-	StreamQueryResponse(qw422016, rs)
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
+	StreamQueryResponse(qw422016, rs, qt, qtDone)
+//line app/vmselect/prometheus/query_response.qtpl:38
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:31
-func QueryResponse(rs []netstorage.Result) string {
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
+func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string {
+//line app/vmselect/prometheus/query_response.qtpl:38
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_response.qtpl:31
-	WriteQueryResponse(qb422016, rs)
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
+	WriteQueryResponse(qb422016, rs, qt, qtDone)
+//line app/vmselect/prometheus/query_response.qtpl:38
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
 	return qs422016
-//line app/vmselect/prometheus/query_response.qtpl:31
+//line app/vmselect/prometheus/query_response.qtpl:38
 }

app/vmselect/prometheus/series_response.qtpl

@@ -1,24 +1,37 @@
 {% import (
 	"github.com/valyala/quicktemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
 {% stripspace %}
 SeriesResponse generates response for /api/v1/series.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers
-{% func SeriesResponse(resultsCh <-chan *quicktemplate.ByteBuffer) %}
+{% func SeriesResponse(resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer, qtDone func()) %}
 {
+	{% code seriesCount := 0 %}
 	"status":"success",
 	"data":[
 		{% code bb, ok := <-resultsCh %}
 		{% if ok %}
 			{%z= bb.B %}
-			{% code quicktemplate.ReleaseByteBuffer(bb) %}
+			{% code
+				quicktemplate.ReleaseByteBuffer(bb)
+				seriesCount++
+			%}
 			{% for bb := range resultsCh %}
 				,{%z= bb.B %}
-				{% code quicktemplate.ReleaseByteBuffer(bb) %}
+				{% code
+					quicktemplate.ReleaseByteBuffer(bb)
+					seriesCount++
+				%}
 			{% endfor %}
 		{% endif %}
 	]
+	{% code
+		qt.Printf("generate response: series=%d", seriesCount)
+		qtDone()
+	%}
+	{%= dumpQueryTrace(qt) %}
 }
 {% endfunc %}
 {% endstripspace %}

app/vmselect/prometheus/series_response.qtpl.go

@@ -6,78 +6,94 @@ package prometheus
 
 //line app/vmselect/prometheus/series_response.qtpl:1
 import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/valyala/quicktemplate"
 )
 
 // SeriesResponse generates response for /api/v1/series.See https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers
 
-//line app/vmselect/prometheus/series_response.qtpl:8
+//line app/vmselect/prometheus/series_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/series_response.qtpl:8
+//line app/vmselect/prometheus/series_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/series_response.qtpl:8
-func StreamSeriesResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/series_response.qtpl:8
-	qw422016.N().S(`{"status":"success","data":[`)
-//line app/vmselect/prometheus/series_response.qtpl:12
+//line app/vmselect/prometheus/series_response.qtpl:9
+func StreamSeriesResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer, qtDone func()) {
+//line app/vmselect/prometheus/series_response.qtpl:9
+	qw422016.N().S(`{`)
+//line app/vmselect/prometheus/series_response.qtpl:11
+	seriesCount := 0
+
+//line app/vmselect/prometheus/series_response.qtpl:11
+	qw422016.N().S(`"status":"success","data":[`)
+//line app/vmselect/prometheus/series_response.qtpl:14
 	bb, ok := <-resultsCh
 
-//line app/vmselect/prometheus/series_response.qtpl:13
-	if ok {
-//line app/vmselect/prometheus/series_response.qtpl:14
-		qw422016.N().Z(bb.B)
 //line app/vmselect/prometheus/series_response.qtpl:15
-		quicktemplate.ReleaseByteBuffer(bb)
-
+	if ok {
 //line app/vmselect/prometheus/series_response.qtpl:16
-		for bb := range resultsCh {
-//line app/vmselect/prometheus/series_response.qtpl:16
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/series_response.qtpl:17
-			qw422016.N().Z(bb.B)
+		qw422016.N().Z(bb.B)
 //line app/vmselect/prometheus/series_response.qtpl:18
+		quicktemplate.ReleaseByteBuffer(bb)
+		seriesCount++
+
+//line app/vmselect/prometheus/series_response.qtpl:21
+		for bb := range resultsCh {
+//line app/vmselect/prometheus/series_response.qtpl:21
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/series_response.qtpl:22
+			qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/series_response.qtpl:24
 			quicktemplate.ReleaseByteBuffer(bb)
+			seriesCount++
 
-//line app/vmselect/prometheus/series_response.qtpl:19
+//line app/vmselect/prometheus/series_response.qtpl:27
 		}
-//line app/vmselect/prometheus/series_response.qtpl:20
+//line app/vmselect/prometheus/series_response.qtpl:28
 	}
-//line app/vmselect/prometheus/series_response.qtpl:20
-	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:28
+	qw422016.N().S(`]`)
+//line app/vmselect/prometheus/series_response.qtpl:31
+	qt.Printf("generate response: series=%d", seriesCount)
+	qtDone()
+
+//line app/vmselect/prometheus/series_response.qtpl:34
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/series_response.qtpl:34
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/series_response.qtpl:36
 }
 
-//line app/vmselect/prometheus/series_response.qtpl:23
-func WriteSeriesResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
+func WriteSeriesResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer, qtDone func()) {
+//line app/vmselect/prometheus/series_response.qtpl:36
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/series_response.qtpl:23
-	StreamSeriesResponse(qw422016, resultsCh)
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
+	StreamSeriesResponse(qw422016, resultsCh, qt, qtDone)
+//line app/vmselect/prometheus/series_response.qtpl:36
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
 }
 
-//line app/vmselect/prometheus/series_response.qtpl:23
-func SeriesResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
+func SeriesResponse(resultsCh <-chan *quicktemplate.ByteBuffer, qt *querytracer.Tracer, qtDone func()) string {
+//line app/vmselect/prometheus/series_response.qtpl:36
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/series_response.qtpl:23
-	WriteSeriesResponse(qb422016, resultsCh)
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
+	WriteSeriesResponse(qb422016, resultsCh, qt, qtDone)
+//line app/vmselect/prometheus/series_response.qtpl:36
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
 	return qs422016
-//line app/vmselect/prometheus/series_response.qtpl:23
+//line app/vmselect/prometheus/series_response.qtpl:36
 }

app/vmselect/prometheus/tsdb_status_response.qtpl

@@ -1,15 +1,24 @@
-{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" %}
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+) %}
 
 {% stripspace %}
 TSDBStatusResponse generates response for /api/v1/status/tsdb .
-{% func TSDBStatusResponse(status *storage.TSDBStatus) %}
+{% func TSDBStatusResponse(status *storage.TSDBStatus, qt *querytracer.Tracer) %}
 {
 	"status":"success",
 	"data":{
+		"totalSeries": {%dul= status.TotalSeries %},
+		"totalLabelValuePairs": {%dul= status.TotalLabelValuePairs %},
 		"seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %},
-		"labelValueCountByLabelName":{%= tsdbStatusEntries(status.LabelValueCountByLabelName) %},
-		"seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %}
+		"seriesCountByLabelName":{%= tsdbStatusEntries(status.SeriesCountByLabelName) %},
+		"seriesCountByFocusLabelValue":{%= tsdbStatusEntries(status.SeriesCountByFocusLabelValue) %},
+		"seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %},
+		"labelValueCountByLabelName":{%= tsdbStatusEntries(status.LabelValueCountByLabelName) %}
 	}
+	{% code	qt.Done() %}
+	{%= dumpQueryTrace(qt) %}
 }
 {% endfunc %}
 

app/vmselect/prometheus/tsdb_status_response.qtpl.go

@@ -5,119 +5,145 @@
 package prometheus
 
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:1
-import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
 
 // TSDBStatusResponse generates response for /api/v1/status/tsdb .
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
-func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBStatus) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
-	qw422016.N().S(`{"status":"success","data":{"seriesCountByMetricName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
+func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBStatus, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
+	qw422016.N().S(`{"status":"success","data":{"totalSeries":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:12
+	qw422016.N().DUL(status.TotalSeries)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:12
+	qw422016.N().S(`,"totalLabelValuePairs":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:13
+	qw422016.N().DUL(status.TotalLabelValuePairs)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:13
+	qw422016.N().S(`,"seriesCountByMetricName":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
-	qw422016.N().S(`,"labelValueCountByLabelName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
-	streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	qw422016.N().S(`,"seriesCountByLabelName":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelName)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+	qw422016.N().S(`,"seriesCountByFocusLabelValue":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+	streamtsdbStatusEntries(qw422016, status.SeriesCountByFocusLabelValue)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
 	qw422016.N().S(`,"seriesCountByLabelValuePair":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
-	qw422016.N().S(`}}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
+	qw422016.N().S(`,"labelValueCountByLabelName":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+	streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
+	qt.Done()
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
-func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
-	StreamTSDBStatusResponse(qw422016, status)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+	StreamTSDBStatusResponse(qw422016, status, qt)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
-func TSDBStatusResponse(status *storage.TSDBStatus) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+func TSDBStatusResponse(status *storage.TSDBStatus, qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
-	WriteTSDBStatusResponse(qb422016, status)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+	WriteTSDBStatusResponse(qb422016, status, qt)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
 func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
 	qw422016.N().S(`[`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
 	for i, e := range a {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
 		qw422016.N().S(`{"name":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
 		qw422016.N().Q(e.Name)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
 		qw422016.N().S(`,"value":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
 		qw422016.N().D(int(e.Count))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
 		if i+1 < len(a) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
 		}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
 	}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
 	qw422016.N().S(`]`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	streamtsdbStatusEntries(qw422016, a)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 func tsdbStatusEntries(a []storage.TopHeapEntry) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	writetsdbStatusEntries(qb422016, a)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 }

app/vmselect/prometheus/util.qtpl

@@ -1,4 +1,5 @@
 {% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 ) %}
 
@@ -45,4 +46,9 @@
 ]
 {% endfunc %}
 
+{% func dumpQueryTrace(qt *querytracer.Tracer) %}
+	{% code	traceJSON := qt.ToJSON() %}
+	{% if traceJSON != "" %},"trace":{%s= traceJSON %}{% endif %}
+{% endfunc %}
+
 {% endstripspace %}

app/vmselect/prometheus/util.qtpl.go

@@ -6,212 +6,255 @@ package prometheus
 
 //line app/vmselect/prometheus/util.qtpl:1
 import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
-//line app/vmselect/prometheus/util.qtpl:7
+//line app/vmselect/prometheus/util.qtpl:8
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/util.qtpl:7
+//line app/vmselect/prometheus/util.qtpl:8
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/util.qtpl:7
+//line app/vmselect/prometheus/util.qtpl:8
 func streammetricNameObject(qw422016 *qt422016.Writer, mn *storage.MetricName) {
-//line app/vmselect/prometheus/util.qtpl:7
+//line app/vmselect/prometheus/util.qtpl:8
 	qw422016.N().S(`{`)
-//line app/vmselect/prometheus/util.qtpl:9
+//line app/vmselect/prometheus/util.qtpl:10
 	if len(mn.MetricGroup) > 0 {
-//line app/vmselect/prometheus/util.qtpl:9
+//line app/vmselect/prometheus/util.qtpl:10
 		qw422016.N().S(`"__name__":`)
-//line app/vmselect/prometheus/util.qtpl:10
-		qw422016.N().QZ(mn.MetricGroup)
-//line app/vmselect/prometheus/util.qtpl:10
-		if len(mn.Tags) > 0 {
-//line app/vmselect/prometheus/util.qtpl:10
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/util.qtpl:10
-		}
 //line app/vmselect/prometheus/util.qtpl:11
-	}
+		qw422016.N().QZ(mn.MetricGroup)
+//line app/vmselect/prometheus/util.qtpl:11
+		if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/util.qtpl:11
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/util.qtpl:11
+		}
 //line app/vmselect/prometheus/util.qtpl:12
-	for j := range mn.Tags {
+	}
 //line app/vmselect/prometheus/util.qtpl:13
+	for j := range mn.Tags {
+//line app/vmselect/prometheus/util.qtpl:14
 		tag := &mn.Tags[j]
 
-//line app/vmselect/prometheus/util.qtpl:14
+//line app/vmselect/prometheus/util.qtpl:15
 		qw422016.N().QZ(tag.Key)
-//line app/vmselect/prometheus/util.qtpl:14
+//line app/vmselect/prometheus/util.qtpl:15
 		qw422016.N().S(`:`)
-//line app/vmselect/prometheus/util.qtpl:14
+//line app/vmselect/prometheus/util.qtpl:15
 		qw422016.N().QZ(tag.Value)
-//line app/vmselect/prometheus/util.qtpl:14
+//line app/vmselect/prometheus/util.qtpl:15
 		if j+1 < len(mn.Tags) {
-//line app/vmselect/prometheus/util.qtpl:14
+//line app/vmselect/prometheus/util.qtpl:15
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/util.qtpl:14
+//line app/vmselect/prometheus/util.qtpl:15
 		}
-//line app/vmselect/prometheus/util.qtpl:15
+//line app/vmselect/prometheus/util.qtpl:16
 	}
-//line app/vmselect/prometheus/util.qtpl:15
+//line app/vmselect/prometheus/util.qtpl:16
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 }
 
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 func writemetricNameObject(qq422016 qtio422016.Writer, mn *storage.MetricName) {
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	streammetricNameObject(qw422016, mn)
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 }
 
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 func metricNameObject(mn *storage.MetricName) string {
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	writemetricNameObject(qb422016, mn)
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 	return qs422016
-//line app/vmselect/prometheus/util.qtpl:17
+//line app/vmselect/prometheus/util.qtpl:18
 }
 
-//line app/vmselect/prometheus/util.qtpl:19
+//line app/vmselect/prometheus/util.qtpl:20
 func streammetricRow(qw422016 *qt422016.Writer, timestamp int64, value float64) {
-//line app/vmselect/prometheus/util.qtpl:19
-	qw422016.N().S(`[`)
 //line app/vmselect/prometheus/util.qtpl:20
+	qw422016.N().S(`[`)
+//line app/vmselect/prometheus/util.qtpl:21
 	qw422016.N().F(float64(timestamp) / 1e3)
-//line app/vmselect/prometheus/util.qtpl:20
+//line app/vmselect/prometheus/util.qtpl:21
 	qw422016.N().S(`,"`)
-//line app/vmselect/prometheus/util.qtpl:20
+//line app/vmselect/prometheus/util.qtpl:21
 	qw422016.N().F(value)
-//line app/vmselect/prometheus/util.qtpl:20
+//line app/vmselect/prometheus/util.qtpl:21
 	qw422016.N().S(`"]`)
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 }
 
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 func writemetricRow(qq422016 qtio422016.Writer, timestamp int64, value float64) {
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	streammetricRow(qw422016, timestamp, value)
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 }
 
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 func metricRow(timestamp int64, value float64) string {
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	writemetricRow(qb422016, timestamp, value)
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 	return qs422016
-//line app/vmselect/prometheus/util.qtpl:21
+//line app/vmselect/prometheus/util.qtpl:22
 }
 
-//line app/vmselect/prometheus/util.qtpl:23
+//line app/vmselect/prometheus/util.qtpl:24
 func streamvaluesWithTimestamps(qw422016 *qt422016.Writer, values []float64, timestamps []int64) {
-//line app/vmselect/prometheus/util.qtpl:24
+//line app/vmselect/prometheus/util.qtpl:25
 	if len(values) == 0 {
-//line app/vmselect/prometheus/util.qtpl:24
+//line app/vmselect/prometheus/util.qtpl:25
 		qw422016.N().S(`[]`)
-//line app/vmselect/prometheus/util.qtpl:26
+//line app/vmselect/prometheus/util.qtpl:27
 		return
-//line app/vmselect/prometheus/util.qtpl:27
+//line app/vmselect/prometheus/util.qtpl:28
 	}
-//line app/vmselect/prometheus/util.qtpl:27
+//line app/vmselect/prometheus/util.qtpl:28
 	qw422016.N().S(`[`)
-//line app/vmselect/prometheus/util.qtpl:29
+//line app/vmselect/prometheus/util.qtpl:30
 	/* inline metricRow call here for the sake of performance optimization */
 
-//line app/vmselect/prometheus/util.qtpl:29
+//line app/vmselect/prometheus/util.qtpl:30
 	qw422016.N().S(`[`)
-//line app/vmselect/prometheus/util.qtpl:30
+//line app/vmselect/prometheus/util.qtpl:31
 	qw422016.N().F(float64(timestamps[0]) / 1e3)
-//line app/vmselect/prometheus/util.qtpl:30
+//line app/vmselect/prometheus/util.qtpl:31
 	qw422016.N().S(`,"`)
-//line app/vmselect/prometheus/util.qtpl:30
+//line app/vmselect/prometheus/util.qtpl:31
 	qw422016.N().F(values[0])
-//line app/vmselect/prometheus/util.qtpl:30
+//line app/vmselect/prometheus/util.qtpl:31
 	qw422016.N().S(`"]`)
-//line app/vmselect/prometheus/util.qtpl:32
+//line app/vmselect/prometheus/util.qtpl:33
 	timestamps = timestamps[1:]
 	values = values[1:]
 
-//line app/vmselect/prometheus/util.qtpl:35
+//line app/vmselect/prometheus/util.qtpl:36
 	if len(values) > 0 {
-//line app/vmselect/prometheus/util.qtpl:37
+//line app/vmselect/prometheus/util.qtpl:38
 		// Remove bounds check inside the loop below
 		_ = timestamps[len(values)-1]
 
-//line app/vmselect/prometheus/util.qtpl:40
-		for i, v := range values {
 //line app/vmselect/prometheus/util.qtpl:41
+		for i, v := range values {
+//line app/vmselect/prometheus/util.qtpl:42
 			/* inline metricRow call here for the sake of performance optimization */
 
-//line app/vmselect/prometheus/util.qtpl:41
+//line app/vmselect/prometheus/util.qtpl:42
 			qw422016.N().S(`,[`)
-//line app/vmselect/prometheus/util.qtpl:42
-			qw422016.N().F(float64(timestamps[i]) / 1e3)
-//line app/vmselect/prometheus/util.qtpl:42
-			qw422016.N().S(`,"`)
-//line app/vmselect/prometheus/util.qtpl:42
-			qw422016.N().F(v)
-//line app/vmselect/prometheus/util.qtpl:42
-			qw422016.N().S(`"]`)
 //line app/vmselect/prometheus/util.qtpl:43
+			qw422016.N().F(float64(timestamps[i]) / 1e3)
+//line app/vmselect/prometheus/util.qtpl:43
+			qw422016.N().S(`,"`)
+//line app/vmselect/prometheus/util.qtpl:43
+			qw422016.N().F(v)
+//line app/vmselect/prometheus/util.qtpl:43
+			qw422016.N().S(`"]`)
+//line app/vmselect/prometheus/util.qtpl:44
 		}
-//line app/vmselect/prometheus/util.qtpl:44
+//line app/vmselect/prometheus/util.qtpl:45
 	}
-//line app/vmselect/prometheus/util.qtpl:44
+//line app/vmselect/prometheus/util.qtpl:45
 	qw422016.N().S(`]`)
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 }
 
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 func writevaluesWithTimestamps(qq422016 qtio422016.Writer, values []float64, timestamps []int64) {
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	streamvaluesWithTimestamps(qw422016, values, timestamps)
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 }
 
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 func valuesWithTimestamps(values []float64, timestamps []int64) string {
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	writevaluesWithTimestamps(qb422016, values, timestamps)
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
 	return qs422016
-//line app/vmselect/prometheus/util.qtpl:46
+//line app/vmselect/prometheus/util.qtpl:47
+}
+
+//line app/vmselect/prometheus/util.qtpl:49
+func streamdumpQueryTrace(qw422016 *qt422016.Writer, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/util.qtpl:50
+	traceJSON := qt.ToJSON()
+
+//line app/vmselect/prometheus/util.qtpl:51
+	if traceJSON != "" {
+//line app/vmselect/prometheus/util.qtpl:51
+		qw422016.N().S(`,"trace":`)
+//line app/vmselect/prometheus/util.qtpl:51
+		qw422016.N().S(traceJSON)
+//line app/vmselect/prometheus/util.qtpl:51
+	}
+//line app/vmselect/prometheus/util.qtpl:52
+}
+
+//line app/vmselect/prometheus/util.qtpl:52
+func writedumpQueryTrace(qq422016 qtio422016.Writer, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/util.qtpl:52
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/util.qtpl:52
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/util.qtpl:52
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/util.qtpl:52
+}
+
+//line app/vmselect/prometheus/util.qtpl:52
+func dumpQueryTrace(qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/util.qtpl:52
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/util.qtpl:52
+	writedumpQueryTrace(qb422016, qt)
+//line app/vmselect/prometheus/util.qtpl:52
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/util.qtpl:52
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/util.qtpl:52
+	return qs422016
+//line app/vmselect/prometheus/util.qtpl:52
 }

app/vmselect/promql/eval.go

@@ -16,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/VictoriaMetrics/metricsql"
@@ -102,6 +103,7 @@ type EvalConfig struct {
 
 	Deadline searchutils.Deadline
 
+	// Whether the response can be cached.
 	MayCache bool
 
 	// LookbackDelta is analog to `-query.lookback-delta` from Prometheus.
@@ -190,19 +192,41 @@ func getTimestamps(start, end, step int64) []int64 {
 	return timestamps
 }
 
-func evalExpr(ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) {
+func evalExpr(qt *querytracer.Tracer, ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) {
+	if qt.Enabled() {
+		query := e.AppendString(nil)
+		mayCache := ec.mayCache()
+		qt = qt.NewChild("eval: query=%s, timeRange=[%d..%d], step=%d, mayCache=%v", query, ec.Start, ec.End, ec.Step, mayCache)
+	}
+	rv, err := evalExprInternal(qt, ec, e)
+	if err != nil {
+		return nil, err
+	}
+	if qt.Enabled() {
+		seriesCount := len(rv)
+		pointsPerSeries := 0
+		if len(rv) > 0 {
+			pointsPerSeries = len(rv[0].Timestamps)
+		}
+		pointsCount := seriesCount * pointsPerSeries
+		qt.Donef("series=%d, points=%d, pointsPerSeries=%d", seriesCount, pointsCount, pointsPerSeries)
+	}
+	return rv, nil
+}
+
+func evalExprInternal(qt *querytracer.Tracer, ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) {
 	if me, ok := e.(*metricsql.MetricExpr); ok {
 		re := &metricsql.RollupExpr{
 			Expr: me,
 		}
-		rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, e, re, nil)
+		rv, err := evalRollupFunc(qt, ec, "default_rollup", rollupDefault, e, re, nil)
 		if err != nil {
 			return nil, fmt.Errorf(`cannot evaluate %q: %w`, me.AppendString(nil), err)
 		}
 		return rv, nil
 	}
 	if re, ok := e.(*metricsql.RollupExpr); ok {
-		rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, e, re, nil)
+		rv, err := evalRollupFunc(qt, ec, "default_rollup", rollupDefault, e, re, nil)
 		if err != nil {
 			return nil, fmt.Errorf(`cannot evaluate %q: %w`, re.AppendString(nil), err)
 		}
@@ -211,26 +235,12 @@ func evalExpr(ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) {
 	if fe, ok := e.(*metricsql.FuncExpr); ok {
 		nrf := getRollupFunc(fe.Name)
 		if nrf == nil {
-			args, err := evalExprs(ec, fe.Args)
-			if err != nil {
-				return nil, err
-			}
-			tf := getTransformFunc(fe.Name)
-			if tf == nil {
-				return nil, fmt.Errorf(`unknown func %q`, fe.Name)
-			}
-			tfa := &transformFuncArg{
-				ec:   ec,
-				fe:   fe,
-				args: args,
-			}
-			rv, err := tf(tfa)
-			if err != nil {
-				return nil, fmt.Errorf(`cannot evaluate %q: %w`, fe.AppendString(nil), err)
-			}
-			return rv, nil
+			qtChild := qt.NewChild("transform %s()", fe.Name)
+			rv, err := evalTransformFunc(qtChild, ec, fe)
+			qtChild.Donef("series=%d", len(rv))
+			return rv, err
 		}
-		args, re, err := evalRollupFuncArgs(ec, fe)
+		args, re, err := evalRollupFuncArgs(qt, ec, fe)
 		if err != nil {
 			return nil, err
 		}
@@ -238,79 +248,23 @@ func evalExpr(ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) {
 		if err != nil {
 			return nil, err
 		}
-		rv, err := evalRollupFunc(ec, fe.Name, rf, e, re, nil)
+		rv, err := evalRollupFunc(qt, ec, fe.Name, rf, e, re, nil)
 		if err != nil {
 			return nil, fmt.Errorf(`cannot evaluate %q: %w`, fe.AppendString(nil), err)
 		}
 		return rv, nil
 	}
 	if ae, ok := e.(*metricsql.AggrFuncExpr); ok {
-		if callbacks := getIncrementalAggrFuncCallbacks(ae.Name); callbacks != nil {
-			fe, nrf := tryGetArgRollupFuncWithMetricExpr(ae)
-			if fe != nil {
-				// There is an optimized path for calculating metricsql.AggrFuncExpr over rollupFunc over metricsql.MetricExpr.
-				// The optimized path saves RAM for aggregates over big number of time series.
-				args, re, err := evalRollupFuncArgs(ec, fe)
-				if err != nil {
-					return nil, err
-				}
-				rf, err := nrf(args)
-				if err != nil {
-					return nil, err
-				}
-				iafc := newIncrementalAggrFuncContext(ae, callbacks)
-				return evalRollupFunc(ec, fe.Name, rf, e, re, iafc)
-			}
-		}
-		args, err := evalExprs(ec, ae.Args)
-		if err != nil {
-			return nil, err
-		}
-		af := getAggrFunc(ae.Name)
-		if af == nil {
-			return nil, fmt.Errorf(`unknown func %q`, ae.Name)
-		}
-		afa := &aggrFuncArg{
-			ae:   ae,
-			args: args,
-			ec:   ec,
-		}
-		rv, err := af(afa)
-		if err != nil {
-			return nil, fmt.Errorf(`cannot evaluate %q: %w`, ae.AppendString(nil), err)
-		}
-		return rv, nil
+		qtChild := qt.NewChild("aggregate %s()", ae.Name)
+		rv, err := evalAggrFunc(qtChild, ec, ae)
+		qtChild.Donef("series=%d", len(rv))
+		return rv, err
 	}
 	if be, ok := e.(*metricsql.BinaryOpExpr); ok {
-		bf := getBinaryOpFunc(be.Op)
-		if bf == nil {
-			return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
-		}
-		var err error
-		var tssLeft, tssRight []*timeseries
-		switch strings.ToLower(be.Op) {
-		case "and", "if":
-			// Fetch right-side series at first, since it usually contains
-			// lower number of time series for `and` and `if` operator.
-			// This should produce more specific label filters for the left side of the query.
-			// This, in turn, should reduce the time to select series for the left side of the query.
-			tssRight, tssLeft, err = execBinaryOpArgs(ec, be.Right, be.Left, be)
-		default:
-			tssLeft, tssRight, err = execBinaryOpArgs(ec, be.Left, be.Right, be)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
-		}
-		bfa := &binaryOpFuncArg{
-			be:    be,
-			left:  tssLeft,
-			right: tssRight,
-		}
-		rv, err := bf(bfa)
-		if err != nil {
-			return nil, fmt.Errorf(`cannot evaluate %q: %w`, be.AppendString(nil), err)
-		}
-		return rv, nil
+		qtChild := qt.NewChild("binary op %q", be.Op)
+		rv, err := evalBinaryOp(qtChild, ec, be)
+		qtChild.Donef("series=%d", len(rv))
+		return rv, err
 	}
 	if ne, ok := e.(*metricsql.NumberExpr); ok {
 		rv := evalNumber(ec, ne.N)
@@ -329,7 +283,98 @@ func evalExpr(ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) {
 	return nil, fmt.Errorf("unexpected expression %q", e.AppendString(nil))
 }
 
-func execBinaryOpArgs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+func evalTransformFunc(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.FuncExpr) ([]*timeseries, error) {
+	args, err := evalExprs(qt, ec, fe.Args)
+	if err != nil {
+		return nil, err
+	}
+	tf := getTransformFunc(fe.Name)
+	if tf == nil {
+		return nil, fmt.Errorf(`unknown func %q`, fe.Name)
+	}
+	tfa := &transformFuncArg{
+		ec:   ec,
+		fe:   fe,
+		args: args,
+	}
+	rv, err := tf(tfa)
+	if err != nil {
+		return nil, fmt.Errorf(`cannot evaluate %q: %w`, fe.AppendString(nil), err)
+	}
+	return rv, nil
+}
+
+func evalAggrFunc(qt *querytracer.Tracer, ec *EvalConfig, ae *metricsql.AggrFuncExpr) ([]*timeseries, error) {
+	if callbacks := getIncrementalAggrFuncCallbacks(ae.Name); callbacks != nil {
+		fe, nrf := tryGetArgRollupFuncWithMetricExpr(ae)
+		if fe != nil {
+			// There is an optimized path for calculating metricsql.AggrFuncExpr over rollupFunc over metricsql.MetricExpr.
+			// The optimized path saves RAM for aggregates over big number of time series.
+			args, re, err := evalRollupFuncArgs(qt, ec, fe)
+			if err != nil {
+				return nil, err
+			}
+			rf, err := nrf(args)
+			if err != nil {
+				return nil, err
+			}
+			iafc := newIncrementalAggrFuncContext(ae, callbacks)
+			return evalRollupFunc(qt, ec, fe.Name, rf, ae, re, iafc)
+		}
+	}
+	args, err := evalExprs(qt, ec, ae.Args)
+	if err != nil {
+		return nil, err
+	}
+	af := getAggrFunc(ae.Name)
+	if af == nil {
+		return nil, fmt.Errorf(`unknown func %q`, ae.Name)
+	}
+	afa := &aggrFuncArg{
+		ae:   ae,
+		args: args,
+		ec:   ec,
+	}
+	rv, err := af(afa)
+	if err != nil {
+		return nil, fmt.Errorf(`cannot evaluate %q: %w`, ae.AppendString(nil), err)
+	}
+	return rv, nil
+}
+
+func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, error) {
+	bf := getBinaryOpFunc(be.Op)
+	if bf == nil {
+		return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
+	}
+	var err error
+	var tssLeft, tssRight []*timeseries
+	switch strings.ToLower(be.Op) {
+	case "and", "if":
+		// Fetch right-side series at first, since it usually contains
+		// lower number of time series for `and` and `if` operator.
+		// This should produce more specific label filters for the left side of the query.
+		// This, in turn, should reduce the time to select series for the left side of the query.
+		tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
+	default:
+		tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
+	}
+	bfa := &binaryOpFuncArg{
+		be:    be,
+		left:  tssLeft,
+		right: tssRight,
+	}
+	rv, err := bf(bfa)
+	if err != nil {
+		return nil, fmt.Errorf(`cannot evaluate %q: %w`, be.AppendString(nil), err)
+	}
+	return rv, nil
+}
+
+func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
 	// Execute binary operation in the following way:
 	//
 	// 1) execute the exprFirst
@@ -353,7 +398,7 @@ func execBinaryOpArgs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *
 	//
 	// - Queries, which get additional labels from `info` metrics.
 	//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
-	tssFirst, err := evalExpr(ec, exprFirst)
+	tssFirst, err := evalExpr(qt, ec, exprFirst)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -366,7 +411,7 @@ func execBinaryOpArgs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *
 		lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
 		exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
 	}
-	tssSecond, err := evalExpr(ec, exprSecond)
+	tssSecond, err := evalExpr(qt, ec, exprSecond)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -503,10 +548,10 @@ func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.F
 	return nil, nil
 }
 
-func evalExprs(ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
+func evalExprs(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
 	var rvs [][]*timeseries
 	for _, e := range es {
-		rv, err := evalExpr(ec, e)
+		rv, err := evalExpr(qt, ec, e)
 		if err != nil {
 			return nil, err
 		}
@@ -515,7 +560,7 @@ func evalExprs(ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
 	return rvs, nil
 }
 
-func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
+func evalRollupFuncArgs(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
 	var re *metricsql.RollupExpr
 	rollupArgIdx := metricsql.GetRollupArgIdx(fe)
 	if len(fe.Args) <= rollupArgIdx {
@@ -528,7 +573,7 @@ func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{},
 			args[i] = re
 			continue
 		}
-		ts, err := evalExpr(ec, arg)
+		ts, err := evalExpr(qt, ec, arg)
 		if err != nil {
 			return nil, nil, fmt.Errorf("cannot evaluate arg #%d for %q: %w", i+1, fe.AppendString(nil), err)
 		}
@@ -568,11 +613,12 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 // expr may contain:
 // - rollupFunc(m) if iafc is nil
 // - aggrFunc(rollupFunc(m)) if iafc isn't nil
-func evalRollupFunc(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr,
+	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
 	if re.At == nil {
-		return evalRollupFuncWithoutAt(ec, funcName, rf, expr, re, iafc)
+		return evalRollupFuncWithoutAt(qt, ec, funcName, rf, expr, re, iafc)
 	}
-	tssAt, err := evalExpr(ec, re.At)
+	tssAt, err := evalExpr(qt, ec, re.At)
 	if err != nil {
 		return nil, fmt.Errorf("cannot evaluate `@` modifier: %w", err)
 	}
@@ -583,7 +629,7 @@ func evalRollupFunc(ec *EvalConfig, funcName string, rf rollupFunc, expr metrics
 	ecNew := copyEvalConfig(ec)
 	ecNew.Start = atTimestamp
 	ecNew.End = atTimestamp
-	tss, err := evalRollupFuncWithoutAt(ecNew, funcName, rf, expr, re, iafc)
+	tss, err := evalRollupFuncWithoutAt(qt, ecNew, funcName, rf, expr, re, iafc)
 	if err != nil {
 		return nil, err
 	}
@@ -601,7 +647,8 @@ func evalRollupFunc(ec *EvalConfig, funcName string, rf rollupFunc, expr metrics
 	return tss, nil
 }
 
-func evalRollupFuncWithoutAt(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+func evalRollupFuncWithoutAt(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
+	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
 	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
@@ -628,12 +675,12 @@ func evalRollupFuncWithoutAt(ec *EvalConfig, funcName string, rf rollupFunc, exp
 	var rvs []*timeseries
 	var err error
 	if me, ok := re.Expr.(*metricsql.MetricExpr); ok {
-		rvs, err = evalRollupFuncWithMetricExpr(ecNew, funcName, rf, expr, me, iafc, re.Window)
+		rvs, err = evalRollupFuncWithMetricExpr(qt, ecNew, funcName, rf, expr, me, iafc, re.Window)
 	} else {
 		if iafc != nil {
 			logger.Panicf("BUG: iafc must be nil for rollup %q over subquery %q", funcName, re.AppendString(nil))
 		}
-		rvs, err = evalRollupFuncWithSubquery(ecNew, funcName, rf, expr, re)
+		rvs, err = evalRollupFuncWithSubquery(qt, ecNew, funcName, rf, expr, re)
 	}
 	if err != nil {
 		return nil, err
@@ -676,8 +723,10 @@ func aggregateAbsentOverTime(ec *EvalConfig, expr metricsql.Expr, tss []*timeser
 	return rvs
 }
 
-func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr) ([]*timeseries, error) {
+func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr) ([]*timeseries, error) {
 	// TODO: determine whether to use rollupResultCacheV here.
+	qt = qt.NewChild("subquery")
+	defer qt.Done()
 	step := re.Step.Duration(ec.Step)
 	if step == 0 {
 		step = ec.Step
@@ -693,7 +742,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
 	}
 	// unconditionally align start and end args to step for subquery as Prometheus does.
 	ecSQ.Start, ecSQ.End = alignStartEnd(ecSQ.Start, ecSQ.End, ecSQ.Step)
-	tssSQ, err := evalExpr(ecSQ, re.Expr)
+	tssSQ, err := evalExpr(qt, ecSQ, re.Expr)
 	if err != nil {
 		return nil, err
 	}
@@ -727,6 +776,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
 		}
 		return values, timestamps
 	})
+	qt.Printf("rollup %s() over %d series returned by subquery: series=%d", funcName, len(tssSQ), len(tss))
 	return tss, nil
 }
 
@@ -802,15 +852,20 @@ var (
 	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
 )
 
-func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc,
+func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
 	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
+	var rollupMemorySize int64
+	window := windowExpr.Duration(ec.Step)
+	qt = qt.NewChild("rollup %s(): timeRange=[%d..%d], step=%d, window=%d", funcName, ec.Start, ec.End, ec.Step, window)
+	defer func() {
+		qt.Donef("neededMemoryBytes=%d", rollupMemorySize)
+	}()
 	if me.IsEmpty() {
 		return evalNumber(ec, nan), nil
 	}
-	window := windowExpr.Duration(ec.Step)
 
 	// Search for partial results in cache.
-	tssCached, start := rollupResultCacheV.Get(ec, expr, window)
+	tssCached, start := rollupResultCacheV.Get(qt, ec, expr, window)
 	if start > ec.End {
 		// The result is fully cached.
 		rollupResultCacheFullHits.Inc()
@@ -840,7 +895,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc
 		minTimestamp -= ec.Step
 	}
 	sq := storage.NewSearchQuery(minTimestamp, ec.End, tfss, ec.MaxSeries)
-	rss, err := netstorage.ProcessSearchQuery(sq, true, ec.Deadline)
+	rss, err := netstorage.ProcessSearchQuery(qt, sq, true, ec.Deadline)
 	if err != nil {
 		return nil, err
 	}
@@ -874,7 +929,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc
 		}
 	}
 	rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(timeseriesLen*len(rcs)))
-	rollupMemorySize := mulNoOverflow(rollupPoints, 16)
+	rollupMemorySize = mulNoOverflow(rollupPoints, 16)
 	rml := getRollupMemoryLimiter()
 	if !rml.Get(uint64(rollupMemorySize)) {
 		rss.Cancel()
@@ -891,15 +946,15 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc
 	keepMetricNames := getKeepMetricNames(expr)
 	var tss []*timeseries
 	if iafc != nil {
-		tss, err = evalRollupWithIncrementalAggregate(funcName, keepMetricNames, iafc, rss, rcs, preFunc, sharedTimestamps)
+		tss, err = evalRollupWithIncrementalAggregate(qt, funcName, keepMetricNames, iafc, rss, rcs, preFunc, sharedTimestamps)
 	} else {
-		tss, err = evalRollupNoIncrementalAggregate(funcName, keepMetricNames, rss, rcs, preFunc, sharedTimestamps)
+		tss, err = evalRollupNoIncrementalAggregate(qt, funcName, keepMetricNames, rss, rcs, preFunc, sharedTimestamps)
 	}
 	if err != nil {
 		return nil, err
 	}
 	tss = mergeTimeseries(tssCached, tss, start, ec)
-	rollupResultCacheV.Put(ec, expr, window, tss)
+	rollupResultCacheV.Put(qt, ec, expr, window, tss)
 	return tss, nil
 }
 
@@ -915,9 +970,12 @@ func getRollupMemoryLimiter() *memoryLimiter {
 	return &rollupMemoryLimiter
 }
 
-func evalRollupWithIncrementalAggregate(funcName string, keepMetricNames bool, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
+func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
+	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
-	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series", funcName, iafc.ae.Name, rss.Len())
+	defer qt.Done()
+	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
 		ts := getTimeseries()
@@ -944,14 +1002,17 @@ func evalRollupWithIncrementalAggregate(funcName string, keepMetricNames bool, i
 		return nil, err
 	}
 	tss := iafc.finalizeTimeseries()
+	qt.Printf("series after aggregation with %s(): %d", iafc.ae.Name, len(tss))
 	return tss, nil
 }
 
-func evalRollupNoIncrementalAggregate(funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
+func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	qt = qt.NewChild("rollup %s() over %d series", funcName, rss.Len())
+	defer qt.Done()
 	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
 	var tssLock sync.Mutex
-	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {

app/vmselect/promql/exec.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/VictoriaMetrics/metricsql"
@@ -26,7 +27,7 @@ var (
 )
 
 // Exec executes q for the given ec.
-func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result, error) {
+func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result, error) {
 	if querystats.Enabled() {
 		startTime := time.Now()
 		defer querystats.RegisterQuery(q, ec.End-ec.Start, startTime)
@@ -40,25 +41,29 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
 	}
 
 	qid := activeQueriesV.Add(ec, q)
-	rv, err := evalExpr(ec, e)
+	rv, err := evalExpr(qt, ec, e)
 	activeQueriesV.Remove(qid)
 	if err != nil {
 		return nil, err
 	}
-
 	if isFirstPointOnly {
 		// Remove all the points except the first one from every time series.
 		for _, ts := range rv {
 			ts.Values = ts.Values[:1]
 			ts.Timestamps = ts.Timestamps[:1]
 		}
+		qt.Printf("leave only the first point in every series")
 	}
-
 	maySort := maySortResults(e, rv)
 	result, err := timeseriesToResult(rv, maySort)
 	if err != nil {
 		return nil, err
 	}
+	if maySort {
+		qt.Printf("sort series by metric name and labels")
+	} else {
+		qt.Printf("do not sort series by metric name and labels")
+	}
 	if n := ec.RoundDigits; n < 100 {
 		for i := range result {
 			values := result[i].Values
@@ -66,6 +71,7 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
 				values[j] = decimal.RoundToDecimalDigits(v, n)
 			}
 		}
+		qt.Printf("round series values to %d decimal digits after the point", n)
 	}
 	return result, err
 }

app/vmselect/promql/exec_test.go

@@ -66,7 +66,7 @@ func TestExecSuccess(t *testing.T) {
 			RoundDigits: 100,
 		}
 		for i := 0; i < 5; i++ {
-			result, err := Exec(ec, q, false)
+			result, err := Exec(nil, ec, q, false)
 			if err != nil {
 				t.Fatalf(`unexpected error when executing %q: %s`, q, err)
 			}
@@ -7728,14 +7728,14 @@ func TestExecError(t *testing.T) {
 			RoundDigits: 100,
 		}
 		for i := 0; i < 4; i++ {
-			rv, err := Exec(ec, q, false)
+			rv, err := Exec(nil, ec, q, false)
 			if err == nil {
 				t.Fatalf(`expecting non-nil error on %q`, q)
 			}
 			if rv != nil {
 				t.Fatalf(`expecting nil rv`)
 			}
-			rv, err = Exec(ec, q, true)
+			rv, err = Exec(nil, ec, q, true)
 			if err == nil {
 				t.Fatalf(`expecting non-nil error on %q`, q)
 			}

app/vmselect/promql/rollup_result_cache.go

@@ -4,6 +4,7 @@ import (
 	"crypto/rand"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -11,8 +12,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 	"github.com/VictoriaMetrics/fastcache"
@@ -110,8 +113,10 @@ func InitRollupResultCache(cachePath string) {
 	if len(rollupResultCachePath) > 0 {
 		logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
 		c = workingsetcache.Load(rollupResultCachePath, cacheSize)
+		mustLoadRollupResultCacheKeyPrefix(rollupResultCachePath)
 	} else {
 		c = workingsetcache.New(cacheSize)
+		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
 	}
 	if *disableCache {
 		c.Reset()
@@ -169,9 +174,10 @@ func StopRollupResultCache() {
 	logger.Infof("saving rollupResult cache to %q...", rollupResultCachePath)
 	startTime := time.Now()
 	if err := rollupResultCacheV.c.Save(rollupResultCachePath); err != nil {
-		logger.Errorf("cannot close rollupResult cache at %q: %s", rollupResultCachePath, err)
+		logger.Errorf("cannot save rollupResult cache at %q: %s", rollupResultCachePath, err)
 		return
 	}
+	mustSaveRollupResultCacheKeyPrefix(rollupResultCachePath)
 	var fcs fastcache.Stats
 	rollupResultCacheV.c.UpdateStats(&fcs)
 	rollupResultCacheV.c.Stop()
@@ -193,8 +199,14 @@ func ResetRollupResultCache() {
 	logger.Infof("rollupResult cache has been cleared")
 }
 
-func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window int64) (tss []*timeseries, newStart int64) {
+func (rrc *rollupResultCache) Get(qt *querytracer.Tracer, ec *EvalConfig, expr metricsql.Expr, window int64) (tss []*timeseries, newStart int64) {
+	if qt.Enabled() {
+		query := expr.AppendString(nil)
+		qt = qt.NewChild("rollup cache get: query=%s, timeRange=[%d..%d], step=%d, window=%d", query, ec.Start, ec.End, ec.Step, window)
+		defer qt.Done()
+	}
 	if !ec.mayCache() {
+		qt.Printf("do not fetch series from cache, since it is disabled in the current context")
 		return nil, ec.Start
 	}
 
@@ -205,6 +217,7 @@ func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window in
 	bb.B = marshalRollupResultCacheKey(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
 	metainfoBuf := rrc.c.Get(nil, bb.B)
 	if len(metainfoBuf) == 0 {
+		qt.Printf("nothing found")
 		return nil, ec.Start
 	}
 	var mi rollupResultCacheMetainfo
@@ -213,6 +226,7 @@ func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window in
 	}
 	key := mi.GetBestKey(ec.Start, ec.End)
 	if key.prefix == 0 && key.suffix == 0 {
+		qt.Printf("nothing found on the timeRange")
 		return nil, ec.Start
 	}
 	bb.B = key.Marshal(bb.B[:0])
@@ -224,18 +238,22 @@ func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window in
 		metainfoBuf = mi.Marshal(metainfoBuf[:0])
 		bb.B = marshalRollupResultCacheKey(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
 		rrc.c.Set(bb.B, metainfoBuf)
+		qt.Printf("missing cache entry")
 		return nil, ec.Start
 	}
 	// Decompress into newly allocated byte slice, since tss returned from unmarshalTimeseriesFast
 	// refers to the byte slice, so it cannot be returned to the resultBufPool.
+	qt.Printf("load compressed entry from cache with size %d bytes", len(compressedResultBuf.B))
 	resultBuf, err := encoding.DecompressZSTD(nil, compressedResultBuf.B)
 	if err != nil {
 		logger.Panicf("BUG: cannot decompress resultBuf from rollupResultCache: %s; it looks like it was improperly saved", err)
 	}
+	qt.Printf("unpack the entry into %d bytes", len(resultBuf))
 	tss, err = unmarshalTimeseriesFast(resultBuf)
 	if err != nil {
 		logger.Panicf("BUG: cannot unmarshal timeseries from rollupResultCache: %s; it looks like it was improperly saved", err)
 	}
+	qt.Printf("unmarshal %d series", len(tss))
 
 	// Extract values for the matching timestamps
 	timestamps := tss[0].Timestamps
@@ -245,10 +263,12 @@ func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window in
 	}
 	if i == len(timestamps) {
 		// no matches.
+		qt.Printf("no datapoints found in the cached series on the given timeRange")
 		return nil, ec.Start
 	}
 	if timestamps[i] != ec.Start {
 		// The cached range doesn't cover the requested range.
+		qt.Printf("cached series don't cover the given timeRange")
 		return nil, ec.Start
 	}
 
@@ -269,13 +289,20 @@ func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window in
 
 	timestamps = tss[0].Timestamps
 	newStart = timestamps[len(timestamps)-1] + ec.Step
+	qt.Printf("return %d series on a timeRange=[%d..%d]", len(tss), ec.Start, newStart-ec.Step)
 	return tss, newStart
 }
 
 var resultBufPool bytesutil.ByteBufferPool
 
-func (rrc *rollupResultCache) Put(ec *EvalConfig, expr metricsql.Expr, window int64, tss []*timeseries) {
+func (rrc *rollupResultCache) Put(qt *querytracer.Tracer, ec *EvalConfig, expr metricsql.Expr, window int64, tss []*timeseries) {
+	if qt.Enabled() {
+		query := expr.AppendString(nil)
+		qt = qt.NewChild("rollup cache put: query=%s, timeRange=[%d..%d], step=%d, window=%d, series=%d", query, ec.Start, ec.End, ec.Step, window, len(tss))
+		defer qt.Done()
+	}
 	if len(tss) == 0 || !ec.mayCache() {
+		qt.Printf("do not store series to cache, since it is disabled in the current context")
 		return
 	}
 
@@ -290,6 +317,7 @@ func (rrc *rollupResultCache) Put(ec *EvalConfig, expr metricsql.Expr, window in
 	i++
 	if i == 0 {
 		// Nothing to store in the cache.
+		qt.Printf("nothing to store in the cache, since all the points have timestamps bigger than %d", deadline)
 		return
 	}
 	if i < len(timestamps) {
@@ -304,52 +332,96 @@ func (rrc *rollupResultCache) Put(ec *EvalConfig, expr metricsql.Expr, window in
 	}
 
 	// Store tss in the cache.
+	metainfoKey := bbPool.Get()
+	defer bbPool.Put(metainfoKey)
+	metainfoBuf := bbPool.Get()
+	defer bbPool.Put(metainfoBuf)
+
+	metainfoKey.B = marshalRollupResultCacheKey(metainfoKey.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
+	metainfoBuf.B = rrc.c.Get(metainfoBuf.B[:0], metainfoKey.B)
+	var mi rollupResultCacheMetainfo
+	if len(metainfoBuf.B) > 0 {
+		if err := mi.Unmarshal(metainfoBuf.B); err != nil {
+			logger.Panicf("BUG: cannot unmarshal rollupResultCacheMetainfo: %s; it looks like it was improperly saved", err)
+		}
+	}
+	start := timestamps[0]
+	end := timestamps[len(timestamps)-1]
+	if mi.CoversTimeRange(start, end) {
+		qt.Printf("series on the given timeRange=[%d..%d] already exist in the cache", start, end)
+		return
+	}
+
 	maxMarshaledSize := getRollupResultCacheSize() / 4
 	resultBuf := resultBufPool.Get()
 	defer resultBufPool.Put(resultBuf)
 	resultBuf.B = marshalTimeseriesFast(resultBuf.B[:0], tss, maxMarshaledSize, ec.Step)
 	if len(resultBuf.B) == 0 {
 		tooBigRollupResults.Inc()
+		qt.Printf("cannot store series in the cache, since they would occupy more than %d bytes", maxMarshaledSize)
 		return
 	}
+	qt.Printf("marshal %d series on a timeRange=[%d..%d] into %d bytes", len(tss), start, end, len(resultBuf.B))
 	compressedResultBuf := resultBufPool.Get()
 	defer resultBufPool.Put(compressedResultBuf)
 	compressedResultBuf.B = encoding.CompressZSTDLevel(compressedResultBuf.B[:0], resultBuf.B, 1)
-
-	bb := bbPool.Get()
-	defer bbPool.Put(bb)
+	qt.Printf("compress %d bytes into %d bytes", len(resultBuf.B), len(compressedResultBuf.B))
 
 	var key rollupResultCacheKey
 	key.prefix = rollupResultCacheKeyPrefix
 	key.suffix = atomic.AddUint64(&rollupResultCacheKeySuffix, 1)
-	bb.B = key.Marshal(bb.B[:0])
-	rrc.c.SetBig(bb.B, compressedResultBuf.B)
+	rollupResultKey := key.Marshal(nil)
+	rrc.c.SetBig(rollupResultKey, compressedResultBuf.B)
+	qt.Printf("store %d bytes in the cache", len(compressedResultBuf.B))
 
-	bb.B = marshalRollupResultCacheKey(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
-	metainfoBuf := rrc.c.Get(nil, bb.B)
-	var mi rollupResultCacheMetainfo
-	if len(metainfoBuf) > 0 {
-		if err := mi.Unmarshal(metainfoBuf); err != nil {
-			logger.Panicf("BUG: cannot unmarshal rollupResultCacheMetainfo: %s; it looks like it was improperly saved", err)
-		}
-	}
 	mi.AddKey(key, timestamps[0], timestamps[len(timestamps)-1])
-	metainfoBuf = mi.Marshal(metainfoBuf[:0])
-	rrc.c.Set(bb.B, metainfoBuf)
+	metainfoBuf.B = mi.Marshal(metainfoBuf.B[:0])
+	rrc.c.Set(metainfoKey.B, metainfoBuf.B)
 }
 
 var (
-	rollupResultCacheKeyPrefix = func() uint64 {
-		var buf [8]byte
-		if _, err := rand.Read(buf[:]); err != nil {
-			// do not use logger.Panicf, since it isn't initialized yet.
-			panic(fmt.Errorf("FATAL: cannot read random data for rollupResultCacheKeyPrefix: %w", err))
-		}
-		return encoding.UnmarshalUint64(buf[:])
-	}()
+	rollupResultCacheKeyPrefix uint64
 	rollupResultCacheKeySuffix = uint64(time.Now().UnixNano())
 )
 
+func newRollupResultCacheKeyPrefix() uint64 {
+	var buf [8]byte
+	if _, err := rand.Read(buf[:]); err != nil {
+		// do not use logger.Panicf, since it isn't initialized yet.
+		panic(fmt.Errorf("FATAL: cannot read random data for rollupResultCacheKeyPrefix: %w", err))
+	}
+	return encoding.UnmarshalUint64(buf[:])
+}
+
+func mustLoadRollupResultCacheKeyPrefix(path string) {
+	path = path + ".key.prefix"
+	if !fs.IsPathExist(path) {
+		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
+		return
+	}
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		logger.Errorf("cannot load %s: %s; reset rollupResult cache", path, err)
+		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
+		return
+	}
+	if len(data) != 8 {
+		logger.Errorf("unexpected size of %s; want 8 bytes; got %d bytes; reset rollupResult cache", path, len(data))
+		rollupResultCacheKeyPrefix = newRollupResultCacheKeyPrefix()
+		return
+	}
+	rollupResultCacheKeyPrefix = encoding.UnmarshalUint64(data)
+}
+
+func mustSaveRollupResultCacheKeyPrefix(path string) {
+	path = path + ".key.prefix"
+	data := encoding.MarshalUint64(nil, rollupResultCacheKeyPrefix)
+	fs.MustRemoveAll(path)
+	if err := fs.WriteFileAtomically(path, data); err != nil {
+		logger.Fatalf("cannot store rollupResult cache key prefix to %q: %s", path, err)
+	}
+}
+
 var tooBigRollupResults = metrics.NewCounter("vm_too_big_rollup_results_total")
 
 // Increment this value every time the format of the cache changes.
@@ -490,20 +562,36 @@ func (mi *rollupResultCacheMetainfo) Unmarshal(src []byte) error {
 	return nil
 }
 
+func (mi *rollupResultCacheMetainfo) CoversTimeRange(start, end int64) bool {
+	if start > end {
+		logger.Panicf("BUG: start cannot exceed end; got %d vs %d", start, end)
+	}
+	for i := range mi.entries {
+		e := &mi.entries[i]
+		if start >= e.start && end <= e.end {
+			return true
+		}
+	}
+	return false
+}
+
 func (mi *rollupResultCacheMetainfo) GetBestKey(start, end int64) rollupResultCacheKey {
 	if start > end {
 		logger.Panicf("BUG: start cannot exceed end; got %d vs %d", start, end)
 	}
 	var bestKey rollupResultCacheKey
-	bestD := int64(1<<63 - 1)
+	dMax := int64(0)
 	for i := range mi.entries {
 		e := &mi.entries[i]
-		if start < e.start || end <= e.start {
+		if start < e.start {
 			continue
 		}
-		d := start - e.start
-		if d < bestD {
-			bestD = d
+		d := e.end - start
+		if end <= e.end {
+			d = end - start
+		}
+		if d >= dMax {
+			dMax = d
 			bestKey = e.key
 		}
 	}

app/vmselect/promql/rollup_result_cache_test.go

@@ -22,6 +22,7 @@ func TestRollupResultCacheInitStop(t *testing.T) {
 			StopRollupResultCache()
 		}
 		fs.MustRemoveAll(cacheFilePath)
+		fs.MustRemoveAll(cacheFilePath + ".key.prefix")
 	})
 }
 
@@ -55,7 +56,7 @@ func TestRollupResultCache(t *testing.T) {
 
 	// Try obtaining an empty value.
 	t.Run("empty", func(t *testing.T) {
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != ec.Start {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, ec.Start)
 		}
@@ -73,8 +74,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 1400 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
 		}
@@ -94,8 +95,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(ec, ae, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, ae, window)
+		rollupResultCacheV.Put(nil, ec, ae, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, ae, window)
 		if newStart != 1400 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
 		}
@@ -117,8 +118,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{333, 0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -136,8 +137,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -155,8 +156,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -174,8 +175,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -193,8 +194,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2, 3, 4, 5, 6, 7},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 2200 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
 		}
@@ -216,8 +217,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{1, 2, 3, 4, 5, 6},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 2200 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
 		}
@@ -241,8 +242,8 @@ func TestRollupResultCache(t *testing.T) {
 			}
 			tss = append(tss, ts)
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss)
-		tssResult, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss)
+		tssResult, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 2200 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
 		}
@@ -270,10 +271,10 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(ec, fe, window, tss1)
-		rollupResultCacheV.Put(ec, fe, window, tss2)
-		rollupResultCacheV.Put(ec, fe, window, tss3)
-		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss1)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss2)
+		rollupResultCacheV.Put(nil, ec, fe, window, tss3)
+		tss, newStart := rollupResultCacheV.Get(nil, ec, fe, window)
 		if newStart != 1400 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
 		}

app/vmselect/searchutils/searchutils.go

@@ -25,6 +25,19 @@ func roundToSeconds(ms int64) int64 {
 	return ms - ms%1000
 }
 
+// GetInt returns integer value from the given argKey.
+func GetInt(r *http.Request, argKey string) (int, error) {
+	argValue := r.FormValue(argKey)
+	if len(argValue) == 0 {
+		return 0, nil
+	}
+	n, err := strconv.Atoi(argValue)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse integer %q=%q: %w", argKey, argValue, err)
+	}
+	return n, nil
+}
+
 // GetTime returns time from the given argKey query arg.
 //
 // If argKey is missing in r, then defaultMs rounded to seconds is returned.
@@ -213,12 +226,16 @@ func GetExtraTagFilters(r *http.Request) ([][]storage.TagFilter, error) {
 		if len(tmp) != 2 {
 			return nil, fmt.Errorf("`extra_label` query arg must have the format `name=value`; got %q", match)
 		}
+		if tmp[0] == "__name__" {
+			// This is required for storage.Search.
+			tmp[0] = ""
+		}
 		tagFilters = append(tagFilters, storage.TagFilter{
 			Key:   []byte(tmp[0]),
 			Value: []byte(tmp[1]),
 		})
 	}
-	extraFilters := r.Form["extra_filters"]
+	extraFilters := append([]string{}, r.Form["extra_filters"]...)
 	extraFilters = append(extraFilters, r.Form["extra_filters[]"]...)
 	if len(extraFilters) == 0 {
 		if len(tagFilters) == 0 {

app/vmselect/vmui/asset-manifest.json

@@ -1,12 +1,12 @@
 {
   "files": {
-    "main.css": "./static/css/main.d8362c27.css",
-    "main.js": "./static/js/main.348f50e1.js",
+    "main.css": "./static/css/main.7e6d0c89.css",
+    "main.js": "./static/js/main.fdf5a65f.js",
     "static/js/27.939f971b.chunk.js": "./static/js/27.939f971b.chunk.js",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.d8362c27.css",
-    "static/js/main.348f50e1.js"
+    "static/css/main.7e6d0c89.css",
+    "static/js/main.fdf5a65f.js"
   ]
 }

app/vmselect/vmui/dashboards/README.md

@@ -1,7 +1,7 @@
 ### Setup
 1. Create `.json` config file in a folder `dashboards`
 2. Import your config file into the `dashboards/index.js`
-3. Add imported variable into the array `window.__VMUI_PREDEFINED_DASHBOARDS__`
+3. Add filename into the array `window.__VMUI_PREDEFINED_DASHBOARDS__`
 
 ### Configuration options
 

app/vmselect/vmui/dashboards/index.js

@@ -1,5 +1,3 @@
-import perJob from "./perJobUsage.json" assert { type: "json" };
-
 window.__VMUI_PREDEFINED_DASHBOARDS__ = [
-  perJob
+  "perJobUsage.json"
 ];

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.348f50e1.js"></script><link href="./static/css/main.d8362c27.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.fdf5a65f.js"></script><link href="./static/css/main.7e6d0c89.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmselect/vmui/static/css/main.7e6d0c89.css

@@ -1 +1 @@
-body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}.MuiAccordionSummary-content{margin:0!important}.uplot,.uplot *,.uplot :after,.uplot :before{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;line-height:1.5;width:-webkit-min-content;width:min-content}.u-title{font-size:18px;font-weight:700;text-align:center}.u-wrap{position:relative;-webkit-user-select:none;-ms-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;height:100%;position:relative;width:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{display:inline-block;vertical-align:middle}.u-legend .u-marker{background-clip:padding-box!important;height:1em;margin-right:4px;width:1em}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:rgba(0,0,0,.07)}.u-cursor-x,.u-cursor-y,.u-select{pointer-events:none;position:absolute}.u-cursor-x,.u-cursor-y{left:0;top:0;will-change:transform;z-index:100}.u-hz .u-cursor-x,.u-vt .u-cursor-y{border-right:1px dashed #607d8b;height:100%}.u-hz .u-cursor-y,.u-vt .u-cursor-x{border-bottom:1px dashed #607d8b;width:100%}.u-cursor-pt{background-clip:padding-box!important;border:0 solid;border-radius:50%;left:0;pointer-events:none;position:absolute;top:0;will-change:transform;z-index:100}.u-axis.u-off,.u-cursor-pt.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-select.u-off,.u-tooltip{display:none}.u-tooltip{grid-gap:12px;word-wrap:break-word;background:rgba(57,57,57,.9);border-radius:4px;color:#fff;font-family:monospace;font-size:10px;font-weight:500;line-height:1.4em;max-width:300px;padding:8px;pointer-events:none;position:absolute;z-index:100}.u-tooltip-data{align-items:center;display:flex;flex-wrap:wrap;font-size:11px;line-height:150%}.u-tooltip-data__value{font-weight:700;padding:4px}.u-tooltip__info{grid-gap:4px;display:grid}.u-tooltip__marker{height:12px;margin-right:4px;width:12px}.legendWrapper{cursor:default;display:flex;flex-wrap:wrap;margin-top:20px;position:relative}.legendGroup{margin:0 12px 24px 0}.legendGroupTitle{align-items:center;display:grid;font-size:11px;grid-template-columns:43px auto;padding:10px}.legendGroupQuery{grid-column:1/3;opacity:.6}.legendGroupLine{margin-right:10px}.legendItem{grid-gap:6px;align-items:start;background-color:#fff;cursor:pointer;display:grid;grid-template-columns:auto auto;justify-content:start;padding:7px 50px 7px 10px;transition:.2s ease}.legendItemHide{opacity:.5;text-decoration:line-through}.legendItem:hover{background-color:rgba(0,0,0,.1)}.legendMarker{border-style:solid;border-width:2px;box-sizing:border-box;height:12px;transition:.2s ease;width:12px}.legendLabel{font-size:11px;font-weight:400;line-height:12px}.legendFreeFields{cursor:pointer;padding:3px}.legendFreeFields:hover{text-decoration:underline}.legendFreeFields:not(:last-child):after{content:","}.legendWrapperHotkey{align-items:center;display:flex;font-size:11px}.legendWrapperHotkey p{margin-right:20px}.legendWrapperHotkey code{word-wrap:break-word;background-color:#f2f2f2;border:1px solid #dedede;border-radius:2px;color:#0a0a0a;display:inline;font-size:10px;font-weight:400;max-width:100%;padding:4px 6px}.panelDescription ul{line-height:2.2}.panelDescription a{color:#fff}.panelDescription code{background-color:rgba(0,0,0,.3);border-radius:2px;color:#fff;display:inline;font-size:inherit;font-weight:400;max-width:100%;padding:4px 6px}
+body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}.MuiAccordionSummary-content{margin:0!important}.uplot,.uplot *,.uplot :after,.uplot :before{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;line-height:1.5;width:-webkit-min-content;width:min-content}.u-title{font-size:18px;font-weight:700;text-align:center}.u-wrap{position:relative;-webkit-user-select:none;-ms-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;height:100%;position:relative;width:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{display:inline-block;vertical-align:middle}.u-legend .u-marker{background-clip:padding-box!important;height:1em;margin-right:4px;width:1em}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:rgba(0,0,0,.07)}.u-cursor-x,.u-cursor-y,.u-select{pointer-events:none;position:absolute}.u-cursor-x,.u-cursor-y{left:0;top:0;will-change:transform;z-index:100}.u-hz .u-cursor-x,.u-vt .u-cursor-y{border-right:1px dashed #607d8b;height:100%}.u-hz .u-cursor-y,.u-vt .u-cursor-x{border-bottom:1px dashed #607d8b;width:100%}.u-cursor-pt{background-clip:padding-box!important;border:0 solid;border-radius:50%;left:0;pointer-events:none;position:absolute;top:0;will-change:transform;z-index:100}.u-axis.u-off,.u-cursor-pt.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-select.u-off,.u-tooltip{display:none}.u-tooltip{grid-gap:12px;word-wrap:break-word;background:rgba(57,57,57,.9);border-radius:4px;color:#fff;font-family:monospace;font-size:10px;font-weight:500;line-height:1.4em;max-width:300px;padding:8px;pointer-events:none;position:absolute;z-index:100}.u-tooltip-data{align-items:center;display:flex;flex-wrap:wrap;font-size:11px;line-height:150%}.u-tooltip-data__value{font-weight:700;padding:4px}.u-tooltip__info{grid-gap:4px;display:grid}.u-tooltip__marker{height:12px;margin-right:4px;width:12px}.legendWrapper{cursor:default;display:flex;flex-wrap:wrap;margin-top:20px;position:relative}.legendGroup{margin:0 12px 24px 0;padding:10px 6px}.legendGroupTitle{align-items:center;border-bottom:1px solid #ecebe6;display:flex;font-size:11px;margin-bottom:5px;padding:0 10px 5px}.legendGroupQuery{font-weight:700;margin-right:4px}.legendGroupLine{margin-right:10px}.legendItem{grid-gap:6px;align-items:start;background-color:#fff;cursor:pointer;display:grid;grid-template-columns:auto auto;justify-content:start;padding:7px 50px 7px 10px;transition:.2s ease}.legendItemHide{opacity:.5;text-decoration:line-through}.legendItem:hover{background-color:rgba(0,0,0,.1)}.legendMarker{border-style:solid;border-width:2px;box-sizing:border-box;height:12px;transition:.2s ease;width:12px}.legendLabel{font-size:11px;font-weight:400;line-height:12px}.legendFreeFields{cursor:pointer;padding:3px}.legendFreeFields:hover{text-decoration:underline}.legendFreeFields:not(:last-child):after{content:","}.legendWrapperHotkey{align-items:center;display:flex;font-size:11px}.legendWrapperHotkey p{margin-right:20px}.legendWrapperHotkey code{word-wrap:break-word;background-color:#f2f2f2;border:1px solid #dedede;border-radius:2px;color:#0a0a0a;display:inline;font-size:10px;font-weight:400;max-width:100%;padding:4px 6px}.panelDescription ul{line-height:2.2}.panelDescription a{color:#fff}.panelDescription code{background-color:rgba(0,0,0,.3);border-radius:2px;color:#fff;display:inline;font-size:inherit;font-weight:400;max-width:100%;padding:4px 6px}

app/vmstorage/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
 	"github.com/VictoriaMetrics/metrics"
@@ -36,8 +37,11 @@ var (
 	finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
 		"Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+
 		"Zero value disables final merge")
-	bigMergeConcurrency   = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
-	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
+	bigMergeConcurrency     = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
+	smallMergeConcurrency   = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
+	retentionTimezoneOffset = flag.Duration("retentionTimezoneOffset", 0, "The offset for performing indexdb rotation. "+
+		"If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+
+		"If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)")
 
 	logNewSeries = flag.Bool("logNewSeries", false, "Whether to log new series. This option is for debug purposes only. It can lead to performance issues "+
 		"when big number of new series are ingested into VictoriaMetrics")
@@ -54,6 +58,7 @@ var (
 	cacheSizeStorageTSID        = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 	cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 	cacheSizeIndexDBDataBlocks  = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
+	cacheSizeIndexDBTagFilters  = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFilters cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning")
 )
 
 // CheckTimeRange returns true if the given tr is denied for querying.
@@ -90,8 +95,10 @@ func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetFinalMergeDelay(*finalMergeDelay)
 	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
 	storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
+	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.N)
+	storage.SetTagFilterCacheSize(cacheSizeIndexDBTagFilters.N)
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.N)
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.N)
 
@@ -166,43 +173,28 @@ func DeleteMetrics(tfss []*storage.TagFilters) (int, error) {
 }
 
 // SearchMetricNames returns metric names for the given tfss on the given tr.
-func SearchMetricNames(tfss []*storage.TagFilters, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]storage.MetricName, error) {
+func SearchMetricNames(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]storage.MetricName, error) {
 	WG.Add(1)
-	mns, err := Storage.SearchMetricNames(tfss, tr, maxMetrics, deadline)
+	mns, err := Storage.SearchMetricNames(qt, tfss, tr, maxMetrics, deadline)
 	WG.Done()
 	return mns, err
 }
 
-// SearchTagKeysOnTimeRange searches for tag keys on tr.
-func SearchTagKeysOnTimeRange(tr storage.TimeRange, maxTagKeys int, deadline uint64) ([]string, error) {
+// SearchLabelNamesWithFiltersOnTimeRange searches for tag keys matching the given tfss on tr.
+func SearchLabelNamesWithFiltersOnTimeRange(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr storage.TimeRange, maxTagKeys, maxMetrics int, deadline uint64) ([]string, error) {
 	WG.Add(1)
-	keys, err := Storage.SearchTagKeysOnTimeRange(tr, maxTagKeys, deadline)
+	labelNames, err := Storage.SearchLabelNamesWithFiltersOnTimeRange(qt, tfss, tr, maxTagKeys, maxMetrics, deadline)
 	WG.Done()
-	return keys, err
+	return labelNames, err
 }
 
-// SearchTagKeys searches for tag keys
-func SearchTagKeys(maxTagKeys int, deadline uint64) ([]string, error) {
+// SearchLabelValuesWithFiltersOnTimeRange searches for label values for the given labelName, tfss and tr.
+func SearchLabelValuesWithFiltersOnTimeRange(qt *querytracer.Tracer, labelName string, tfss []*storage.TagFilters,
+	tr storage.TimeRange, maxLabelValues, maxMetrics int, deadline uint64) ([]string, error) {
 	WG.Add(1)
-	keys, err := Storage.SearchTagKeys(maxTagKeys, deadline)
+	labelValues, err := Storage.SearchLabelValuesWithFiltersOnTimeRange(qt, labelName, tfss, tr, maxLabelValues, maxMetrics, deadline)
 	WG.Done()
-	return keys, err
-}
-
-// SearchTagValuesOnTimeRange searches for tag values for the given tagKey on tr.
-func SearchTagValuesOnTimeRange(tagKey []byte, tr storage.TimeRange, maxTagValues int, deadline uint64) ([]string, error) {
-	WG.Add(1)
-	values, err := Storage.SearchTagValuesOnTimeRange(tagKey, tr, maxTagValues, deadline)
-	WG.Done()
-	return values, err
-}
-
-// SearchTagValues searches for tag values for the given tagKey
-func SearchTagValues(tagKey []byte, maxTagValues int, deadline uint64) ([]string, error) {
-	WG.Add(1)
-	values, err := Storage.SearchTagValues(tagKey, maxTagValues, deadline)
-	WG.Done()
-	return values, err
+	return labelValues, err
 }
 
 // SearchTagValueSuffixes returns all the tag value suffixes for the given tagKey and tagValuePrefix on the given tr.
@@ -223,26 +215,10 @@ func SearchGraphitePaths(tr storage.TimeRange, query []byte, maxPaths int, deadl
 	return paths, err
 }
 
-// SearchTagEntries searches for tag entries.
-func SearchTagEntries(maxTagKeys, maxTagValues int, deadline uint64) ([]storage.TagEntry, error) {
+// GetTSDBStatus returns TSDB status for given filters on the given date.
+func GetTSDBStatus(qt *querytracer.Tracer, tfss []*storage.TagFilters, date uint64, focusLabel string, topN, maxMetrics int, deadline uint64) (*storage.TSDBStatus, error) {
 	WG.Add(1)
-	tagEntries, err := Storage.SearchTagEntries(maxTagKeys, maxTagValues, deadline)
-	WG.Done()
-	return tagEntries, err
-}
-
-// GetTSDBStatusForDate returns TSDB status for the given date.
-func GetTSDBStatusForDate(date uint64, topN, maxMetrics int, deadline uint64) (*storage.TSDBStatus, error) {
-	WG.Add(1)
-	status, err := Storage.GetTSDBStatusWithFiltersForDate(nil, date, topN, maxMetrics, deadline)
-	WG.Done()
-	return status, err
-}
-
-// GetTSDBStatusWithFiltersForDate returns TSDB status for given filters on the given date.
-func GetTSDBStatusWithFiltersForDate(tfss []*storage.TagFilters, date uint64, topN, maxMetrics int, deadline uint64) (*storage.TSDBStatus, error) {
-	WG.Add(1)
-	status, err := Storage.GetTSDBStatusWithFiltersForDate(tfss, date, topN, maxMetrics, deadline)
+	status, err := Storage.GetTSDBStatus(qt, tfss, date, focusLabel, topN, maxMetrics, deadline)
 	WG.Done()
 	return status, err
 }

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.18.1 as build-web-stage
+FROM golang:1.18.3 as build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 GO111MODULE=on CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 GO111MODULE=on CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.15.4
+FROM alpine:3.16.0
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/.eslintrc.js

@@ -42,7 +42,7 @@ module.exports = {
     "max-lines": [
       "error",
       {
-        "max": 150,
+        "max": 1000,
         "skipBlankLines": true,
         "skipComments": true,
       }

app/vmui/packages/vmui/package-lock.json

@@ -17808,16 +17808,6 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
-    "node_modules/svgo/node_modules/nth-check": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
-      "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "boolbase": "~1.0.0"
-      }
-    },
     "node_modules/symbol-tree": {
       "version": "3.2.4",
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
@@ -32702,7 +32692,7 @@
             "boolbase": "^1.0.0",
             "css-what": "^3.2.1",
             "domutils": "^1.7.0",
-            "nth-check": "^1.0.2"
+            "nth-check": "^2.0.1"
           }
         },
         "css-what": {
@@ -32753,16 +32743,6 @@
             "argparse": "^1.0.7",
             "esprima": "^4.0.0"
           }
-        },
-        "nth-check": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
-          "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
-          "dev": true,
-          "peer": true,
-          "requires": {
-            "boolbase": "~1.0.0"
-          }
         }
       }
     },

app/vmui/packages/vmui/package.json

@@ -69,6 +69,9 @@
   "overrides": {
     "react-app-rewired": {
       "nth-check": "^2.0.1"
+    },
+    "css-select": {
+      "nth-check": "^2.0.1"
     }
   }
 }

app/vmui/packages/vmui/public/dashboards/README.md

@@ -1,7 +1,7 @@
 ### Setup
 1. Create `.json` config file in a folder `dashboards`
 2. Import your config file into the `dashboards/index.js`
-3. Add imported variable into the array `window.__VMUI_PREDEFINED_DASHBOARDS__`
+3. Add filename into the array `window.__VMUI_PREDEFINED_DASHBOARDS__`
 
 ### Configuration options
 

app/vmui/packages/vmui/public/dashboards/index.js

@@ -1,5 +1,3 @@
-import perJob from "./perJobUsage.json" assert { type: "json" };
-
 window.__VMUI_PREDEFINED_DASHBOARDS__ = [
-  perJob
+  "perJobUsage.json"
 ];

app/vmui/packages/vmui/src/App.tsx

@@ -4,6 +4,7 @@ import {SnackbarProvider} from "./contexts/Snackbar";
 import {StateProvider} from "./state/common/StateContext";
 import {AuthStateProvider} from "./state/auth/AuthStateContext";
 import {GraphStateProvider} from "./state/graph/GraphStateContext";
+import {CardinalityStateProvider} from "./state/cardinality/CardinalityStateContext";
 import THEME from "./theme/theme";
 import { ThemeProvider, StyledEngineProvider } from "@mui/material/styles";
 import CssBaseline from "@mui/material/CssBaseline";
@@ -14,6 +15,7 @@ import router from "./router/index";
 import CustomPanel from "./components/CustomPanel/CustomPanel";
 import HomeLayout from "./components/Home/HomeLayout";
 import DashboardsLayout from "./components/PredefinedPanels/DashboardsLayout";
+import CardinalityPanel from "./components/CardinalityPanel/CardinalityPanel";
 
 
 const App: FC = () => {
@@ -27,14 +29,17 @@ const App: FC = () => {
             <StateProvider> {/* Serialized into query string, common app settings */}
               <AuthStateProvider> {/* Auth related info - optionally persisted to Local Storage */}
                 <GraphStateProvider> {/* Graph settings */}
-                  <SnackbarProvider> {/* Display various snackbars */}
-                    <Routes>
-                      <Route path={"/"} element={<HomeLayout/>}>
-                        <Route path={router.home} element={<CustomPanel/>}/>
-                        <Route path={router.dashboards} element={<DashboardsLayout/>}/>
-                      </Route>
-                    </Routes>
-                  </SnackbarProvider>
+                  <CardinalityStateProvider> {/* Cardinality settings */}
+                    <SnackbarProvider> {/* Display various snackbars */}
+                      <Routes>
+                        <Route path={"/"} element={<HomeLayout/>}>
+                          <Route path={router.home} element={<CustomPanel/>}/>
+                          <Route path={router.dashboards} element={<DashboardsLayout/>}/>
+                          <Route path={router.cardinality} element={<CardinalityPanel/>} />
+                        </Route>
+                      </Routes>
+                    </SnackbarProvider>
+                  </CardinalityStateProvider>
                 </GraphStateProvider>
               </AuthStateProvider>
             </StateProvider>

app/vmui/packages/vmui/src/api/tsdb.ts

@@ -0,0 +1,14 @@
+export interface CardinalityRequestsParams {
+  topN: number,
+  extraLabel: string | null,
+  match: string | null,
+  date: string | null,
+  focusLabel: string | null,
+}
+
+export const getCardinalityInfo = (server: string, requestsParam: CardinalityRequestsParams) => {
+  const match = requestsParam.match ? "&match[]=" + encodeURIComponent(requestsParam.match) : "";
+  const focusLabel = requestsParam.focusLabel ? "&focusLabel=" + encodeURIComponent(requestsParam.focusLabel) : "";
+  return `${server}/api/v1/status/tsdb?topN=${requestsParam.topN}&date=${requestsParam.date}${match}${focusLabel}`;
+};
+

app/vmui/packages/vmui/src/components/BarChart/BarChart.tsx

@@ -0,0 +1,41 @@
+import React, {FC, useEffect, useRef, useState} from "preact/compat";
+import uPlot, {Options as uPlotOptions} from "uplot";
+import useResize from "../../hooks/useResize";
+import {BarChartProps} from "./types";
+
+const BarChart: FC<BarChartProps> = ({
+  data,
+  container,
+  configs}) => {
+
+  const uPlotRef = useRef<HTMLDivElement>(null);
+  const [isPanning] = useState(false);
+  const [uPlotInst, setUPlotInst] = useState<uPlot>();
+  const layoutSize = useResize(container);
+
+  const options: uPlotOptions ={
+    ...configs,
+    width: layoutSize.width || 400,
+  };
+
+  const updateChart = (): void => {
+    if (!uPlotInst) return;
+    uPlotInst.setData(data);
+    if (!isPanning) uPlotInst.redraw();
+  };
+
+  useEffect(() => {
+    if (!uPlotRef.current) return;
+    const u = new uPlot(options, data, uPlotRef.current);
+    setUPlotInst(u);
+    return u.destroy;
+  }, [uPlotRef.current, layoutSize]);
+
+  useEffect(() => updateChart(), [data]);
+
+  return <div style={{pointerEvents: isPanning ? "none" : "auto", height: "100%"}}>
+    <div ref={uPlotRef}/>
+  </div>;
+};
+
+export default BarChart;

app/vmui/packages/vmui/src/components/BarChart/consts.ts

@@ -0,0 +1,51 @@
+import {seriesBarsPlugin} from "../../utils/uplot/plugin";
+import {barDisp, getBarSeries} from "../../utils/uplot/series";
+import {Fill, Stroke} from "../../utils/uplot/types";
+import {PaddingSide, Series} from "uplot";
+
+
+const stroke: Stroke = {
+  unit: 3,
+  values: (u: { data: number[][]; }) => u.data[1].map((_: number, idx) =>
+    idx !== 0 ? "#33BB55" : "#F79420"
+  ),
+};
+
+const fill: Fill = {
+  unit: 3,
+  values: (u: { data: number[][]; }) => u.data[1].map((_: number, idx) =>
+    idx !== 0 ? "#33BB55" : "#F79420"
+  ),
+};
+
+export const barOptions = {
+  height: 500,
+  width: 500,
+  padding: [null, 0, null, 0] as [top: PaddingSide, right: PaddingSide, bottom: PaddingSide, left: PaddingSide],
+  axes: [{ show: false }],
+  series: [
+    {
+      label: "",
+      value: (u: uPlot, v: string) => v
+    },
+    {
+      label: " ",
+      width: 0,
+      fill: "",
+      values: (u: uPlot, seriesIdx: number) => {
+        const idxs = u.legend.idxs || [];
+
+        if (u.data === null || idxs.length === 0)
+          return {"Name": null, "Value": null,};
+
+        const dataIdx = idxs[seriesIdx] || 0;
+
+        const build = u.data[0][dataIdx];
+        const duration = u.data[seriesIdx][dataIdx];
+
+        return {"Name": build, "Value": duration};
+      }
+    },
+  ] as Series[],
+  plugins: [seriesBarsPlugin(getBarSeries([1], 0, 1, 0, barDisp(stroke, fill)))],
+};

app/vmui/packages/vmui/src/components/BarChart/types.ts

@@ -0,0 +1,7 @@
+import {AlignedData as uPlotData, Options as uPlotOptions} from "uplot";
+
+export interface BarChartProps {
+  data: uPlotData;
+  container: HTMLDivElement | null,
+  configs: uPlotOptions,
+}

app/vmui/packages/vmui/src/components/BorderLineProgress/BorderLinearProgress.tsx

@@ -0,0 +1,27 @@
+import React from "preact/compat";
+import { styled } from "@mui/material/styles";
+import LinearProgressWithLabel, {linearProgressClasses, LinearProgressProps} from "@mui/material/LinearProgress";
+import {Box, Typography} from "@mui/material";
+
+export const BorderLinearProgress = styled(LinearProgressWithLabel)(({ theme }) => ({
+  height: 20,
+  borderRadius: 5,
+  [`&.${linearProgressClasses.colorPrimary}`]: {
+    backgroundColor: theme.palette.grey[theme.palette.mode === "light" ? 200 : 800],
+  },
+  [`& .${linearProgressClasses.bar}`]: {
+    borderRadius: 5,
+    backgroundColor: theme.palette.mode === "light" ? "#1a90ff" : "#308fe8",
+  },
+}));
+
+export const BorderLinearProgressWithLabel = (props: LinearProgressProps & { value: number }) => (
+  <Box sx={{ display: "flex", alignItems: "center" }}>
+    <Box sx={{ width: "100%", mr: 1 }}>
+      <BorderLinearProgress variant="determinate" {...props} />
+    </Box>
+    <Box sx={{ minWidth: 35 }}>
+      <Typography variant="body2" color="text.secondary">{`${props.value.toFixed(2)}%`}</Typography>
+    </Box>
+  </Box>
+);

app/vmui/packages/vmui/src/components/CardinalityPanel/CardinalityConfigurator/CardinalityConfigurator.tsx

@@ -0,0 +1,103 @@
+import React, {ChangeEvent, FC} from "react";
+import Box from "@mui/material/Box";
+import QueryEditor from "../../CustomPanel/Configurator/Query/QueryEditor";
+import Tooltip from "@mui/material/Tooltip";
+import IconButton from "@mui/material/IconButton";
+import PlayCircleOutlineIcon from "@mui/icons-material/PlayCircleOutline";
+import {useFetchQueryOptions} from "../../../hooks/useFetchQueryOptions";
+import {useAppDispatch, useAppState} from "../../../state/common/StateContext";
+import FormControlLabel from "@mui/material/FormControlLabel";
+import BasicSwitch from "../../../theme/switch";
+import {saveToStorage} from "../../../utils/storage";
+import TextField from "@mui/material/TextField";
+import {ErrorTypes} from "../../../types";
+
+export interface CardinalityConfiguratorProps {
+  onSetHistory: (step: number, index: number) => void;
+  onSetQuery: (query: string, index: number) => void;
+  onRunQuery: () => void;
+  onTopNChange: (e: ChangeEvent<HTMLTextAreaElement|HTMLInputElement>) => void;
+  onFocusLabelChange: (e: ChangeEvent<HTMLTextAreaElement|HTMLInputElement>) => void;
+  query: string;
+  topN: number;
+  error?: ErrorTypes | string;
+  totalSeries: number;
+  totalLabelValuePairs: number;
+  date: string | null;
+  match: string | null;
+  focusLabel: string | null;
+}
+
+const CardinalityConfigurator: FC<CardinalityConfiguratorProps> = ({
+  topN,
+  error,
+  query,
+  onSetHistory,
+  onRunQuery,
+  onSetQuery,
+  onTopNChange,
+  onFocusLabelChange,
+  totalSeries,
+  totalLabelValuePairs,
+  date,
+  match,
+  focusLabel
+}) => {
+  const dispatch = useAppDispatch();
+  const {queryControls: {autocomplete}} = useAppState();
+  const {queryOptions} = useFetchQueryOptions();
+
+  const onChangeAutocomplete = () => {
+    dispatch({type: "TOGGLE_AUTOCOMPLETE"});
+    saveToStorage("AUTOCOMPLETE", !autocomplete);
+  };
+
+  return <Box boxShadow="rgba(99, 99, 99, 0.2) 0px 2px 8px 0px;" p={4} pb={2} mb={2}>
+    <Box>
+      <Box display="grid" gridTemplateColumns="1fr auto auto auto auto" gap="4px" width="100%" mb={4}>
+        <QueryEditor
+          query={query} index={0} autocomplete={autocomplete} queryOptions={queryOptions}
+          error={error} setHistoryIndex={onSetHistory} runQuery={onRunQuery} setQuery={onSetQuery}
+          label={"Time series selector"}
+        />
+        <Box mr={2}>
+          <TextField
+            label="Number of entries per table"
+            type="number"
+            size="medium"
+            variant="outlined"
+            value={topN}
+            error={topN < 1}
+            helperText={topN < 1 ? "Number must be bigger than zero" : " "}
+            onChange={onTopNChange}/>
+        </Box>
+        <Box mr={2}>
+          <TextField
+            label="Focus label"
+            type="text"
+            size="medium"
+            variant="outlined"
+            value={focusLabel}
+            onChange={onFocusLabelChange} />
+        </Box>
+        <Box>
+          <FormControlLabel label="Enable autocomplete"
+            control={<BasicSwitch checked={autocomplete} onChange={onChangeAutocomplete}/>}
+          />
+        </Box>
+        <Tooltip title="Execute Query">
+          <IconButton onClick={onRunQuery} sx={{height: "49px", width: "49px"}}>
+            <PlayCircleOutlineIcon/>
+          </IconButton>
+        </Tooltip>
+      </Box>
+    </Box>
+    <Box>
+      Analyzed <b>{totalSeries}</b> series with <b>{totalLabelValuePairs}</b> &quot;label=value&quot; pairs
+      at <b>{date}</b> {match && <span>for series selector <b>{match}</b></span>}.
+      Show top {topN} entries per table.
+    </Box>
+  </Box>;
+};
+
+export default CardinalityConfigurator;

app/vmui/packages/vmui/src/components/CardinalityPanel/CardinalityPanel.tsx

@@ -0,0 +1,119 @@
+import React, {ChangeEvent, FC, useState} from "react";
+import {SyntheticEvent} from "react";
+import {Alert} from "@mui/material";
+import {useFetchQuery} from "../../hooks/useCardinalityFetch";
+import {queryUpdater} from "./helpers";
+import {Data} from "../Table/types";
+import CardinalityConfigurator from "./CardinalityConfigurator/CardinalityConfigurator";
+import Spinner from "../common/Spinner";
+import {useCardinalityDispatch, useCardinalityState} from "../../state/cardinality/CardinalityStateContext";
+import MetricsContent from "./MetricsContent/MetricsContent";
+import {DefaultActiveTab, Tabs, TSDBStatus, Containers} from "./types";
+
+const spinnerContainerStyles = (height: string) =>  {
+  return {
+    width: "100%",
+    maxWidth: "100%",
+    position: "absolute",
+    height: height ?? "50%",
+    background: "rgba(255, 255, 255, 0.7)",
+    pointerEvents: "none",
+    zIndex: 1000,
+  };
+};
+
+const CardinalityPanel: FC = () => {
+  const cardinalityDispatch = useCardinalityDispatch();
+
+  const {topN, match, date, focusLabel} = useCardinalityState();
+  const configError = "";
+  const [query, setQuery] = useState(match || "");
+  const [queryHistoryIndex, setQueryHistoryIndex] = useState(0);
+  const [queryHistory, setQueryHistory] = useState<string[]>([]);
+
+  const onRunQuery = () => {
+    setQueryHistory(prev => [...prev, query]);
+    setQueryHistoryIndex(prev => prev + 1);
+    cardinalityDispatch({type: "SET_MATCH", payload: query});
+    cardinalityDispatch({type: "RUN_QUERY"});
+  };
+
+  const onSetQuery = (query: string) => {
+    setQuery(query);
+  };
+
+  const onSetHistory = (step: number) => {
+    const newIndexHistory = queryHistoryIndex + step;
+    if (newIndexHistory < 0 || newIndexHistory >= queryHistory.length) return;
+    setQueryHistoryIndex(newIndexHistory);
+    setQuery(queryHistory[newIndexHistory]);
+  };
+
+  const onTopNChange = (e: ChangeEvent<HTMLTextAreaElement|HTMLInputElement>) => {
+    cardinalityDispatch({type: "SET_TOP_N", payload: +e.target.value});
+  };
+
+  const onFocusLabelChange = (e: ChangeEvent<HTMLTextAreaElement|HTMLInputElement>) => {
+    cardinalityDispatch({type: "SET_FOCUS_LABEL", payload: e.target.value});
+  };
+
+  const {isLoading, appConfigurator, error} = useFetchQuery();
+  const [stateTabs, setTab] = useState(appConfigurator.defaultState.defaultActiveTab);
+  const {tsdbStatusData, defaultState, tablesHeaders} = appConfigurator;
+  const handleTabChange = (e: SyntheticEvent, newValue: number) => {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    setTab({...stateTabs, [e.target.id]: newValue});
+  };
+
+  const handleFilterClick = (key: string) => (e: SyntheticEvent) => {
+    const name = e.currentTarget.id;
+    const query = queryUpdater[key](focusLabel, name);
+    setQuery(query);
+    setQueryHistory(prev => [...prev, query]);
+    setQueryHistoryIndex(prev => prev + 1);
+    cardinalityDispatch({type: "SET_MATCH", payload: query});
+    let newFocusLabel = "";
+    if (key === "labelValueCountByLabelName" || key == "seriesCountByLabelName") {
+      newFocusLabel = name;
+    }
+    cardinalityDispatch({type: "SET_FOCUS_LABEL", payload: newFocusLabel});
+    cardinalityDispatch({type: "RUN_QUERY"});
+  };
+
+  return (
+    <>
+      {isLoading && <Spinner
+        isLoading={isLoading}
+        height={"800px"}
+        containerStyles={spinnerContainerStyles("100%")}
+        title={<Alert color="error" severity="error" sx={{whiteSpace: "pre-wrap", mt: 2}}>
+          Please wait while cardinality stats is calculated. This may take some time if the db contains big number of time series
+        </Alert>}
+      />}
+      <CardinalityConfigurator error={configError} query={query} onRunQuery={onRunQuery} onSetQuery={onSetQuery}
+        onSetHistory={onSetHistory} onTopNChange={onTopNChange} topN={topN} date={date} match={match}
+        totalSeries={tsdbStatusData.totalSeries} totalLabelValuePairs={tsdbStatusData.totalLabelValuePairs}
+        focusLabel={focusLabel} onFocusLabelChange={onFocusLabelChange}
+      />
+      {error && <Alert color="error" severity="error" sx={{whiteSpace: "pre-wrap", mt: 2}}>{error}</Alert>}
+      {appConfigurator.keys(focusLabel).map((keyName) => (
+        <MetricsContent
+          key={keyName}
+          sectionTitle={appConfigurator.sectionsTitles(focusLabel)[keyName]}
+          activeTab={stateTabs[keyName as keyof DefaultActiveTab]}
+          rows={tsdbStatusData[keyName as keyof TSDBStatus] as unknown as Data[]}
+          onChange={handleTabChange}
+          onActionClick={handleFilterClick(keyName)}
+          tabs={defaultState.tabs[keyName as keyof Tabs]}
+          chartContainer={defaultState.containerRefs[keyName as keyof Containers<HTMLDivElement>]}
+          totalSeries={appConfigurator.totalSeries(keyName)}
+          tabId={keyName}
+          tableHeaderCells={tablesHeaders[keyName]}
+        />
+      ))}
+    </>
+  );
+};
+
+export default CardinalityPanel;