app/vmctl/terminal: fix builds for GOOS=freebsd and GOOS=openbsd

This is a follow-up for 8da9502df6

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.2
+          go-version: 1.20.3
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis.yml

@@ -55,9 +55,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.2
+          go-version: 1.20.3
           check-latest: true
           cache: true
         if: ${{ matrix.language == 'go' }}

.github/workflows/main.yml

@@ -30,9 +30,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.2
+          go-version: 1.20.3
           check-latest: true
           cache: true
 
@@ -54,9 +54,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.2
+          go-version: 1.20.3
           check-latest: true
           cache: true
 
@@ -79,9 +79,9 @@ jobs:
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.20.2
+          go-version: 1.20.3
           check-latest: true
           cache: true
 

.github/workflows/nightly-build.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.2
+          go-version: 1.20.3
         id: go
 
       - name: Setup docker scan

Makefile

@@ -141,6 +141,8 @@ vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
 	vmalert-windows-amd64 \
 	vmauth-windows-amd64 \
+	vmbackup-windows-amd64 \
+	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
 victoria-metrics-crossbuild: \
@@ -186,7 +188,8 @@ release-victoria-metrics: \
 	release-victoria-metrics-darwin-amd64 \
 	release-victoria-metrics-darwin-arm64 \
 	release-victoria-metrics-freebsd-amd64 \
-	release-victoria-metrics-openbsd-amd64
+	release-victoria-metrics-openbsd-amd64 \
+	release-victoria-metrics-windows-amd64
 
 # adds i386 arch
 release-victoria-metrics-linux-386:
@@ -213,6 +216,9 @@ release-victoria-metrics-freebsd-amd64:
 release-victoria-metrics-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
+release-victoria-metrics-windows-amd64:
+	GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
+
 release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
 		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
@@ -222,6 +228,16 @@ release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
 	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
 
+release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
+	cd bin && \
+		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-metrics-windows-$(GOARCH)-prod.exe \
+		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-metrics-windows-$(GOARCH)-prod.exe \
+			> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		victoria-metrics-windows-$(GOARCH)-prod.exe
+
 release-vmutils: \
 	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
@@ -295,26 +311,33 @@ release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
+	vmbackup-windows-$(GOARCH)-prod \
+	vmrestore-windows-$(GOARCH)-prod \
 	vmctl-windows-$(GOARCH)-prod
 	cd bin && \
 		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
+			vmbackup-windows-$(GOARCH)-prod.exe \
+			vmrestore-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
+			vmbackup-windows-$(GOARCH)-prod.exe \
+			vmrestore-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
 	cd bin && rm -rf \
 		vmagent-windows-$(GOARCH)-prod.exe \
 		vmalert-windows-$(GOARCH)-prod.exe \
 		vmauth-windows-$(GOARCH)-prod.exe \
+		vmbackup-windows-$(GOARCH)-prod.exe \
+		vmrestore-windows-$(GOARCH)-prod.exe \
 		vmctl-windows-$(GOARCH)-prod.exe
 
-
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 

README.md

@@ -40,7 +40,8 @@ VictoriaMetrics has the following prominent features:
 * It can be used as long-term storage for Prometheus. See [these docs](#prometheus-setup) for details.
 * It can be used as a drop-in replacement for Prometheus in Grafana, because it supports [Prometheus querying API](#prometheus-querying-api-usage).
 * It can be used as a drop-in replacement for Graphite in Grafana, because it supports [Graphite API](#graphite-api-usage).
-* It features easy setup and operation:
+  VictoriaMetrics allows reducing infrastructure costs by more than 10x comparing to Graphite - see [this case study](https://docs.victoriametrics.com/CaseStudies.html#grammarly).
+* It is easy to setup and operate:
   * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d)
     without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
@@ -201,7 +202,7 @@ Changing scrape configuration is possible with text editor:
 vi $SNAP_DATA/var/snap/victoriametrics/current/etc/victoriametrics-scrape-config.yaml
 ```
 
-After changes were made, trigger config re-read with the command `curl 127.0.0.1:8248/-/reload`.
+After changes were made, trigger config re-read with the command `curl 127.0.0.1:8428/-/reload`.
 
 ## Prometheus setup
 
@@ -627,7 +628,6 @@ The `__graphite__` pseudo-label supports e.g. alternate regexp filters such as `
 
 VictoriaMetrics also supports Graphite query language - see [these docs](#graphite-render-api-usage).
 
-
 ## How to send data from OpenTSDB-compatible agents
 
 VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html)
@@ -742,20 +742,45 @@ All the Prometheus querying API handlers can be prepended with `/prometheus` pre
 
 ### Prometheus querying API enhancements
 
-VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg, which can be used for enforcing additional label filters for queries. For example,
-`/api/v1/query_range?extra_label=user_id=123&extra_label=group_id=456&query=<query>` would automatically add `{user_id="123",group_id="456"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_label` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query arg, which can be used
+for enforcing additional label filters for queries. For example, `/api/v1/query_range?extra_label=user_id=123&extra_label=group_id=456&query=<query>`
+would automatically add `{user_id="123",group_id="456"}` label filters to the given `<query>`.
+This functionality can be used for limiting the scope of time series visible to the given tenant.
+It is expected that the `extra_label` query args are automatically set by auth proxy sitting in front of VictoriaMetrics.
+See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
 
-VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example,
-`/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
+VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries.
+For example, `/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically
+add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting
+the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically
+set by auth proxy sitting in front of VictoriaMetrics.
+See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
 
 VictoriaMetrics accepts multiple formats for `time`, `start` and `end` query args - see [these docs](#timestamp-formats).
 
-VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
+VictoriaMetrics accepts `round_digits` query arg for [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query)
+and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) handlers. It can be used for rounding response values
+to the given number of digits after the decimal point.
+For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
 
-VictoriaMetrics accepts `limit` query arg for `/api/v1/labels` and `/api/v1/label/<labelName>/values` handlers for limiting the number of returned entries. For example, the query to `/api/v1/labels?limit=5` returns a sample of up to 5 unique labels, while ignoring the rest of labels. If the provided `limit` value exceeds the corresponding `-search.maxTagKeys` / `-search.maxTagValues` command-line flag values, then limits specified in the command-line flags are used.
+VictoriaMetrics accepts `limit` query arg for [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels)
+and [`/api/v1/label/<labelName>/values`](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues) handlers for limiting the number of returned entries.
+For example, the query to `/api/v1/labels?limit=5` returns a sample of up to 5 unique labels, while ignoring the rest of labels.
+If the provided `limit` value exceeds the corresponding `-search.maxTagKeys` / `-search.maxTagValues` command-line flag values,
+then limits specified in the command-line flags are used.
 
-By default, VictoriaMetrics returns time series for the last 5 minutes from `/api/v1/series`, `/api/v1/labels` and `/api/v1/label/<labelName>/values` while the Prometheus API defaults to all time.  Explicitly set `start` and `end` to select the desired time range.
-VictoriaMetrics accepts `limit` query arg for `/api/v1/series` handlers for limiting the number of returned entries. For example, the query to `/api/v1/series?limit=5` returns a sample of up to 5 series, while ignoring the rest. If the provided `limit` value exceeds the corresponding `-search.maxSeries` command-line flag values, then limits specified in the command-line flags are used.
+By default, VictoriaMetrics returns time series for the last day starting at 00:00 UTC
+from [/api/v1/series](https://docs.victoriametrics.com/url-examples.html#apiv1series),
+[/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and
+[`/api/v1/label/<labelName>/values`](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues),
+while the Prometheus API defaults to all time.  Explicitly set `start` and `end` to select the desired time range.
+VictoriaMetrics rounds the specified `start..end` time range to day granularity because of performance optimization concerns.
+If you need the exact set of label names and label values on the given time range, then send queries
+to [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) or to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query).
+
+VictoriaMetrics accepts `limit` query arg at [/api/v1/series](https://docs.victoriametrics.com/url-examples.html#apiv1series)
+for limiting the number of returned entries. For example, the query to `/api/v1/series?limit=5` returns a sample of up to 5 series, while ignoring the rest of series.
+If the provided `limit` value exceeds the corresponding `-search.maxSeries` command-line flag values, then limits specified in the command-line flags are used.
 
 Additionally, VictoriaMetrics provides the following handlers:
 
@@ -804,10 +829,10 @@ VictoriaMetrics supports `__graphite__` pseudo-label for filtering time series w
 
 ### Graphite Render API usage
 
-[VictoriaMetrics Enterprise](https://docs.victoriametrics.com/enterprise.html) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
+VictoriaMetrics supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
 at `/render` endpoint, which is used by [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
-When configuring Graphite datasource in Grafana, the `Storage-Step` http request header must be set to a step between Graphite data points stored in VictoriaMetrics. For example, `Storage-Step: 10s` would mean 10 seconds distance between Graphite datapoints stored in VictoriaMetrics.
-Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
+When configuring Graphite datasource in Grafana, the `Storage-Step` http request header must be set to a step between Graphite data points
+stored in VictoriaMetrics. For example, `Storage-Step: 10s` would mean 10 seconds distance between Graphite datapoints stored in VictoriaMetrics.
 
 ### Graphite Metrics API usage
 
@@ -1447,12 +1472,14 @@ can be configured with the `-inmemoryDataFlushInterval` command-line flag (note
 In-memory parts are persisted to disk into `part` directories under the `<-storageDataPath>/data/small/YYYY_MM/` folder,
 where `YYYY_MM` is the month partition for the stored data. For example, `2022_11` is the partition for `parts`
 with [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) from `November 2022`.
+Each partition directory contains `parts.json` file with the actual list of parts in the partition.
 
-The `part` directory has the following name pattern: `rowsCount_blocksCount_minTimestamp_maxTimestamp`, where:
+Every `part` directory contains `metadata.json` file with the following fields:
 
-- `rowsCount` - the number of [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) stored in the part
-- `blocksCount` - the number of blocks stored in the part (see details about blocks below)
-- `minTimestamp` and `maxTimestamp` - minimum and maximum timestamps across raw samples stored in the part
+- `RowsCount` - the number of [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) stored in the part
+- `BlocksCount` - the number of blocks stored in the part (see details about blocks below)
+- `MinTimestamp` and `MaxTimestamp` - minimum and maximum timestamps across raw samples stored in the part
+- `MinDedupInterval` - the [deduplication interval](#deduplication) applied to the given part.
 
 Each `part` consists of `blocks` sorted by internal time series id (aka `TSID`).
 Each `block` contains up to 8K [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples),
@@ -1474,9 +1501,8 @@ for fast block lookups, which belong to the given `TSID` and cover the given tim
   and [freeing up disk space for the deleted time series](#how-to-delete-time-series) are performed during the merge
 
 Newly added `parts` either successfully appear in the storage or fail to appear.
-The newly added `parts` are being created in a temporary directory under `<-storageDataPath>/data/{small,big}/YYYY_MM/tmp` folder.
-When the newly added `part` is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html)
-to a temporary directory, then it is atomically moved to the storage directory.
+The newly added `part` is atomically registered in the `parts.json` file under the corresponding partition
+after it is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html) to the storage.
 Thanks to this alogrithm, storage never contains partially created parts, even if hardware power off
 occurrs in the middle of writing the `part` to disk - such incompletely written `parts`
 are automatically deleted on the next VictoriaMetrics start.
@@ -1505,8 +1531,7 @@ Retention is configured with the `-retentionPeriod` command-line flag, which tak
 
 Data is split in per-month partitions inside `<-storageDataPath>/data/{small,big}` folders.
 Data partitions outside the configured retention are deleted on the first day of the new month.
-Each partition consists of one or more data parts with the following name pattern `rowsCount_blocksCount_minTimestamp_maxTimestamp`.
-Data parts outside of the configured retention are eventually deleted during
+Each partition consists of one or more data parts. Data parts outside of the configured retention are eventually deleted during
 [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
 
 The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
@@ -1899,7 +1924,14 @@ are added to all the metrics before sending them to the remote storage:
 
 ## Cache removal
 
-VictoriaMetrics uses various internal caches. These caches are stored to `<-storageDataPath>/cache` directory during graceful shutdown (e.g. when VictoriaMetrics is stopped by sending `SIGINT` signal). The caches are read on the next VictoriaMetrics startup. Sometimes it is needed to remove such caches on the next startup. This can be performed by placing `reset_cache_on_startup` file inside the `<-storageDataPath>/cache` directory before the restart of VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447) for details.
+VictoriaMetrics uses various internal caches. These caches are stored to `<-storageDataPath>/cache` directory during graceful shutdown
+(e.g. when VictoriaMetrics is stopped by sending `SIGINT` signal). The caches are read on the next VictoriaMetrics startup.
+Sometimes it is needed to remove such caches on the next startup. This can be done in the following ways:
+
+- By manually removing the `<-storageDataPath>/cache` directory when VictoriaMetrics is stopped.
+- By placing `reset_cache_on_startup` file inside the `<-storageDataPath>/cache` directory before the restart of VictoriaMetrics.
+  In this case VictoriaMetrics will automatically remove all the caches on the next start.
+  See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447) for details.
 
 ## Cache tuning
 
@@ -2161,7 +2193,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
   -dryRun
-     Whether to check only -promscrape.config and then exit. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag
+     Whether to check config files without running VictoriaMetrics. The following config files are checked: -promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
@@ -2406,9 +2438,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.disableCache
      Whether to disable response caching. This may be useful during data backfilling
   -search.graphiteMaxPointsPerSeries int
-     The maximum number of points per series Graphite render API can return. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 1000000)
+     The maximum number of points per series Graphite render API can return (default 1000000)
   -search.graphiteStorageStep duration
-     The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 10s)
+     The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API (default 10s)
   -search.latencyOffset duration
      The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
   -search.logQueryMemoryUsage size
@@ -2425,7 +2457,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxFederateSeries int
      The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
   -search.maxGraphiteSeries int
-     The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 300000)
+     The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons
   -search.maxMemoryPerQuery size

SECURITY.md

@@ -4,10 +4,10 @@
 
 | Version | Supported          |
 |---------|--------------------|
-| 1.81.x  | :white_check_mark: |
-| 1.80.x  | :x:                |
-| 1.79.x  | :white_check_mark: |
-| < 1.78  | :x:                |
+| [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
+| v1.87.x LTS release | :white_check_mark: |
+| v1.79.x LTS release | :white_check_mark: |
+| other releases  | :x:                |
 
 ## Reporting a Vulnerability
 

app/victoria-metrics/Makefile

@@ -39,6 +39,9 @@ victoria-metrics-freebsd-amd64-prod:
 victoria-metrics-openbsd-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-openbsd-amd64
 
+victoria-metrics-windows-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-windows-amd64
+
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker
 
@@ -100,6 +103,9 @@ victoria-metrics-freebsd-amd64:
 victoria-metrics-openbsd-amd64:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
+victoria-metrics-windows-amd64:
+	GOARCH=amd64 APP_NAME=victoria-metrics $(MAKE) app-local-windows-goarch
+
 victoria-metrics-pure:
 	APP_NAME=victoria-metrics $(MAKE) app-local-pure
 

app/victoria-metrics/main.go

@@ -8,6 +8,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
+	vminsertcommon "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	vminsertrelabel "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -30,8 +32,9 @@ var (
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
-	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
-		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
+	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
+		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
+		"This can be changed with -promscrape.config.strictParse=false command-line flag")
 	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
 		"The saved data survives unclean shutdown such as OOM crash, hardware reset, SIGKILL, etc. "+
 		"Bigger intervals may help increasing lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
@@ -54,6 +57,12 @@ func main() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
+		if err := vminsertrelabel.CheckRelabelConfig(); err != nil {
+			logger.Fatalf("error when checking -relabelConfig: %s", err)
+		}
+		if err := vminsertcommon.CheckStreamAggrConfig(); err != nil {
+			logger.Fatalf("error when checking -streamAggr.config: %s", err)
+		}
 		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
 		return
 	}

app/vmagent/README.md

@@ -54,10 +54,19 @@ and sending the data to the Prometheus-compatible remote storage:
   The path can point either to local file or to http url. `vmagent` doesn't support some sections of Prometheus config file,
   so you may need either to delete these sections or to run `vmagent` with `-promscrape.config.strictParse=false` command-line flag.
   In this case `vmagent` ignores unsupported sections. See [the list of unsupported sections](#unsupported-prometheus-config-sections).
-* `-remoteWrite.url` with Prometheus-compatible remote storage endpoint such as VictoriaMetrics, the `-remoteWrite.url` argument can be specified
-  multiple times to replicate data concurrently to multiple remote storage systems. See [various use cases](#use-cases).
+* `-remoteWrite.url` with Prometheus-compatible remote storage endpoint such as VictoriaMetrics.
 
-Example command line:
+Example command for writing the data recieved via [supported push-based protocols](#how-to-push-data-to-vmagent)
+to [single-node VictoriaMetrics](https://docs.victoriametrics.com/) located at `victoria-metrics-host:8428`:
+
+```console
+/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) if you need writing
+the data to [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+
+Example command for scraping Prometheus targets and writing the data to single-node VictoriaMetrics:
 
 ```console
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
@@ -68,18 +77,12 @@ See [how to scrape Prometheus-compatible targets](#how-to-collect-metrics-in-pro
 If you use single-node VictoriaMetrics, then you can discover and scrape Prometheus-compatible targets directly from VictoriaMetrics
 without the need to use `vmagent` - see [these docs](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter).
 
-If you don't need to scrape Prometheus-compatible targets, then the `-promscrape.config` option isn't needed.
-For example, the following command is sufficient for accepting data via [supported push-based protocols](#how-to-push-data-to-vmagent)
-and sending it to the provided `-remoteWrite.url`:
-
-```console
-/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
-```
-
-`vmagent` can save network bandwidth usage costs under high load when [VictoriaMetrics remote write protocol is enabled](#victoriametrics-remote-write-protocol).
+`vmagent` can save network bandwidth usage costs under high load when [VictoriaMetrics remote write protocol is used](#victoriametrics-remote-write-protocol).
 
 See [troubleshooting docs](#troubleshooting) if you encounter common issues with `vmagent`.
 
+See [various use cases](#use-cases) for vmagent.
+
 Pass `-help` to `vmagent` in order to see [the full list of supported command-line flags with their descriptions](#advanced-usage).
 
 ## How to push data to vmagent
@@ -101,7 +104,7 @@ additionally to pull-based Prometheus-compatible targets' scraping:
 
 `vmagent` should be restarted in order to update config options set via command-line args.
 `vmagent` supports multiple approaches for reloading configs from updated config files such as
-`-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
+`-promscrape.config`, `-remoteWrite.relabelConfig`, `-remoteWrite.urlRelabelConfig` and `-remoteWrite.streamAggr.config`:
 
 * Sending `SIGHUP` signal to `vmagent` process:
 
@@ -151,6 +154,11 @@ If a single remote storage instance temporarily is out of service, then the coll
 `vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again
 and then it sends the buffered data to the remote storage in order to prevent data gaps.
 
+[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) already supports replication,
+so there is no need in specifying multiple `-remoteWrite.url` flags when writing data to the same cluster.
+See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#replication-and-data-safety).
+
+
 ### Relabeling and filtering
 
 `vmagent` can add, remove or update labels on the collected data before sending it to the remote storage. Additionally,
@@ -637,18 +645,19 @@ provide the following tools for debugging target-level and metric-level relabeli
 - Target-level debugging (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs))
   can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
   and clicking the `debug target relabeling` link at the target, which must be debugged.
-  The opened page will show step-by-step results for the actual target relabeling rules applied to the discovered target labels.
+  The opened page shows step-by-step results for the actual target relabeling rules applied to the discovered target labels.
+  The page shows also the target URL generated after applying all the relabeling rules.
 
   The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target
   is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page
   (`http://victoriametrics:8428/service-discovery` for single-node VictoriaMetrics), find the dropped target
-  and click the `debug` link there. The opened page will show step-by-step results for the actual relabeling rules,
+  and click the `debug` link there. The opened page shows step-by-step results for the actual relabeling rules,
   which result to target drop.
 
 - Metric-level debugging (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)
   can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
   and clicking the `debug metrics relabeling` link at the target, which must be debugged.
-  The opened page will show step-by-step results for the actual metric relabeling rules applied to the given target labels.
+  The opened page shows step-by-step results for the actual metric relabeling rules applied to the given target labels.
 
 ## Prometheus staleness markers
 
@@ -1177,7 +1186,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -denyQueryTracing
      Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/#query-tracing
   -dryRun
-     Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
+     Whether to check config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
@@ -1509,6 +1518,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
      Optional path to file with relabeling configs, which are applied to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling
+  -remoteWrite.keepDanglingQueues
+     Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.
   -remoteWrite.roundDigits array
      Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
      Supports array of values separated by comma or specified via multiple flags.

app/vmagent/main.go

@@ -67,8 +67,8 @@ var (
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	dryRun        = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
-		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
+	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
 
@@ -103,11 +103,14 @@ func main() {
 		return
 	}
 	if *dryRun {
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		}
 		if err := remotewrite.CheckRelabelConfigs(); err != nil {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
-		if err := promscrape.CheckConfig(); err != nil {
-			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
+			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return

app/vmagent/remotewrite/remotewrite.go

@@ -4,17 +4,22 @@ import (
 	"flag"
 	"fmt"
 	"net/url"
+	"os"
+	"path/filepath"
 	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -24,7 +29,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
 )
 
 var (
@@ -36,6 +40,8 @@ var (
 		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
 		"See also -remoteWrite.maxDiskUsagePerURL")
+	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
+		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -94,6 +100,8 @@ var allRelabelConfigs atomic.Value
 // since it may lead to high memory usage due to big number of buffers.
 var maxQueues = cgroup.AvailableCPUs() * 16
 
+const persistentQueueDirname = "persistent-queue"
+
 // InitSecretFlags must be called after flag.Parse and before any logging.
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
@@ -150,9 +158,8 @@ func Init() {
 		logger.Fatalf("cannot load relabel configs: %s", err)
 	}
 	allRelabelConfigs.Store(rcs)
-
-	configSuccess.Set(1)
-	configTimestamp.Set(fasttime.UnixTimestamp())
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
 	if len(*remoteWriteURLs) > 0 {
 		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
@@ -165,34 +172,56 @@ func Init() {
 		for {
 			select {
 			case <-sighupCh:
-			case <-stopCh:
+			case <-configReloaderStopCh:
 				return
 			}
-			configReloads.Inc()
-			logger.Infof("SIGHUP received; reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
-			rcs, err := loadRelabelConfigs()
-			if err != nil {
-				configReloadErrors.Inc()
-				configSuccess.Set(0)
-				logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
-				continue
-			}
-
-			allRelabelConfigs.Store(rcs)
-			configSuccess.Set(1)
-			configTimestamp.Set(fasttime.UnixTimestamp())
-			logger.Infof("Successfully reloaded relabel configs")
+			reloadRelabelConfigs()
+			reloadStreamAggrConfigs()
 		}
 	}()
 }
 
+func reloadRelabelConfigs() {
+	relabelConfigReloads.Inc()
+	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		relabelConfigReloadErrors.Inc()
+		relabelConfigSuccess.Set(0)
+		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+		return
+	}
+	allRelabelConfigs.Store(rcs)
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	logger.Infof("successfully reloaded relabel configs")
+}
+
 var (
-	configReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
-	configReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	configSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
-	configTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
+	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
+	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
+	relabelConfigSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
+	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
+func reloadStreamAggrConfigs() {
+	if len(*remoteWriteMultitenantURLs) > 0 {
+		rwctxsMapLock.Lock()
+		for _, rwctxs := range rwctxsMap {
+			reinitStreamAggr(rwctxs)
+		}
+		rwctxsMapLock.Unlock()
+	} else {
+		reinitStreamAggr(rwctxsDefault)
+	}
+}
+
+func reinitStreamAggr(rwctxs []*remoteWriteCtx) {
+	for _, rwctx := range rwctxs {
+		rwctx.reinitStreamAggr()
+	}
+}
+
 func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
@@ -225,17 +254,47 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		}
 		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
+
+	if !*keepDanglingQueues {
+		// Remove dangling queues, if any.
+		// This is required for the case when the number of queues has been changed or URL have been changed.
+		// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
+		existingQueues := make(map[string]struct{}, len(rwctxs))
+		for _, rwctx := range rwctxs {
+			existingQueues[rwctx.fq.Dirname()] = struct{}{}
+		}
+
+		queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
+		files, err := os.ReadDir(queuesDir)
+		if err != nil {
+			logger.Fatalf("cannot read queues dir %q: %s", queuesDir, err)
+		}
+		removed := 0
+		for _, f := range files {
+			dirname := f.Name()
+			if _, ok := existingQueues[dirname]; !ok {
+				logger.Infof("removing dangling queue %q", dirname)
+				fullPath := filepath.Join(queuesDir, dirname)
+				fs.MustRemoveAll(fullPath)
+				removed++
+			}
+		}
+		if removed > 0 {
+			logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
+		}
+	}
+
 	return rwctxs
 }
 
-var stopCh = make(chan struct{})
+var configReloaderStopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
 // Stop stops remotewrite.
 //
 // It is expected that nobody calls Push during and after the call to this func.
 func Stop() {
-	close(stopCh)
+	close(configReloaderStopCh)
 	configReloaderWG.Wait()
 
 	for _, rwctx := range rwctxsDefault {
@@ -450,7 +509,7 @@ type remoteWriteCtx struct {
 	fq  *persistentqueue.FastQueue
 	c   *client
 
-	sas                 *streamaggr.Aggregators
+	sas                 atomic.Pointer[streamaggr.Aggregators]
 	streamAggrKeepInput bool
 
 	pss        []*pendingSeries
@@ -466,7 +525,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	pqURL.RawQuery = ""
 	pqURL.Fragment = ""
 	h := xxhash.Sum64([]byte(pqURL.String()))
-	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
 	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -515,10 +574,12 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(argIdx, 0)
 		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
 		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggrFile=%q: %s", sasFile, err)
+			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
 		}
-		rwctx.sas = sas
+		rwctx.sas.Store(sas)
 		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
 	}
 
 	return rwctx
@@ -533,8 +594,10 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.UnblockAllReaders()
 	rwctx.c.MustStop()
 	rwctx.c = nil
-	rwctx.sas.MustStop()
-	rwctx.sas = nil
+
+	sas := rwctx.sas.Swap(nil)
+	sas.MustStop()
+
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
@@ -565,8 +628,9 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 
 	// Apply stream aggregation if any
-	rwctx.sas.Push(tss)
-	if rwctx.sas == nil || rwctx.streamAggrKeepInput {
+	sas := rwctx.sas.Load()
+	sas.Push(tss)
+	if sas == nil || rwctx.streamAggrKeepInput {
 		// Push samples to the remote storage
 		rwctx.pushInternal(tss)
 	}
@@ -585,6 +649,36 @@ func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
 	pss[idx].Push(tss)
 }
 
+func (rwctx *remoteWriteCtx) reinitStreamAggr() {
+	sas := rwctx.sas.Load()
+	if sas == nil {
+		// There is no stream aggregation for rwctx
+		return
+	}
+
+	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
+	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
+	dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(rwctx.idx, 0)
+	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
+		logger.Errorf("cannot reload stream aggregation config from -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", sasFile, err)
+		return
+	}
+	if !sasNew.Equal(sas) {
+		sasOld := rwctx.sas.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded stream aggregation configs at -remoteWrite.streamAggr.config=%q", sasFile)
+	} else {
+		sasNew.MustStop()
+		logger.Infof("the config at -remoteWrite.streamAggr.config=%q wasn't changed", sasFile)
+	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
+}
+
 var tssRelabelPool = &sync.Pool{
 	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}
@@ -599,3 +693,20 @@ func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 	}
 	return rowsCount
 }
+
+// CheckStreamAggrConfigs checks configs pointed by -remoteWrite.streamAggr.config
+func CheckStreamAggrConfigs() error {
+	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
+	for idx, sasFile := range *streamAggrConfig {
+		if sasFile == "" {
+			continue
+		}
+		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(idx, 0)
+		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, dedupInterval)
+		if err != nil {
+			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)
+		}
+		sas.MustStop()
+	}
+	return nil
+}

app/vmalert/README.md

@@ -916,7 +916,7 @@ The shortlist of configuration flags is the following:
   -evaluationInterval duration
      How often to evaluate the rules (default 1m0s)
   -external.alert.source string
-     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.
+     External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.
   -external.label array
      Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
      Supports an array of values separated by comma or specified via multiple flags.

app/vmalert/config/config.go

@@ -10,9 +10,9 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -199,9 +199,17 @@ func (r *Rule) Validate() error {
 // ValidateTplFn must validate the given annotations
 type ValidateTplFn func(annotations map[string]string) error
 
+// cLogger is a logger with support of logs suppressing.
+// it is used when logs emitted by config package needs
+// to be suppressed.
+var cLogger = &log.Logger{}
+
 // ParseSilent parses rule configs from given file patterns without emitting logs
 func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns, true)
+	cLogger.Suppress(true)
+	defer cLogger.Suppress(false)
+
+	files, err := readFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %s", err)
 	}
@@ -210,7 +218,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns, false)
+	files, err := readFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %s", err)
 	}
@@ -219,7 +227,7 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio
 		return nil, fmt.Errorf("failed to parse %s: %s", pathPatterns, err)
 	}
 	if len(groups) < 1 {
-		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+		cLogger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}
 	return groups, nil
 }

app/vmalert/config/fs.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 // FS represent a file system abstract for reading files.
@@ -36,10 +36,9 @@ var (
 // readFromFS returns an error if at least one FS failed to init.
 // The function can be called multiple times but each unique path
 // will be inited only once.
-// If silent == true, readFromFS will not emit any logs.
 //
 // It is allowed to mix different FS types in path list.
-func readFromFS(paths []string, silent bool) (map[string][]byte, error) {
+func readFromFS(paths []string) (map[string][]byte, error) {
 	var err error
 	result := make(map[string][]byte)
 	for _, path := range paths {
@@ -65,18 +64,19 @@ func readFromFS(paths []string, silent bool) (map[string][]byte, error) {
 			return nil, fmt.Errorf("failed to list files from %q", fs)
 		}
 
-		if !silent {
-			logger.Infof("found %d files to read from %q", len(list), fs)
-		}
+		cLogger.Infof("found %d files to read from %q", len(list), fs)
 
 		if len(list) < 1 {
 			continue
 		}
 
+		ts := time.Now()
 		files, err := fs.Read(list)
 		if err != nil {
 			return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
 		}
+		cLogger.Infof("finished reading %d files in %v from %q", len(list), time.Since(ts), fs)
+
 		for k, v := range files {
 			if _, ok := result[k]; ok {
 				return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k)

app/vmalert/config/log/logger.go

@@ -0,0 +1,59 @@
+package log
+
+import (
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// Logger is using lib/logger for logging
+// but can be suppressed via Suppress method
+type Logger struct {
+	mu       sync.RWMutex
+	disabled bool
+}
+
+// Suppress whether to ignore message logging.
+// Once suppressed, logging continues to be ignored
+// until logger is un-suppressed.
+func (l *Logger) Suppress(v bool) {
+	l.mu.Lock()
+	l.disabled = v
+	l.mu.Unlock()
+}
+
+func (l *Logger) isDisabled() bool {
+	l.mu.RLock()
+	defer l.mu.RUnlock()
+	return l.disabled
+}
+
+// Errorf logs error message.
+func (l *Logger) Errorf(format string, args ...interface{}) {
+	if l.isDisabled() {
+		return
+	}
+	logger.Errorf(format, args...)
+}
+
+// Warnf logs warning message.
+func (l *Logger) Warnf(format string, args ...interface{}) {
+	if l.isDisabled() {
+		return
+	}
+	logger.Warnf(format, args...)
+}
+
+// Infof logs info message.
+func (l *Logger) Infof(format string, args ...interface{}) {
+	if l.isDisabled() {
+		return
+	}
+	logger.Infof(format, args...)
+}
+
+// Panicf logs panic message and panics.
+// Panicf can't be suppressed
+func (l *Logger) Panicf(format string, args ...interface{}) {
+	logger.Panicf(format, args...)
+}

app/vmalert/config/log/logger_test.go

@@ -0,0 +1,54 @@
+package log
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+func TestOutput(t *testing.T) {
+	testOutput := &bytes.Buffer{}
+	logger.SetOutputForTests(testOutput)
+	defer logger.ResetOutputForTest()
+
+	log := &Logger{}
+
+	mustMatch := func(exp string) {
+		t.Helper()
+		if exp == "" {
+			if testOutput.String() != "" {
+				t.Errorf("expected output to be empty; got %q", testOutput.String())
+				return
+			}
+		}
+		if !strings.Contains(testOutput.String(), exp) {
+			t.Errorf("output %q should contain %q", testOutput.String(), exp)
+		}
+		fmt.Println(testOutput.String())
+		testOutput.Reset()
+	}
+
+	log.Warnf("foo")
+	mustMatch("foo")
+
+	log.Infof("info %d", 2)
+	mustMatch("info 2")
+
+	log.Errorf("error %s %d", "baz", 5)
+	mustMatch("error baz 5")
+
+	log.Suppress(true)
+
+	log.Warnf("foo")
+	mustMatch("")
+
+	log.Infof("info %d", 2)
+	mustMatch("")
+
+	log.Errorf("error %q %d", "baz", 5)
+	mustMatch("")
+
+}

app/vmalert/main.go

@@ -73,7 +73,8 @@ absolute path to all .tpl files in root.`)
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
 		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
 		`Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . `+
-		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . `+
+		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
+		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"Pass multiple -label flags in order to add multiple label sets.")
@@ -319,6 +320,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 	// init reload metrics with positive values to improve alerting conditions
 	configSuccess.Set(1)
 	configTimestamp.Set(fasttime.UnixTimestamp())
+	parseFn := config.Parse
 	for {
 		select {
 		case <-ctx.Done():
@@ -330,7 +332,11 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			}
 			logger.Infof("SIGHUP received. Going to reload rules %q %s...", *rulePath, tmplMsg)
 			configReloads.Inc()
+			// allow logs emitting during manual config reload
+			parseFn = config.Parse
 		case <-configCheckCh:
+			// disable logs emitting during per-interval config reload
+			parseFn = config.ParseSilent
 		}
 		if err := notifier.Reload(); err != nil {
 			configReloadErrors.Inc()
@@ -345,7 +351,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to load new templates: %s", err)
 			continue
 		}
-		newGroupsCfg, err := config.ParseSilent(*rulePath, validateTplFn, *validateExpressions)
+		newGroupsCfg, err := parseFn(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
 			configReloadErrors.Inc()
 			configSuccess.Set(0)

app/vmalert/notifier/alert.go

@@ -111,11 +111,7 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
 		ActiveAt: a.ActiveAt,
 		For:      a.For,
 	}
-	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
-	if err != nil {
-		return nil, fmt.Errorf("error getting a template: %w", err)
-	}
-	return templateAnnotations(annotations, tplData, tmpl, true)
+	return ExecTemplate(q, annotations, tplData)
 }
 
 // ExecTemplate executes the given template for given annotations map.
@@ -174,6 +170,8 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
 	if err != nil {
 		return fmt.Errorf("error cloning template before parse annotation: %w", err)
 	}
+	// Clone() doesn't copy tpl Options, so we set them manually
+	tpl = tpl.Option("missingkey=zero")
 	tpl, err = tpl.Parse(text)
 	if err != nil {
 		return fmt.Errorf("error parsing annotation template: %w", err)

app/vmalert/notifier/alert_test.go

@@ -200,6 +200,9 @@ func TestAlert_ExecTemplate(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
+			if err := ValidateTemplates(tc.annotations); err != nil {
+				t.Fatal(err)
+			}
 			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
 			if err != nil {
 				t.Fatal(err)

app/vmalert/notifier/init.go

@@ -81,10 +81,6 @@ var (
 //
 // Init returns an error if both mods are used.
 func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
-	if externalLabels != nil || externalURL != "" {
-		return nil, fmt.Errorf("BUG: notifier.Init was called multiple times")
-	}
-
 	externalURL = extURL
 	externalLabels = extLabels
 	eu, err := url.Parse(externalURL)

app/vmalert/notifier/init_test.go

@@ -0,0 +1,37 @@
+package notifier
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"testing"
+)
+
+func TestInit(t *testing.T) {
+	oldAddrs := *addrs
+	defer func() { *addrs = oldAddrs }()
+
+	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
+
+	fn, err := Init(nil, nil, "")
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+
+	nfs := fn()
+	if len(nfs) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(nfs))
+	}
+
+	targets := GetTargets()
+	if targets == nil || targets[TargetStatic] == nil {
+		t.Fatalf("expected to get static targets in response")
+	}
+
+	nf1 := targets[TargetStatic][0]
+	if nf1.Addr() != "127.0.0.1/api/v2/alerts" {
+		t.Fatalf("expected to get \"127.0.0.1/api/v2/alerts\"; got %q instead", nf1.Addr())
+	}
+	nf2 := targets[TargetStatic][1]
+	if nf2.Addr() != "127.0.0.2/api/v2/alerts" {
+		t.Fatalf("expected to get \"127.0.0.2/api/v2/alerts\"; got %q instead", nf2.Addr())
+	}
+}

app/vmalert/remotewrite/init_test.go

@@ -0,0 +1,20 @@
+package remotewrite
+
+import (
+	"context"
+	"testing"
+)
+
+func TestInit(t *testing.T) {
+	oldAddr := *addr
+	defer func() { *addr = oldAddr }()
+
+	*addr = "http://localhost:8428"
+	cl, err := Init(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := cl.Close(); err != nil {
+		t.Fatal(err)
+	}
+}

app/vmalert/templates/template_test.go

@@ -76,6 +76,20 @@ func TestTemplateFuncs(t *testing.T) {
 	formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
 	formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
 	formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
+
+	formatting("humanize", float64(127087), "127.1k")
+	formatting("humanize", float64(136458627186688), "136.5T")
+
+	formatting("humanizeDuration", 1, "1s")
+	formatting("humanizeDuration", 0.2, "200ms")
+	formatting("humanizeDuration", 42000, "11h 40m 0s")
+	formatting("humanizeDuration", 16790555, "194d 8h 2m 35s")
+
+	formatting("humanizePercentage", 1, "100%")
+	formatting("humanizePercentage", 0.8, "80%")
+	formatting("humanizePercentage", 0.015, "1.5%")
+
+	formatting("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
 }
 
 func mkTemplate(current, replacement interface{}) textTemplate {

app/vmalert/web.qtpl

@@ -52,7 +52,7 @@
         {% for _, g := range groups  %}
               <div class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
-                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                  {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
                 <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.ID] %}</span>
                 <p class="fs-6 fw-lighter">{%s g.File %}</p>

app/vmalert/web.qtpl.go

@@ -227,7 +227,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 //line app/vmalert/web.qtpl:55
 			qw422016.N().FPrec(g.Interval, 0)
 //line app/vmalert/web.qtpl:55
-			qw422016.N().S(`s)</a>
+			qw422016.N().S(`s) #</a>
                  `)
 //line app/vmalert/web.qtpl:56
 			if rNotOk[g.ID] > 0 {

app/vmalert/web_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptest"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 )
@@ -19,9 +20,18 @@ func TestHandler(t *testing.T) {
 		},
 		state: newRuleState(10),
 	}
+	ar.state.add(ruleStateEntry{
+		time:    time.Now(),
+		at:      time.Now(),
+		samples: 10,
+	})
+	rr := &RecordingRule{
+		Name:  "record",
+		state: newRuleState(10),
+	}
 	g := &Group{
 		Name:  "group",
-		Rules: []Rule{ar},
+		Rules: []Rule{ar, rr},
 	}
 	m := &manager{groups: make(map[uint64]*Group)}
 	m.groups[0] = g
@@ -62,6 +72,14 @@ func TestHandler(t *testing.T) {
 	t.Run("/vmalert/rule", func(t *testing.T) {
 		a := ar.ToAPI()
 		getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+		r := rr.ToAPI()
+		getResp(ts.URL+"/vmalert/"+r.WebLink(), nil, 200)
+	})
+	t.Run("/vmalert/alert", func(t *testing.T) {
+		alerts := ar.AlertsToAPI()
+		for _, a := range alerts {
+			getResp(ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
+		}
 	})
 	t.Run("/vmalert/rule?badParam", func(t *testing.T) {
 		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramRuleID)

app/vmauth/README.md

@@ -17,7 +17,12 @@ and pass the following flag to `vmauth` binary in order to start authorizing and
 After that `vmauth` starts accepting HTTP requests on port `8427` and routing them according to the provided [-auth.config](#auth-config).
 The port can be modified via `-httpListenAddr` command-line flag.
 
-The auth config can be reloaded either by passing `SIGHUP` signal to `vmauth` or by querying `/-/reload` http endpoint.
+The auth config can be reloaded via the following ways:
+
+- By passing `SIGHUP` signal to `vmauth`.
+- By querying `/-/reload` http endpoint. This endpoint can be protected with `-reloadAuthKey` command-line flag. See [security docs](#security) for more details.
+- By specifying `-configCheckInterval` command-line flag to the interval between config re-reads. For example, `-configCheckInterval=5s` will re-read the config
+  and apply new changes every 5 seconds.
 
 Docker images for `vmauth` are available [here](https://hub.docker.com/r/victoriametrics/vmauth/tags).
 
@@ -260,6 +265,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
 
   -auth.config string
      Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
+  -configCheckInterval duration
+     Interval for config file re-read. Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable

app/vmauth/auth_config.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -24,6 +25,8 @@ import (
 var (
 	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
 		"See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config")
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "interval for config file re-read. "+
+		"Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.")
 )
 
 // AuthConfig represents auth config.
@@ -305,10 +308,20 @@ func stopAuthConfig() {
 }
 
 func authConfigReloader(sighupCh <-chan os.Signal) {
+	var refreshCh <-chan time.Time
+	// initialize auth refresh interval
+	if *configCheckInterval > 0 {
+		ticker := time.NewTicker(*configCheckInterval)
+		defer ticker.Stop()
+		refreshCh = ticker.C
+	}
+
 	for {
 		select {
 		case <-stopCh:
 			return
+		case <-refreshCh:
+			procutil.SelfSIGHUP()
 		case <-sighupCh:
 			logger.Infof("SIGHUP received; loading -auth.config=%q", *authConfigPath)
 			m, err := readAuthConfig(*authConfigPath)

app/vmbackup/Makefile

@@ -39,6 +39,9 @@ vmbackup-freebsd-amd64-prod:
 vmbackup-openbsd-amd64-prod:
 	APP_NAME=vmbackup $(MAKE) app-via-docker-openbsd-amd64
 
+vmbackup-windows-amd64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-windows-amd64
+
 package-vmbackup:
 	APP_NAME=vmbackup $(MAKE) package-via-docker
 
@@ -93,5 +96,8 @@ vmbackup-freebsd-amd64:
 vmbackup-openbsd-amd64:
 	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
+vmbackup-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmbackup $(MAKE) app-local-windows-goarch
+
 vmbackup-pure:
 	APP_NAME=vmbackup $(MAKE) app-local-pure

app/vmbackup/main.go

@@ -49,6 +49,12 @@ func main() {
 	logger.Init()
 	pushmetrics.Init()
 
+	// Storing snapshot delete function to be able to call it in case
+	// of error since logger.Fatal will exit the program without
+	// calling deferred functions.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2055
+	deleteSnapshot := func() {}
+
 	if len(*snapshotCreateURL) > 0 {
 		// create net/url object
 		createURL, err := url.Parse(*snapshotCreateURL)
@@ -80,32 +86,48 @@ func main() {
 			logger.Fatalf("cannot set snapshotName flag: %v", err)
 		}
 
-		defer func() {
+		deleteSnapshot = func() {
 			err := snapshot.Delete(deleteURL.String(), name)
 			if err != nil {
 				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
-		}()
+		}
 	} else if len(*snapshotName) == 0 {
 		logger.Fatalf("`-snapshotName` or `-snapshot.createURL` must be provided")
 	}
-	if err := snapshot.Validate(*snapshotName); err != nil {
-		logger.Fatalf("invalid -snapshotName=%q: %s", *snapshotName, err)
-	}
 
 	go httpserver.Serve(*httpListenAddr, false, nil)
 
+	err := makeBackup()
+	deleteSnapshot()
+	if err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+
+	startTime := time.Now()
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http server for metrics: %s", err)
+	}
+	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func makeBackup() error {
+	if err := snapshot.Validate(*snapshotName); err != nil {
+		return fmt.Errorf("invalid -snapshotName=%q: %s", *snapshotName, err)
+	}
+
 	srcFS, err := newSrcFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	dstFS, err := newDstFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	originFS, err := newOriginFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	a := &actions.Backup{
 		Concurrency: *concurrency,
@@ -114,18 +136,12 @@ func main() {
 		Origin:      originFS,
 	}
 	if err := a.Run(); err != nil {
-		logger.Fatalf("cannot create backup: %s", err)
+		return err
 	}
 	srcFS.MustStop()
 	dstFS.MustStop()
 	originFS.MustStop()
-
-	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
-		logger.Fatalf("cannot stop http server for metrics: %s", err)
-	}
-	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
+	return nil
 }
 
 func usage() {

app/vmctl/README.md

@@ -781,7 +781,25 @@ To avoid such situation try to filter out VM process metrics via `--vm-native-fi
 4. `vmctl` doesn't provide relabeling or other types of labels management in this mode.
 Instead, use [relabeling in VictoriaMetrics](https://github.com/VictoriaMetrics/vmctl/issues/4#issuecomment-683424375).
 5. When importing in or from cluster version remember to use correct [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
-and specify `accountID` param.
+and specify `accountID` param. Example formats:
+
+```console
+# Migrating from cluster to single
+--vm-native-src-addr=http://<src-vmselect>:8481/select/0/prometheus
+--vm-native-dst-addr=http://<dst-vmsingle>:8428
+
+ # Migrating from single to cluster
+--vm-native-src-addr=http://<src-vmsingle>:8428
+--vm-native-src-addr=http://<dst-vminsert>:8480/insert/0/prometheus
+
+# Migrating single to single
+--vm-native-src-addr=http://<src-vmsingle>:8428
+--vm-native-dst-addr=http://<dst-vmsingle>:8428
+
+# Migrating cluster to cluster
+--vm-native-src-addr=http://<src-vmselect>:8481/select/0/prometheus
+--vm-native-dst-addr=http://<dst-vminsert>:8480/insert/0/prometheus
+```
 6. When migrating large volumes of data it might be useful to use `--vm-native-step-interval` flag to split single process into smaller steps.
 7. `vmctl` supports `--vm-concurrency` which controls the number of concurrent workers that process the input from source query results.
 Please note that each import request can load up to a single vCPU core on VictoriaMetrics. So try to set it according

app/vmctl/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/terminal"
 	"github.com/urfave/cli/v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
@@ -71,7 +72,7 @@ func main() {
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency))
-					return otsdbProcessor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
+					return otsdbProcessor.run(isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -112,7 +113,7 @@ func main() {
 						c.String(influxMeasurementFieldSeparator),
 						c.Bool(influxSkipDatabaseLabel),
 						c.Bool(influxPrometheusMode))
-					return processor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
+					return processor.run(isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -152,7 +153,7 @@ func main() {
 						},
 						cc: c.Int(remoteReadConcurrency),
 					}
-					return rmp.run(ctx, c.Bool(globalSilent), c.Bool(globalVerbose))
+					return rmp.run(ctx, isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -186,7 +187,7 @@ func main() {
 						im: importer,
 						cc: c.Int(promConcurrency),
 					}
-					return pp.run(c.Bool(globalSilent), c.Bool(globalVerbose))
+					return pp.run(isNonInteractive(c), c.Bool(globalVerbose))
 				},
 			},
 			{
@@ -244,7 +245,7 @@ func main() {
 						backoff: backoff.New(),
 						cc:      c.Int(vmConcurrency),
 					}
-					return p.run(ctx, c.Bool(globalSilent))
+					return p.run(ctx, isNonInteractive(c))
 				},
 			},
 			{
@@ -317,3 +318,8 @@ func initConfigVM(c *cli.Context) vm.Config {
 		DisableProgressBar: c.Bool(vmDisableProgressBar),
 	}
 }
+
+func isNonInteractive(c *cli.Context) bool {
+	isTerminal := terminal.IsTerminal(int(os.Stdout.Fd()))
+	return c.Bool(globalSilent) || !isTerminal
+}

app/vmctl/terminal/terminal.go

@@ -0,0 +1,6 @@
+package terminal
+
+// IsTerminal returns true if the file descriptor is terminal
+func IsTerminal(fd int) bool {
+	return isTerminal(fd)
+}

app/vmctl/terminal/unix.go

@@ -0,0 +1,13 @@
+//go:build aix || linux || solaris || zos
+// +build aix linux solaris zos
+
+package terminal
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TCGETS
+
+func isTerminal(fd int) bool {
+	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	return err == nil
+}

app/vmctl/terminal/unix_bsd.go

@@ -0,0 +1,13 @@
+//go:build darwin || freebsd || openbsd
+// +build darwin freebsd openbsd
+
+package terminal
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TIOCGETA
+
+func isTerminal(fd int) bool {
+	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	return err == nil
+}

app/vmctl/terminal/windows..go

@@ -0,0 +1,8 @@
+//go:build windows
+// +build windows
+
+package terminal
+
+func isTerminal(fd int) bool {
+	return true
+}

app/vmctl/testdata/servers_integration_test/remote_write_server.go

@@ -2,39 +2,70 @@ package remote_read_integration
 
 import (
 	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"sort"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 )
 
+// LabelValues represents series from api/v1/series response
+type LabelValues map[string]string
+
+// Response represents response from api/v1/series
+type Response struct {
+	Status string        `json:"status"`
+	Series []LabelValues `json:"data"`
+}
+
+// RemoteWriteServer represents fake remote write server with database
 type RemoteWriteServer struct {
-	server *httptest.Server
-	series []vm.TimeSeries
+	server         *httptest.Server
+	series         []vm.TimeSeries
+	expectedSeries []vm.TimeSeries
 }
 
 // NewRemoteWriteServer prepares test remote write server
 func NewRemoteWriteServer(t *testing.T) *RemoteWriteServer {
 	rws := &RemoteWriteServer{series: make([]vm.TimeSeries, 0)}
 	mux := http.NewServeMux()
+
 	mux.Handle("/api/v1/import", rws.getWriteHandler(t))
 	mux.Handle("/health", rws.handlePing())
+	mux.Handle("/api/v1/series", rws.seriesHandler())
+	mux.Handle("/api/v1/export/native", rws.exportNativeHandler())
+	mux.Handle("/api/v1/import/native", rws.importNativeHandler(t))
 	rws.server = httptest.NewServer(mux)
 	return rws
 }
 
-// Close closes the server.
+// Close closes the server
 func (rws *RemoteWriteServer) Close() {
 	rws.server.Close()
 }
 
-func (rws *RemoteWriteServer) ExpectedSeries(series []vm.TimeSeries) {
+// Series saves generated series for fake database
+func (rws *RemoteWriteServer) Series(series []vm.TimeSeries) {
 	rws.series = append(rws.series, series...)
 }
 
+// ExpectedSeries saves expected results to check in the handler
+func (rws *RemoteWriteServer) ExpectedSeries(series []vm.TimeSeries) {
+	rws.expectedSeries = append(rws.expectedSeries, series...)
+}
+
+// URL returns server url
 func (rws *RemoteWriteServer) URL() string {
 	return rws.server.URL
 }
@@ -68,13 +99,14 @@ func (rws *RemoteWriteServer) getWriteHandler(t *testing.T) http.Handler {
 			rows.Reset()
 		}
 
-		if !reflect.DeepEqual(tss, rws.series) {
+		if !reflect.DeepEqual(tss, rws.expectedSeries) {
 			w.WriteHeader(http.StatusInternalServerError)
-			t.Fatalf("datasets not equal, expected: %#v; \n got: %#v", rws.series, tss)
+			t.Fatalf("datasets not equal, expected: %#v; \n got: %#v", rws.expectedSeries, tss)
 			return
 		}
 
 		w.WriteHeader(http.StatusNoContent)
+		return
 	})
 }
 
@@ -84,3 +116,146 @@ func (rws *RemoteWriteServer) handlePing() http.Handler {
 		_, _ = w.Write([]byte("OK"))
 	})
 }
+
+func (rws *RemoteWriteServer) seriesHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var labelValues []LabelValues
+		for _, ser := range rws.series {
+			metricNames := make(LabelValues)
+			if ser.Name != "" {
+				metricNames["__name__"] = ser.Name
+			}
+			for _, p := range ser.LabelPairs {
+				metricNames[p.Name] = p.Value
+			}
+			labelValues = append(labelValues, metricNames)
+		}
+
+		resp := Response{
+			Status: "success",
+			Series: labelValues,
+		}
+
+		err := json.NewEncoder(w).Encode(resp)
+		if err != nil {
+			log.Printf("error send series: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+	})
+}
+
+func (rws *RemoteWriteServer) exportNativeHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		now := time.Now()
+		err := prometheus.ExportNativeHandler(now, w, r)
+		if err != nil {
+			log.Printf("error export series via native protocol: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+		return
+	})
+}
+
+func (rws *RemoteWriteServer) importNativeHandler(t *testing.T) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		common.StartUnmarshalWorkers()
+		defer common.StopUnmarshalWorkers()
+
+		var gotTimeSeries []vm.TimeSeries
+
+		err := stream.Parse(r.Body, false, func(block *stream.Block) error {
+			mn := &block.MetricName
+			var timeseries vm.TimeSeries
+			timeseries.Name = string(mn.MetricGroup)
+			timeseries.Timestamps = append(timeseries.Timestamps, block.Timestamps...)
+			timeseries.Values = append(timeseries.Values, block.Values...)
+
+			for i := range mn.Tags {
+				tag := &mn.Tags[i]
+				timeseries.LabelPairs = append(timeseries.LabelPairs, vm.LabelPair{
+					Name:  string(tag.Key),
+					Value: string(tag.Value),
+				})
+			}
+
+			gotTimeSeries = append(gotTimeSeries, timeseries)
+
+			return nil
+		})
+		if err != nil {
+			log.Printf("error parse stream blocks: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		// got timeseries should be sorted
+		// because they are processed independently
+		sort.SliceStable(gotTimeSeries, func(i, j int) bool {
+			iv, jv := gotTimeSeries[i], gotTimeSeries[j]
+			switch {
+			case iv.Values[0] != jv.Values[0]:
+				return iv.Values[0] < jv.Values[0]
+			case iv.Timestamps[0] != jv.Timestamps[0]:
+				return iv.Timestamps[0] < jv.Timestamps[0]
+			default:
+				return iv.Name < jv.Name
+			}
+		})
+
+		if !reflect.DeepEqual(gotTimeSeries, rws.expectedSeries) {
+			w.WriteHeader(http.StatusInternalServerError)
+			t.Fatalf("datasets not equal, expected: %#v;\n got: %#v", rws.expectedSeries, gotTimeSeries)
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+		return
+	})
+}
+
+// GenerateVNSeries generates test timeseries
+func GenerateVNSeries(start, end, numOfSeries, numOfSamples int64) []vm.TimeSeries {
+	var ts []vm.TimeSeries
+	j := 0
+	for i := 0; i < int(numOfSeries); i++ {
+		if i%3 == 0 {
+			j++
+		}
+
+		timeSeries := vm.TimeSeries{
+			Name: fmt.Sprintf("vm_metric_%d", j),
+			LabelPairs: []vm.LabelPair{
+				{Name: "job", Value: strconv.Itoa(i)},
+			},
+		}
+
+		ts = append(ts, timeSeries)
+	}
+
+	for i := range ts {
+		t, v := generateTimeStampsAndValues(i, start, end, numOfSamples)
+		ts[i].Timestamps = t
+		ts[i].Values = v
+	}
+
+	return ts
+}
+
+func generateTimeStampsAndValues(idx int, startTime, endTime, numOfSamples int64) ([]int64, []float64) {
+	delta := (endTime - startTime) / numOfSamples
+
+	var timestamps []int64
+	var values []float64
+	t := startTime
+	for t != endTime {
+		v := 100 * int64(idx)
+		timestamps = append(timestamps, t*1000)
+		values = append(values, float64(v))
+		t = t + delta
+	}
+
+	return timestamps, values
+}

app/vmctl/vm_native.go

@@ -14,6 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/cheggaaa/pb/v3"
 )
 
@@ -232,6 +233,13 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	// any error breaks the import
 	for s := range metrics {
+
+		match, err := buildMatchWithFilter(p.filter.Match, s)
+		if err != nil {
+			logger.Errorf("failed to build export filters: %s", err)
+			continue
+		}
+
 		for _, times := range ranges {
 			select {
 			case <-ctx.Done():
@@ -239,7 +247,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			case infErr := <-errCh:
 				return fmt.Errorf("native error: %s", infErr)
 			case filterCh <- native.Filter{
-				Match:     fmt.Sprintf("{%s=%q}", nameLabel, s),
+				Match:     match,
 				TimeStart: times[0].Format(time.RFC3339),
 				TimeEnd:   times[1].Format(time.RFC3339),
 			}:
@@ -303,3 +311,13 @@ func byteCountSI(b int64) string {
 	return fmt.Sprintf("%.1f %cB",
 		float64(b)/float64(div), "kMGTPE"[exp])
 }
+
+func buildMatchWithFilter(filter string, metricName string) (string, error) {
+	labels, err := promutils.NewLabelsFromString(filter)
+	if err != nil {
+		return "", err
+	}
+	labels.Set("__name__", metricName)
+
+	return labels.String(), nil
+}

app/vmctl/vm_native_test.go

@@ -2,118 +2,360 @@ package main
 
 import (
 	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
+	remote_read_integration "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
-// If you want to run this test:
-// 1. run two instances of victoriametrics and define -httpListenAddr for both or just for second instance
-// 2. define srcAddr and dstAddr const with your victoriametrics addresses
-// 3. define matchFilter const with your importing data
-// 4. define timeStartFilter
-// 5. run each test one by one
-
 const (
-	matchFilter     = `{job="avalanche"}`
-	timeStartFilter = "2020-01-01T20:07:00Z"
-	timeEndFilter   = "2020-08-01T20:07:00Z"
-	srcAddr         = "http://127.0.0.1:8428"
-	dstAddr         = "http://127.0.0.1:8528"
+	storagePath     = "TestStorage"
+	retentionPeriod = "100y"
 )
 
-// This test simulates close process if user abort it
 func Test_vmNativeProcessor_run(t *testing.T) {
-	t.Skip()
+
+	processFlags()
+	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
+	defer func() {
+		vmstorage.Stop()
+		if err := os.RemoveAll(storagePath); err != nil {
+			log.Fatalf("cannot remove %q: %s", storagePath, err)
+		}
+	}()
+
 	type fields struct {
-		filter    native.Filter
-		rateLimit int64
-		dst       *native.Client
-		src       *native.Client
+		filter       native.Filter
+		dst          *native.Client
+		src          *native.Client
+		backoff      *backoff.Backoff
+		s            *stats
+		rateLimit    int64
+		interCluster bool
+		cc           int
+		matchName    string
+		matchValue   string
 	}
+	type args struct {
+		ctx    context.Context
+		silent bool
+	}
+
 	tests := []struct {
-		name    string
-		fields  fields
-		closer  func(cancelFunc context.CancelFunc)
-		wantErr bool
+		name           string
+		fields         fields
+		args           args
+		vmSeries       func(start, end, numOfSeries, numOfSamples int64) []vm.TimeSeries
+		expectedSeries []vm.TimeSeries
+		start          string
+		end            string
+		numOfSamples   int64
+		numOfSeries    int64
+		chunk          string
+		wantErr        bool
 	}{
 		{
-			name: "simulate syscall.SIGINT",
+			name:         "step minute on minute time range",
+			start:        "2022-11-25T11:23:05+02:00",
+			end:          "2022-11-27T11:24:05+02:00",
+			numOfSamples: 2,
+			numOfSeries:  3,
+			chunk:        stepper.StepMinute,
 			fields: fields{
-				filter: native.Filter{
-					Match:     matchFilter,
-					TimeStart: timeStartFilter,
+				filter:       native.Filter{},
+				backoff:      backoff.New(),
+				rateLimit:    0,
+				interCluster: false,
+				cc:           1,
+				matchName:    "__name__",
+				matchValue:   ".*",
+			},
+			args: args{
+				ctx:    context.Background(),
+				silent: true,
+			},
+			vmSeries: remote_read_integration.GenerateVNSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1669368185000, 1669454615000},
+					Values:     []float64{0, 0},
 				},
-				rateLimit: 0,
-				dst: &native.Client{
-					Addr: dstAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1669368185000, 1669454615000},
+					Values:     []float64{100, 100},
 				},
-				src: &native.Client{
-					Addr: srcAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1669368185000, 1669454615000},
+					Values:     []float64{200, 200},
 				},
 			},
-			closer: func(cancelFunc context.CancelFunc) {
-				time.Sleep(time.Second * 5)
-				cancelFunc()
-			},
-			wantErr: true,
-		},
-		{
-			name: "simulate correct work",
-			fields: fields{
-				filter: native.Filter{
-					Match:     matchFilter,
-					TimeStart: timeStartFilter,
-				},
-				rateLimit: 0,
-				dst: &native.Client{
-					Addr: dstAddr,
-				},
-				src: &native.Client{
-					Addr: srcAddr,
-				},
-			},
-			closer:  func(cancelFunc context.CancelFunc) {},
 			wantErr: false,
 		},
 		{
-			name: "simulate correct work with chunking",
+			name:         "step month on month time range",
+			start:        "2022-09-26T11:23:05+02:00",
+			end:          "2022-11-26T11:24:05+02:00",
+			numOfSamples: 2,
+			numOfSeries:  3,
+			chunk:        stepper.StepMonth,
 			fields: fields{
-				filter: native.Filter{
-					Match:     matchFilter,
-					TimeStart: timeStartFilter,
-					TimeEnd:   timeEndFilter,
-					Chunk:     stepper.StepMonth,
+				filter:       native.Filter{},
+				backoff:      backoff.New(),
+				rateLimit:    0,
+				interCluster: false,
+				cc:           1,
+				matchName:    "__name__",
+				matchValue:   ".*",
+			},
+			args: args{
+				ctx:    context.Background(),
+				silent: true,
+			},
+			vmSeries: remote_read_integration.GenerateVNSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{0},
 				},
-				rateLimit: 0,
-				dst: &native.Client{
-					Addr: dstAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{0},
 				},
-				src: &native.Client{
-					Addr: srcAddr,
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{200},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{200},
 				},
 			},
-			closer:  func(cancelFunc context.CancelFunc) {},
 			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			ctx, cancelFn := context.WithCancel(context.Background())
-			p := &vmNativeProcessor{
-				filter:    tt.fields.filter,
-				rateLimit: tt.fields.rateLimit,
-				dst:       tt.fields.dst,
-				src:       tt.fields.src,
+			src := remote_read_integration.NewRemoteWriteServer(t)
+			dst := remote_read_integration.NewRemoteWriteServer(t)
+
+			defer func() {
+				src.Close()
+				dst.Close()
+			}()
+
+			start, err := time.Parse(time.RFC3339, tt.start)
+			if err != nil {
+				t.Fatalf("Error parse start time: %s", err)
 			}
 
-			tt.closer(cancelFn)
+			end, err := time.Parse(time.RFC3339, tt.end)
+			if err != nil {
+				t.Fatalf("Error parse end time: %s", err)
+			}
 
-			if err := p.run(ctx, true); (err != nil) != tt.wantErr {
+			tt.fields.filter.Match = fmt.Sprintf("%s=%q", tt.fields.matchName, tt.fields.matchValue)
+			tt.fields.filter.TimeStart = tt.start
+			tt.fields.filter.TimeEnd = tt.end
+
+			rws := tt.vmSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
+
+			src.Series(rws)
+			dst.ExpectedSeries(tt.expectedSeries)
+
+			if err := fillStorage(rws); err != nil {
+				t.Fatalf("error add series to storage: %s", err)
+			}
+
+			tt.fields.src = &native.Client{
+				AuthCfg:              nil,
+				Addr:                 src.URL(),
+				ExtraLabels:          []string{},
+				DisableHTTPKeepAlive: false,
+			}
+			tt.fields.dst = &native.Client{
+				AuthCfg:              nil,
+				Addr:                 dst.URL(),
+				ExtraLabels:          []string{},
+				DisableHTTPKeepAlive: false,
+			}
+
+			p := &vmNativeProcessor{
+				filter:       tt.fields.filter,
+				dst:          tt.fields.dst,
+				src:          tt.fields.src,
+				backoff:      tt.fields.backoff,
+				s:            tt.fields.s,
+				rateLimit:    tt.fields.rateLimit,
+				interCluster: tt.fields.interCluster,
+				cc:           tt.fields.cc,
+			}
+
+			if err := p.run(tt.args.ctx, tt.args.silent); (err != nil) != tt.wantErr {
 				t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr)
 			}
+			deleted, err := deleteSeries(tt.fields.matchName, tt.fields.matchValue)
+			if err != nil {
+				t.Fatalf("error delete series: %s", err)
+			}
+			if int64(deleted) != tt.numOfSeries {
+				t.Fatalf("expected deleted series %d; got deleted series %d", tt.numOfSeries, deleted)
+			}
+		})
+	}
+}
+
+func processFlags() {
+	flag.Parse()
+	for _, fv := range []struct {
+		flag  string
+		value string
+	}{
+		{flag: "storageDataPath", value: storagePath},
+		{flag: "retentionPeriod", value: retentionPeriod},
+	} {
+		// panics if flag doesn't exist
+		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
+			log.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
+		}
+	}
+}
+
+func fillStorage(series []vm.TimeSeries) error {
+	var mrs []storage.MetricRow
+	for _, series := range series {
+		var labels []prompb.Label
+		for _, lp := range series.LabelPairs {
+			labels = append(labels, prompb.Label{Name: []byte(lp.Name), Value: []byte(lp.Value)})
+		}
+		if series.Name != "" {
+			labels = append(labels, prompb.Label{Name: []byte("__name__"), Value: []byte(series.Name)})
+		}
+		mr := storage.MetricRow{}
+		mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
+
+		timestamps := series.Timestamps
+		values := series.Values
+		for i, value := range values {
+			mr.Timestamp = timestamps[i]
+			mr.Value = value
+			mrs = append(mrs, mr)
+		}
+	}
+
+	if err := vmstorage.AddRows(mrs); err != nil {
+		return fmt.Errorf("unexpected error in AddRows: %s", err)
+	}
+	vmstorage.Storage.DebugFlush()
+	return nil
+}
+
+func deleteSeries(name, value string) (int, error) {
+	tfs := storage.NewTagFilters()
+	if err := tfs.Add([]byte(name), []byte(value), false, true); err != nil {
+		return 0, fmt.Errorf("unexpected error in TagFilters.Add: %w", err)
+	}
+	return vmstorage.DeleteSeries(nil, []*storage.TagFilters{tfs})
+}
+
+func Test_buildMatchWithFilter(t *testing.T) {
+	tests := []struct {
+		name       string
+		filter     string
+		metricName string
+		want       string
+		wantErr    bool
+	}{
+		{
+			name:       "parsed metric with label",
+			filter:     `{__name__="http_request_count_total",cluster="kube1"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster="kube1"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "metric name with label",
+			filter:     `http_request_count_total{cluster="kube1"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster="kube1"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "parsed metric with regexp value",
+			filter:     `{__name__="http_request_count_total",cluster~="kube.*"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster~="kube.*"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "only label with regexp",
+			filter:     `{cluster~=".*"}`,
+			metricName: "http_request_count_total",
+			want:       `{cluster~=".*",__name__="http_request_count_total"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "many labels in filter with regexp",
+			filter:     `{cluster~=".*",job!=""}`,
+			metricName: "http_request_count_total",
+			want:       `{cluster~=".*",job!="",__name__="http_request_count_total"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "match with error",
+			filter:     `{cluster=~".*"}`,
+			metricName: "http_request_count_total",
+			want:       ``,
+			wantErr:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildMatchWithFilter(tt.filter, tt.metricName)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("buildMatchWithFilter() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("buildMatchWithFilter() got = %v, want %v", got, tt.want)
+			}
 		})
 	}
 }

app/vminsert/common/insert_ctx.go

@@ -137,7 +137,8 @@ func (ctx *InsertCtx) ApplyRelabeling() {
 
 // FlushBufs flushes buffered rows to the underlying storage.
 func (ctx *InsertCtx) FlushBufs() error {
-	if sa != nil && !ctx.skipStreamAggr {
+	sas := sasGlobal.Load()
+	if sas != nil && !ctx.skipStreamAggr {
 		ctx.streamAggrCtx.push(ctx.mrs)
 		if !*streamAggrKeepInput {
 			ctx.Reset(0)

app/vminsert/common/streamaggr.go

@@ -2,15 +2,20 @@ package common
 
 import (
 	"flag"
+	"fmt"
+	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -24,28 +29,97 @@ var (
 		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
 )
 
+var (
+	saCfgReloaderStopCh = make(chan struct{})
+	saCfgReloaderWG     sync.WaitGroup
+
+	saCfgReloads   = metrics.NewCounter(`vminsert_streamagg_config_reloads_total`)
+	saCfgReloadErr = metrics.NewCounter(`vminsert_streamagg_config_reloads_errors_total`)
+	saCfgSuccess   = metrics.NewCounter(`vminsert_streamagg_config_last_reload_successful`)
+	saCfgTimestamp = metrics.NewCounter(`vminsert_streamagg_config_last_reload_success_timestamp_seconds`)
+
+	sasGlobal atomic.Pointer[streamaggr.Aggregators]
+)
+
+// CheckStreamAggrConfig checks config pointed by -stramaggr.config
+func CheckStreamAggrConfig() error {
+	if *streamAggrConfig == "" {
+		return nil
+	}
+	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
+	sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushNoop, *streamAggrDedupInterval)
+	if err != nil {
+		return fmt.Errorf("error when loading -streamAggr.config=%q: %w", *streamAggrConfig, err)
+	}
+	sas.MustStop()
+	return nil
+}
+
 // InitStreamAggr must be called after flag.Parse and before using the common package.
 //
 // MustStopStreamAggr must be called when stream aggr is no longer needed.
 func InitStreamAggr() {
 	if *streamAggrConfig == "" {
-		// Nothing to initialize
 		return
 	}
-	a, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, *streamAggrDedupInterval)
+
+	sighupCh := procutil.NewSighupChan()
+
+	sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, *streamAggrDedupInterval)
 	if err != nil {
 		logger.Fatalf("cannot load -streamAggr.config=%q: %s", *streamAggrConfig, err)
 	}
-	sa = a
+	sasGlobal.Store(sas)
+	saCfgSuccess.Set(1)
+	saCfgTimestamp.Set(fasttime.UnixTimestamp())
+
+	// Start config reloader.
+	saCfgReloaderWG.Add(1)
+	go func() {
+		defer saCfgReloaderWG.Done()
+		for {
+			select {
+			case <-sighupCh:
+			case <-saCfgReloaderStopCh:
+				return
+			}
+			reloadStreamAggrConfig()
+		}
+	}()
+}
+
+func reloadStreamAggrConfig() {
+	logger.Infof("reloading -streamAggr.config=%q", *streamAggrConfig)
+	saCfgReloads.Inc()
+
+	sasNew, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, *streamAggrDedupInterval)
+	if err != nil {
+		saCfgSuccess.Set(0)
+		saCfgReloadErr.Inc()
+		logger.Errorf("cannot reload -streamAggr.config=%q: use the previously loaded config; error: %s", *streamAggrConfig, err)
+		return
+	}
+	sas := sasGlobal.Load()
+	if !sasNew.Equal(sas) {
+		sasOld := sasGlobal.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded stream aggregation config at -streamAggr.config=%q", *streamAggrConfig)
+	} else {
+		logger.Infof("nothing changed in -streamAggr.config=%q", *streamAggrConfig)
+		sasNew.MustStop()
+	}
+	saCfgSuccess.Set(1)
+	saCfgTimestamp.Set(fasttime.UnixTimestamp())
 }
 
 // MustStopStreamAggr stops stream aggregators.
 func MustStopStreamAggr() {
-	sa.MustStop()
-	sa = nil
-}
+	close(saCfgReloaderStopCh)
+	saCfgReloaderWG.Wait()
 
-var sa *streamaggr.Aggregators
+	sas := sasGlobal.Swap(nil)
+	sas.MustStop()
+}
 
 type streamAggrCtx struct {
 	mn  storage.MetricName
@@ -64,6 +138,7 @@ func (ctx *streamAggrCtx) push(mrs []storage.MetricRow) {
 	ts := &tss[0]
 	labels := ts.Labels
 	samples := ts.Samples
+	sas := sasGlobal.Load()
 	for _, mr := range mrs {
 		if err := mn.UnmarshalRaw(mr.MetricNameRaw); err != nil {
 			logger.Panicf("BUG: cannot unmarshal recently marshaled MetricName: %s", err)
@@ -88,7 +163,7 @@ func (ctx *streamAggrCtx) push(mrs []storage.MetricRow) {
 		ts.Labels = labels
 		ts.Samples = samples
 
-		sa.Push(tss)
+		sas.Push(tss)
 	}
 }
 

app/vminsert/relabel/relabel.go

@@ -71,6 +71,12 @@ var (
 
 var pcsGlobal atomic.Value
 
+// CheckRelabelConfig checks config pointed by -relabelConfig
+func CheckRelabelConfig() error {
+	_, err := loadRelabelConfig()
+	return err
+}
+
 func loadRelabelConfig() (*promrelabel.ParsedConfigs, error) {
 	if len(*relabelConfig) == 0 {
 		return nil, nil

app/vmrestore/Makefile

@@ -39,6 +39,9 @@ vmrestore-freebsd-amd64-prod:
 vmrestore-openbsd-amd64-prod:
 	APP_NAME=vmrestore $(MAKE) app-via-docker-openbsd-amd64
 
+vmrestore-windows-amd64-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-windows-amd64
+
 package-vmrestore:
 	APP_NAME=vmrestore $(MAKE) package-via-docker
 
@@ -93,5 +96,8 @@ vmrestore-freebsd-amd64:
 vmrestore-openbsd-amd64:
 	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
+vmrestore-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmrestore $(MAKE) app-local-windows-goarch
+
 vmrestore-pure:
 	APP_NAME=vmrestore $(MAKE) app-local-pure

app/vmselect/graphite/aggr.go

@@ -0,0 +1,259 @@
+package graphite
+
+import (
+	"fmt"
+	"math"
+	"strings"
+	"sync"
+
+	"github.com/valyala/histogram"
+)
+
+var aggrFuncs = map[string]aggrFunc{
+	"average":  aggrAvg,
+	"avg":      aggrAvg,
+	"avg_zero": aggrAvgZero,
+	"median":   aggrMedian,
+	"sum":      aggrSum,
+	"total":    aggrSum,
+	"min":      aggrMin,
+	"max":      aggrMax,
+	"diff":     aggrDiff,
+	"pow":      aggrPow,
+	"stddev":   aggrStddev,
+	"count":    aggrCount,
+	"range":    aggrRange,
+	"rangeOf":  aggrRange,
+	"multiply": aggrMultiply,
+	"first":    aggrFirst,
+	"last":     aggrLast,
+	"current":  aggrLast,
+}
+
+func getAggrFunc(funcName string) (aggrFunc, error) {
+	s := strings.TrimSuffix(funcName, "Series")
+	aggrFunc := aggrFuncs[s]
+	if aggrFunc == nil {
+		return nil, fmt.Errorf("unsupported aggregate function %q", funcName)
+	}
+	return aggrFunc, nil
+}
+
+type aggrFunc func(values []float64) float64
+
+func (af aggrFunc) apply(xFilesFactor float64, values []float64) float64 {
+	if aggrCount(values) >= float64(len(values))*xFilesFactor {
+		return af(values)
+	}
+	return nan
+}
+
+func aggrAvg(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	sum := values[pos]
+	count := 1
+	for _, v := range values[pos+1:] {
+		if !math.IsNaN(v) {
+			sum += v
+			count++
+		}
+	}
+	return sum / float64(count)
+}
+
+func aggrAvgZero(values []float64) float64 {
+	if len(values) == 0 {
+		return nan
+	}
+	sum := float64(0)
+	for _, v := range values {
+		if !math.IsNaN(v) {
+			sum += v
+		}
+	}
+	return sum / float64(len(values))
+}
+
+var aggrMedian = newAggrFuncPercentile(50)
+
+func aggrSum(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	sum := values[pos]
+	for _, v := range values[pos+1:] {
+		if !math.IsNaN(v) {
+			sum += v
+		}
+	}
+	return sum
+}
+
+func aggrMin(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	min := values[pos]
+	for _, v := range values[pos+1:] {
+		if !math.IsNaN(v) && v < min {
+			min = v
+		}
+	}
+	return min
+}
+
+func aggrMax(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	max := values[pos]
+	for _, v := range values[pos+1:] {
+		if !math.IsNaN(v) && v > max {
+			max = v
+		}
+	}
+	return max
+}
+
+func aggrDiff(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	sum := float64(0)
+	for _, v := range values[pos+1:] {
+		if !math.IsNaN(v) {
+			sum += v
+		}
+	}
+	return values[pos] - sum
+}
+
+func aggrPow(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	pow := values[pos]
+	for _, v := range values[pos+1:] {
+		if !math.IsNaN(v) {
+			pow = math.Pow(pow, v)
+		}
+	}
+	return pow
+}
+
+func aggrStddev(values []float64) float64 {
+	avg := aggrAvg(values)
+	if math.IsNaN(avg) {
+		return nan
+	}
+	sum := float64(0)
+	count := 0
+	for _, v := range values {
+		if !math.IsNaN(v) {
+			d := avg - v
+			sum += d * d
+			count++
+		}
+	}
+	return math.Sqrt(sum / float64(count))
+}
+
+func aggrCount(values []float64) float64 {
+	count := 0
+	for _, v := range values {
+		if !math.IsNaN(v) {
+			count++
+		}
+	}
+	return float64(count)
+}
+
+func aggrRange(values []float64) float64 {
+	min := aggrMin(values)
+	if math.IsNaN(min) {
+		return nan
+	}
+	max := aggrMax(values)
+	return max - min
+}
+
+func aggrMultiply(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	p := values[pos]
+	for _, v := range values[pos+1:] {
+		if !math.IsNaN(v) {
+			p *= v
+		}
+	}
+	return p
+}
+
+func aggrFirst(values []float64) float64 {
+	pos := getFirstNonNaNPos(values)
+	if pos < 0 {
+		return nan
+	}
+	return values[pos]
+}
+
+func aggrLast(values []float64) float64 {
+	for i := len(values) - 1; i >= 0; i-- {
+		v := values[i]
+		if !math.IsNaN(v) {
+			return v
+		}
+	}
+	return nan
+}
+
+func getFirstNonNaNPos(values []float64) int {
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			return i
+		}
+	}
+	return -1
+}
+
+var nan = math.NaN()
+
+func newAggrFuncPercentile(n float64) aggrFunc {
+	f := func(values []float64) float64 {
+		h := getHistogram()
+		for _, v := range values {
+			if !math.IsNaN(v) {
+				h.Update(v)
+			}
+		}
+		p := h.Quantile(n / 100)
+		putHistogram(h)
+		return p
+	}
+	return f
+}
+
+func getHistogram() *histogram.Fast {
+	return histogramPool.Get().(*histogram.Fast)
+}
+
+func putHistogram(h *histogram.Fast) {
+	h.Reset()
+	histogramPool.Put(h)
+}
+
+var histogramPool = &sync.Pool{
+	New: func() interface{} {
+		return histogram.NewFast()
+	},
+}

app/vmselect/graphite/aggr_state.go

@@ -0,0 +1,724 @@
+package graphite
+
+import (
+	"fmt"
+	"math"
+	"strings"
+
+	"github.com/valyala/histogram"
+)
+
+var aggrStateFuncs = map[string]func(int) aggrState{
+	"average":  newAggrStateAvg,
+	"avg":      newAggrStateAvg,
+	"avg_zero": newAggrStateAvgZero,
+	"median":   newAggrStateMedian,
+	"sum":      newAggrStateSum,
+	"total":    newAggrStateSum,
+	"min":      newAggrStateMin,
+	"max":      newAggrStateMax,
+	"diff":     newAggrStateDiff,
+	"pow":      newAggrStatePow,
+	"stddev":   newAggrStateStddev,
+	"count":    newAggrStateCount,
+	"range":    newAggrStateRange,
+	"rangeOf":  newAggrStateRange,
+	"multiply": newAggrStateMultiply,
+	"first":    newAggrStateFirst,
+	"last":     newAggrStateLast,
+	"current":  newAggrStateLast,
+}
+
+type aggrState interface {
+	Update(values []float64)
+	Finalize(xFilesFactor float64) []float64
+}
+
+func newAggrState(pointsLen int, funcName string) (aggrState, error) {
+	s := strings.TrimSuffix(funcName, "Series")
+	asf := aggrStateFuncs[s]
+	if asf == nil {
+		return nil, fmt.Errorf("unsupported aggregate function %q", funcName)
+	}
+	return asf(pointsLen), nil
+}
+
+type aggrStateAvg struct {
+	pointsLen   int
+	sums        []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateAvg(pointsLen int) aggrState {
+	return &aggrStateAvg{
+		pointsLen: pointsLen,
+		sums:      make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateAvg) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	sums := as.sums
+	counts := as.counts
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			sums[i] += v
+			counts[i]++
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateAvg) Finalize(xFilesFactor float64) []float64 {
+	sums := as.sums
+	counts := as.counts
+	values := make([]float64, as.pointsLen)
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = sums[i] / float64(count)
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateAvgZero struct {
+	pointsLen   int
+	sums        []float64
+	seriesTotal int
+}
+
+func newAggrStateAvgZero(pointsLen int) aggrState {
+	return &aggrStateAvgZero{
+		pointsLen: pointsLen,
+		sums:      make([]float64, pointsLen),
+	}
+}
+
+func (as *aggrStateAvgZero) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	sums := as.sums
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			sums[i] += v
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateAvgZero) Finalize(xFilesFactor float64) []float64 {
+	sums := as.sums
+	values := make([]float64, as.pointsLen)
+	count := float64(as.seriesTotal)
+	for i, sum := range sums {
+		v := nan
+		if count > 0 {
+			v = sum / count
+		}
+		values[i] = v
+	}
+	return values
+}
+
+func newAggrStateMedian(pointsLen int) aggrState {
+	return newAggrStatePercentile(pointsLen, 50)
+}
+
+type aggrStatePercentile struct {
+	phi         float64
+	pointsLen   int
+	hs          []*histogram.Fast
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStatePercentile(pointsLen int, n float64) aggrState {
+	hs := make([]*histogram.Fast, pointsLen)
+	for i := 0; i < pointsLen; i++ {
+		hs[i] = histogram.NewFast()
+	}
+	return &aggrStatePercentile{
+		phi:       n / 100,
+		pointsLen: pointsLen,
+		hs:        hs,
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStatePercentile) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	hs := as.hs
+	counts := as.counts
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			hs[i].Update(v)
+			counts[i]++
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStatePercentile) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	hs := as.hs
+	for i, count := range as.counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = hs[i].Quantile(as.phi)
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateSum struct {
+	pointsLen   int
+	sums        []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateSum(pointsLen int) aggrState {
+	return &aggrStateSum{
+		pointsLen: pointsLen,
+		sums:      make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateSum) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	sums := as.sums
+	counts := as.counts
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			sums[i] += v
+			counts[i]++
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateSum) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	sums := as.sums
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = sums[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateMin struct {
+	pointsLen   int
+	mins        []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateMin(pointsLen int) aggrState {
+	return &aggrStateMin{
+		pointsLen: pointsLen,
+		mins:      make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateMin) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	mins := as.mins
+	counts := as.counts
+	for i, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		counts[i]++
+		if counts[i] == 1 {
+			mins[i] = v
+		} else if v < mins[i] {
+			mins[i] = v
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateMin) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	mins := as.mins
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = mins[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateMax struct {
+	pointsLen   int
+	maxs        []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateMax(pointsLen int) aggrState {
+	return &aggrStateMax{
+		pointsLen: pointsLen,
+		maxs:      make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateMax) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	maxs := as.maxs
+	counts := as.counts
+	for i, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		counts[i]++
+		if counts[i] == 1 {
+			maxs[i] = v
+		} else if v > maxs[i] {
+			maxs[i] = v
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateMax) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	maxs := as.maxs
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = maxs[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateDiff struct {
+	pointsLen   int
+	vs          []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateDiff(pointsLen int) aggrState {
+	return &aggrStateDiff{
+		pointsLen: pointsLen,
+		vs:        make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateDiff) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	vs := as.vs
+	counts := as.counts
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			if counts[i] == 0 {
+				vs[i] = v
+			} else {
+				vs[i] -= v
+			}
+			counts[i]++
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateDiff) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	vs := as.vs
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = vs[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStatePow struct {
+	pointsLen   int
+	vs          []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStatePow(pointsLen int) aggrState {
+	return &aggrStatePow{
+		pointsLen: pointsLen,
+		vs:        make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStatePow) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	vs := as.vs
+	counts := as.counts
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			if counts[i] == 0 {
+				vs[i] = v
+			} else {
+				vs[i] = math.Pow(vs[i], v)
+			}
+			counts[i]++
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStatePow) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	vs := as.vs
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = vs[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateStddev struct {
+	pointsLen   int
+	means       []float64
+	m2s         []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateStddev(pointsLen int) aggrState {
+	return &aggrStateStddev{
+		pointsLen: pointsLen,
+		means:     make([]float64, pointsLen),
+		m2s:       make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateStddev) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	means := as.means
+	m2s := as.m2s
+	counts := as.counts
+	for i, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		// See https://en.m.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
+		count := counts[i]
+		mean := means[i]
+		count++
+		delta := v - mean
+		mean += delta / float64(count)
+		delta2 := v - mean
+		means[i] = mean
+		m2s[i] += delta * delta2
+		counts[i] = count
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateStddev) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	m2s := as.m2s
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = math.Sqrt(m2s[i] / float64(count))
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateCount struct {
+	pointsLen   int
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateCount(pointsLen int) aggrState {
+	return &aggrStateCount{
+		pointsLen: pointsLen,
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateCount) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	counts := as.counts
+	for i, v := range values {
+		if !math.IsNaN(v) {
+			counts[i]++
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateCount) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = float64(count)
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateRange struct {
+	pointsLen   int
+	mins        []float64
+	maxs        []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateRange(pointsLen int) aggrState {
+	return &aggrStateRange{
+		pointsLen: pointsLen,
+		mins:      make([]float64, pointsLen),
+		maxs:      make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateRange) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	mins := as.mins
+	maxs := as.maxs
+	counts := as.counts
+	for i, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		counts[i]++
+		if counts[i] == 1 {
+			mins[i] = v
+			maxs[i] = v
+		} else if v < mins[i] {
+			mins[i] = v
+		} else if v > maxs[i] {
+			maxs[i] = v
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateRange) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	mins := as.mins
+	maxs := as.maxs
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = maxs[i] - mins[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateMultiply struct {
+	pointsLen   int
+	ms          []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateMultiply(pointsLen int) aggrState {
+	return &aggrStateMultiply{
+		pointsLen: pointsLen,
+		ms:        make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateMultiply) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	ms := as.ms
+	counts := as.counts
+	for i, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		counts[i]++
+		if counts[i] == 1 {
+			ms[i] = v
+		} else {
+			ms[i] *= v
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateMultiply) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	ms := as.ms
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = ms[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateFirst struct {
+	pointsLen   int
+	vs          []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateFirst(pointsLen int) aggrState {
+	return &aggrStateFirst{
+		pointsLen: pointsLen,
+		vs:        make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateFirst) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	vs := as.vs
+	counts := as.counts
+	for i, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		counts[i]++
+		if counts[i] == 1 {
+			vs[i] = v
+		}
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateFirst) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	vs := as.vs
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = vs[i]
+		}
+		values[i] = v
+	}
+	return values
+}
+
+type aggrStateLast struct {
+	pointsLen   int
+	vs          []float64
+	counts      []int
+	seriesTotal int
+}
+
+func newAggrStateLast(pointsLen int) aggrState {
+	return &aggrStateLast{
+		pointsLen: pointsLen,
+		vs:        make([]float64, pointsLen),
+		counts:    make([]int, pointsLen),
+	}
+}
+
+func (as *aggrStateLast) Update(values []float64) {
+	if len(values) != as.pointsLen {
+		panic(fmt.Errorf("BUG: unexpected number of points in values; got %d; want %d", len(values), as.pointsLen))
+	}
+	vs := as.vs
+	counts := as.counts
+	for i, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		vs[i] = v
+		counts[i]++
+	}
+	as.seriesTotal++
+}
+
+func (as *aggrStateLast) Finalize(xFilesFactor float64) []float64 {
+	xff := int(xFilesFactor * float64(as.seriesTotal))
+	values := make([]float64, as.pointsLen)
+	vs := as.vs
+	counts := as.counts
+	for i, count := range counts {
+		v := nan
+		if count > 0 && count >= xff {
+			v = vs[i]
+		}
+		values[i] = v
+	}
+	return values
+}

app/vmselect/graphite/eval.go

@@ -0,0 +1,210 @@
+package graphite
+
+import (
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+)
+
+var maxGraphiteSeries = flag.Int("search.maxGraphiteSeries", 300e3, "The maximum number of time series, which can be scanned during queries to Graphite Render API. "+
+	"See https://docs.victoriametrics.com/#graphite-render-api-usage")
+
+type evalConfig struct {
+	startTime   int64
+	endTime     int64
+	storageStep int64
+	deadline    searchutils.Deadline
+
+	currentTime time.Time
+
+	// xFilesFactor is used for determining when consolidateFunc must be applied.
+	//
+	// 0 means that consolidateFunc should be applied if at least a single non-NaN data point exists on the given step.
+	// 1 means that consolidateFunc should be applied if all the data points are non-NaN on the given step.
+	xFilesFactor float64
+
+	// Enforced tag filters
+	etfs [][]storage.TagFilter
+
+	// originalQuery contains the original query - used for debug logging.
+	originalQuery string
+}
+
+func (ec *evalConfig) pointsLen(step int64) int {
+	return int((ec.endTime - ec.startTime) / step)
+}
+
+func (ec *evalConfig) newTimestamps(step int64) []int64 {
+	pointsLen := ec.pointsLen(step)
+	timestamps := make([]int64, pointsLen)
+	ts := ec.startTime
+	for i := 0; i < pointsLen; i++ {
+		timestamps[i] = ts
+		ts += step
+	}
+	return timestamps
+}
+
+type series struct {
+	Name       string
+	Tags       map[string]string
+	Timestamps []int64
+	Values     []float64
+
+	// holds current path expression like graphite does.
+	pathExpression string
+
+	expr graphiteql.Expr
+
+	// consolidateFunc is applied to raw samples in order to generate data points algined to the given step.
+	// see series.consolidate() function for details.
+	consolidateFunc aggrFunc
+
+	// xFilesFactor is used for determining when consolidateFunc must be applied.
+	//
+	// 0 means that consolidateFunc should be applied if at least a single non-NaN data point exists on the given step.
+	// 1 means that consolidateFunc should be applied if all the data points are non-NaN on the given step.
+	xFilesFactor float64
+
+	step int64
+}
+
+func (s *series) consolidate(ec *evalConfig, step int64) {
+	aggrFunc := s.consolidateFunc
+	if aggrFunc == nil {
+		aggrFunc = aggrAvg
+	}
+	xFilesFactor := s.xFilesFactor
+	if s.xFilesFactor <= 0 {
+		xFilesFactor = ec.xFilesFactor
+	}
+	s.summarize(aggrFunc, ec.startTime, ec.endTime, step, xFilesFactor)
+}
+
+func (s *series) summarize(aggrFunc aggrFunc, startTime, endTime, step int64, xFilesFactor float64) {
+	pointsLen := int((endTime - startTime) / step)
+	timestamps := s.Timestamps
+	values := s.Values
+	dstTimestamps := make([]int64, 0, pointsLen)
+	dstValues := make([]float64, 0, pointsLen)
+	ts := startTime
+	i := 0
+	for len(dstTimestamps) < pointsLen {
+		tsEnd := ts + step
+		j := i
+		for j < len(timestamps) && timestamps[j] < tsEnd {
+			j++
+		}
+		if i == j && i > 0 && ts-timestamps[i-1] <= 2000 {
+			// The current [ts ... tsEnd) interval has no samples,
+			// but the last sample on the previous interval [ts - step ... ts)
+			// is closer than 2 seconds to the current interval.
+			// Let's consider that this sample belongs to the current interval,
+			// since such discrepancy could appear because of small jitter in samples' ingestion.
+			i--
+		}
+		v := aggrFunc.apply(xFilesFactor, values[i:j])
+		dstTimestamps = append(dstTimestamps, ts)
+		dstValues = append(dstValues, v)
+		ts = tsEnd
+		i = j
+	}
+	// Do not reuse s.Timestamps and s.Values, since they can be too big
+	s.Timestamps = dstTimestamps
+	s.Values = dstValues
+	s.step = step
+}
+
+func execExpr(ec *evalConfig, query string) (nextSeriesFunc, error) {
+	expr, err := graphiteql.Parse(query)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse %q: %w", query, err)
+	}
+	return evalExpr(ec, expr)
+}
+
+func evalExpr(ec *evalConfig, expr graphiteql.Expr) (nextSeriesFunc, error) {
+	switch t := expr.(type) {
+	case *graphiteql.MetricExpr:
+		return evalMetricExpr(ec, t)
+	case *graphiteql.FuncExpr:
+		return evalFuncExpr(ec, t)
+	default:
+		return nil, fmt.Errorf("unexpected expression type %T; want graphiteql.MetricExpr or graphiteql.FuncExpr; expr: %q", t, t.AppendString(nil))
+	}
+}
+
+func evalMetricExpr(ec *evalConfig, me *graphiteql.MetricExpr) (nextSeriesFunc, error) {
+	tfs := []storage.TagFilter{{
+		Key:   []byte("__graphite__"),
+		Value: []byte(me.Query),
+	}}
+	tfss := joinTagFilterss(tfs, ec.etfs)
+	sq := storage.NewSearchQuery(ec.startTime, ec.endTime, tfss, *maxGraphiteSeries)
+	return newNextSeriesForSearchQuery(ec, sq, me)
+}
+
+func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr graphiteql.Expr) (nextSeriesFunc, error) {
+	rss, err := netstorage.ProcessSearchQuery(nil, sq, ec.deadline)
+	if err != nil {
+		return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
+	}
+	seriesCh := make(chan *series, cgroup.AvailableCPUs())
+	errCh := make(chan error, 1)
+	go func() {
+		err := rss.RunParallel(nil, func(rs *netstorage.Result, workerID uint) error {
+			nameWithTags := getCanonicalPath(&rs.MetricName)
+			tags := unmarshalTags(nameWithTags)
+			s := &series{
+				Name:           tags["name"],
+				Tags:           tags,
+				Timestamps:     append([]int64{}, rs.Timestamps...),
+				Values:         append([]float64{}, rs.Values...),
+				expr:           expr,
+				pathExpression: string(expr.AppendString(nil)),
+			}
+			s.summarize(aggrAvg, ec.startTime, ec.endTime, ec.storageStep, 0)
+			t := timerpool.Get(30 * time.Second)
+			select {
+			case seriesCh <- s:
+			case <-t.C:
+				logger.Errorf("resource leak when processing the %s (full query: %s); please report this error to VictoriaMetrics developers",
+					expr.AppendString(nil), ec.originalQuery)
+			}
+			timerpool.Put(t)
+			return nil
+		})
+		close(seriesCh)
+		errCh <- err
+	}()
+	f := func() (*series, error) {
+		s := <-seriesCh
+		if s != nil {
+			return s, nil
+		}
+		err := <-errCh
+		return nil, err
+	}
+	return f, nil
+}
+
+func evalFuncExpr(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, error) {
+	// Do not lowercase the fe.FuncName, since Graphite function names are case-sensitive.
+	tf := transformFuncs[fe.FuncName]
+	if tf == nil {
+		return nil, fmt.Errorf("unknown function %q", fe.FuncName)
+	}
+	nextSeries, err := tf(ec, fe)
+	if err != nil {
+		return nil, fmt.Errorf("cannot evaluate %s: %w", fe.AppendString(nil), err)
+	}
+	return nextSeries, nil
+}

app/vmselect/graphite/functions_api.go

@@ -0,0 +1,88 @@
+package graphite
+
+import (
+	// embed functions.json file
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+)
+
+// FunctionsHandler implements /functions handler.
+//
+// See https://graphite.readthedocs.io/en/latest/functions.html#function-api
+func FunctionsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	grouped := searchutils.GetBool(r, "grouped")
+	group := r.FormValue("group")
+	result := make(map[string]interface{})
+	for funcName, fi := range funcs {
+		if group != "" && fi.Group != group {
+			continue
+		}
+		if grouped {
+			v := result[fi.Group]
+			if v == nil {
+				v = make(map[string]*funcInfo)
+				result[fi.Group] = v
+			}
+			m := v.(map[string]*funcInfo)
+			m[funcName] = fi
+		} else {
+			result[funcName] = fi
+		}
+	}
+	return writeJSON(result, w, r)
+}
+
+// FunctionDetailsHandler implements /functions/<func_name> handler.
+//
+// See https://graphite.readthedocs.io/en/latest/functions.html#function-api
+func FunctionDetailsHandler(startTime time.Time, funcName string, w http.ResponseWriter, r *http.Request) error {
+	result := funcs[funcName]
+	if result == nil {
+		return fmt.Errorf("cannot find function %q", funcName)
+	}
+	return writeJSON(result, w, r)
+}
+
+func writeJSON(result interface{}, w http.ResponseWriter, r *http.Request) error {
+	data, err := json.Marshal(result)
+	if err != nil {
+		return fmt.Errorf("cannot marshal response to JSON: %w", err)
+	}
+	jsonp := r.FormValue("jsonp")
+	contentType := getContentType(jsonp)
+	w.Header().Set("Content-Type", contentType)
+	if jsonp != "" {
+		fmt.Fprintf(w, "%s(", jsonp)
+	}
+	w.Write(data)
+	if jsonp != "" {
+		fmt.Fprintf(w, ")")
+	}
+	return nil
+}
+
+//go:embed functions.json
+var funcsJSON []byte
+
+type funcInfo struct {
+	Name        string          `json:"name"`
+	Function    string          `json:"function"`
+	Description string          `json:"description"`
+	Module      string          `json:"module"`
+	Group       string          `json:"group"`
+	Params      json.RawMessage `json:"params"`
+}
+
+var funcs = func() map[string]*funcInfo {
+	var m map[string]*funcInfo
+	if err := json.Unmarshal(funcsJSON, &m); err != nil {
+		// Do not use logger.Panicf, since it isn't ready yet.
+		panic(fmt.Errorf("cannot parse funcsJSON: %s", err))
+	}
+	return m
+}()

app/vmselect/graphite/natural_compare.go

@@ -0,0 +1,48 @@
+package graphite
+
+import (
+	"strconv"
+)
+
+func naturalLess(a, b string) bool {
+	for {
+		var aPrefix, bPrefix string
+		aPrefix, a = getNonNumPrefix(a)
+		bPrefix, b = getNonNumPrefix(b)
+		if aPrefix != bPrefix {
+			return aPrefix < bPrefix
+		}
+		if len(a) == 0 || len(b) == 0 {
+			return a < b
+		}
+		var aNum, bNum int
+		aNum, a = getNumPrefix(a)
+		bNum, b = getNumPrefix(b)
+		if aNum != bNum {
+			return aNum < bNum
+		}
+	}
+}
+
+func getNonNumPrefix(s string) (prefix string, tail string) {
+	for i := 0; i < len(s); i++ {
+		ch := s[i]
+		if ch >= '0' && ch <= '9' {
+			return s[:i], s[i:]
+		}
+	}
+	return s, ""
+}
+
+func getNumPrefix(s string) (prefix int, tail string) {
+	i := 0
+	for i < len(s) {
+		ch := s[i]
+		if ch < '0' || ch > '9' {
+			break
+		}
+		i++
+	}
+	prefix, _ = strconv.Atoi(s[:i])
+	return prefix, s[i:]
+}

app/vmselect/graphite/natural_compare_test.go

@@ -0,0 +1,29 @@
+package graphite
+
+import (
+	"testing"
+)
+
+func TestNaturalLess(t *testing.T) {
+	f := func(a, b string, okExpected bool) {
+		t.Helper()
+		ok := naturalLess(a, b)
+		if ok != okExpected {
+			t.Fatalf("unexpected result for naturalLess(%q, %q); got %v; want %v", a, b, ok, okExpected)
+		}
+	}
+	f("", "", false)
+	f("a", "b", true)
+	f("", "foo", true)
+	f("foo", "", false)
+	f("foo", "foo", false)
+	f("b", "a", false)
+	f("1", "2", true)
+	f("10", "2", false)
+	f("foo100", "foo12", false)
+	f("foo12", "foo100", true)
+	f("10foo2", "10foo10", true)
+	f("10foo10", "10foo2", false)
+	f("foo1bar10", "foo1bar2aa", false)
+	f("foo1bar2aa", "foo1bar10aa", true)
+}

app/vmselect/graphite/render_api.go

@@ -0,0 +1,273 @@
+package graphite
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	storageStep = flag.Duration("search.graphiteStorageStep", 10*time.Second, "The interval between datapoints stored in the database. "+
+		"It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. "+
+		"It can be overridden by sending 'storage_step' query arg to /render API or "+
+		"by sending the desired interval via 'Storage-Step' http header during querying /render API")
+	maxPointsPerSeries = flag.Int("search.graphiteMaxPointsPerSeries", 1e6, "The maximum number of points per series Graphite render API can return")
+)
+
+// RenderHandler implements /render endpoint from Graphite Render API.
+//
+// See https://graphite.readthedocs.io/en/stable/render_api.html
+func RenderHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	format := r.FormValue("format")
+	if format != "json" {
+		return fmt.Errorf("unsupported format=%q; supported values: json", format)
+	}
+	xFilesFactor := float64(0)
+	if xff := r.FormValue("xFilesFactor"); len(xff) > 0 {
+		f, err := strconv.ParseFloat(xff, 64)
+		if err != nil {
+			return fmt.Errorf("cannot parse xFilesFactor=%q: %w", xff, err)
+		}
+		xFilesFactor = f
+	}
+	from := r.FormValue("from")
+	fromTime := startTime.UnixNano()/1e6 - 24*3600*1000
+	if len(from) != 0 {
+		fv, err := parseTime(startTime, from)
+		if err != nil {
+			return fmt.Errorf("cannot parse from=%q: %w", from, err)
+		}
+		fromTime = fv
+	}
+	until := r.FormValue("until")
+	untilTime := startTime.UnixNano() / 1e6
+	if len(until) != 0 {
+		uv, err := parseTime(startTime, until)
+		if err != nil {
+			return fmt.Errorf("cannot parse until=%q: %w", until, err)
+		}
+		untilTime = uv
+	}
+	storageStep, err := getStorageStep(r)
+	if err != nil {
+		return err
+	}
+	fromAlign := fromTime % storageStep
+	fromTime -= fromAlign
+	if fromAlign > 0 {
+		fromTime += storageStep
+	}
+	untilAlign := untilTime % storageStep
+	untilTime -= untilAlign
+	if untilAlign > 0 {
+		untilTime += storageStep
+	}
+	if untilTime < fromTime {
+		return fmt.Errorf("from=%s cannot exceed until=%s", from, until)
+	}
+	pointsPerSeries := (untilTime - fromTime) / storageStep
+	if pointsPerSeries > int64(*maxPointsPerSeries) {
+		return fmt.Errorf("too many points per series must be returned on the given [from=%s ... until=%s] time range and the given storageStep=%d: %d; "+
+			"either reduce the time range or increase -search.graphiteMaxPointsPerSeries=%d", from, until, storageStep, pointsPerSeries, *maxPointsPerSeries)
+	}
+	maxDataPoints := 0
+	if s := r.FormValue("maxDataPoints"); len(s) > 0 {
+		n, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return fmt.Errorf("cannot parse maxDataPoints=%q: %w", maxDataPoints, err)
+		}
+		if n <= 0 {
+			return fmt.Errorf("maxDataPoints must be greater than 0; got %f", n)
+		}
+		maxDataPoints = int(n)
+	}
+	etfs, err := searchutils.GetExtraTagFilters(r)
+	if err != nil {
+		return fmt.Errorf("cannot setup tag filters: %w", err)
+	}
+	var nextSeriess []nextSeriesFunc
+	targets := r.Form["target"]
+	for _, target := range targets {
+		ec := &evalConfig{
+			startTime:     fromTime,
+			endTime:       untilTime,
+			storageStep:   storageStep,
+			deadline:      deadline,
+			currentTime:   startTime,
+			xFilesFactor:  xFilesFactor,
+			etfs:          etfs,
+			originalQuery: target,
+		}
+		nextSeries, err := execExpr(ec, target)
+		if err != nil {
+			for _, f := range nextSeriess {
+				_, _ = drainAllSeries(f)
+			}
+			return fmt.Errorf("cannot eval target=%q: %w", target, err)
+		}
+		// do not use nextSeriesConcurrentWrapper here in order to preserve series order.
+		if maxDataPoints > 0 {
+			step := (ec.endTime - ec.startTime) / int64(maxDataPoints)
+			nextSeries = nextSeriesSerialWrapper(nextSeries, func(s *series) (*series, error) {
+				aggrFunc := s.consolidateFunc
+				if aggrFunc == nil {
+					aggrFunc = aggrAvg
+				}
+				xFilesFactor := s.xFilesFactor
+				if s.xFilesFactor <= 0 {
+					xFilesFactor = ec.xFilesFactor
+				}
+				if len(s.Values) > maxDataPoints {
+					s.summarize(aggrFunc, ec.startTime, ec.endTime, step, xFilesFactor)
+				}
+				return s, nil
+			})
+		}
+		nextSeriess = append(nextSeriess, nextSeries)
+	}
+	f := nextSeriesGroup(nextSeriess, nil)
+	jsonp := r.FormValue("jsonp")
+	contentType := getContentType(jsonp)
+	w.Header().Set("Content-Type", contentType)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteRenderJSONResponse(bw, f, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	renderDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var renderDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/render"}`)
+
+const msecsPerDay = 24 * 3600 * 1000
+
+// parseTime parses Graphite time in s.
+//
+// If the time in s is relative, then it is relative to startTime.
+func parseTime(startTime time.Time, s string) (int64, error) {
+	switch s {
+	case "now":
+		return startTime.UnixNano() / 1e6, nil
+	case "today":
+		ts := startTime.UnixNano() / 1e6
+		return ts - ts%msecsPerDay, nil
+	case "yesterday":
+		ts := startTime.UnixNano() / 1e6
+		return ts - (ts % msecsPerDay) - msecsPerDay, nil
+	}
+	// Attempt to parse RFC3339 (YYYY-MM-DDTHH:mm:SSZTZ:00)
+	if t, err := time.Parse(time.RFC3339, s); err == nil {
+		return t.UnixNano() / 1e6, nil
+	}
+	// Attempt to parse HH:MM_YYYYMMDD
+	if t, err := time.Parse("15:04_20060102", s); err == nil {
+		return t.UnixNano() / 1e6, nil
+	}
+	// Attempt to parse HH:MMYYYYMMDD
+	if t, err := time.Parse("15:0420060102", s); err == nil {
+		return t.UnixNano() / 1e6, nil
+	}
+	// Attempt to parse YYYYMMDD
+	if t, err := time.Parse("20060102", s); err == nil {
+		return t.UnixNano() / 1e6, nil
+	}
+	// Attempt to parse HH:MM YYYYMMDD
+	if t, err := time.Parse("15:04 20060102", s); err == nil {
+		return t.UnixNano() / 1e6, nil
+	}
+	// Attempt to parse YYYY-MM-DD
+	if t, err := time.Parse("2006-01-02", s); err == nil {
+		return t.UnixNano() / 1e6, nil
+	}
+	// Attempt to parse MM/DD/YY
+	if t, err := time.Parse("01/02/06", s); err == nil {
+		return t.UnixNano() / 1e6, nil
+	}
+
+	// Attempt to parse time as unix timestamp
+	if n, err := strconv.ParseInt(s, 10, 64); err == nil {
+		return n * 1000, nil
+	}
+	// Attempt to parse interval
+	if interval, err := parseInterval(s); err == nil {
+		return startTime.UnixNano()/1e6 + interval, nil
+	}
+	return 0, fmt.Errorf("unsupported time %q", s)
+}
+
+func parseInterval(s string) (int64, error) {
+	s = strings.TrimSpace(s)
+	prefix := s
+	var suffix string
+	for i := 0; i < len(s); i++ {
+		ch := s[i]
+		if ch != '-' && ch != '+' && ch != '.' && (ch < '0' || ch > '9') {
+			prefix = s[:i]
+			suffix = s[i:]
+			break
+		}
+	}
+	n, err := strconv.ParseFloat(prefix, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse interval %q: %w", s, err)
+	}
+	suffix = strings.TrimSpace(suffix)
+	if len(suffix) == 0 {
+		return 0, fmt.Errorf("missing suffix for interval %q; expecting s, min, h, d, w, mon or y suffix", s)
+	}
+	var m float64
+	switch {
+	case strings.HasPrefix(suffix, "ms"):
+		m = 1
+	case strings.HasPrefix(suffix, "s"):
+		m = 1000
+	case strings.HasPrefix(suffix, "mi"),
+		strings.HasPrefix(suffix, "m") && !strings.HasPrefix(suffix, "mo"):
+		m = 60 * 1000
+	case strings.HasPrefix(suffix, "h"):
+		m = 3600 * 1000
+	case strings.HasPrefix(suffix, "d"):
+		m = 24 * 3600 * 1000
+	case strings.HasPrefix(suffix, "w"):
+		m = 7 * 24 * 3600 * 1000
+	case strings.HasPrefix(suffix, "mo"):
+		m = 30 * 24 * 3600 * 1000
+	case strings.HasPrefix(suffix, "y"):
+		m = 365 * 24 * 3600 * 1000
+	default:
+		return 0, fmt.Errorf("unsupported interval %q", s)
+	}
+	return int64(n * m), nil
+}
+
+func getStorageStep(r *http.Request) (int64, error) {
+	s := r.FormValue("storage_step")
+	if len(s) == 0 {
+		s = r.Header.Get("Storage-Step")
+	}
+	if len(s) == 0 {
+		step := int64(storageStep.Seconds() * 1000)
+		if step <= 0 {
+			return 0, fmt.Errorf("the `-search.graphiteStorageStep` command-line flag value must be positive; got %s", storageStep.String())
+		}
+		return step, nil
+	}
+	step, err := parseInterval(s)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse datapoints interval %s: %w", s, err)
+	}
+	if step <= 0 {
+		return 0, fmt.Errorf("storage_step cannot be negative; got %s", s)
+	}
+	return step, nil
+}

app/vmselect/graphite/render_api_test.go

@@ -0,0 +1,103 @@
+package graphite
+
+import (
+	"testing"
+	"time"
+)
+
+func TestParseIntervalSuccess(t *testing.T) {
+	f := func(s string, intervalExpected int64) {
+		t.Helper()
+		interval, err := parseInterval(s)
+		if err != nil {
+			t.Fatalf("unexpected error in parseInterva(%q): %s", s, err)
+		}
+		if interval != intervalExpected {
+			t.Fatalf("unexpected result for parseInterval(%q); got %d; want %d", s, interval, intervalExpected)
+		}
+	}
+	f(`1ms`, 1)
+	f(`-10.5ms`, -10)
+	f(`+5.5s`, 5500)
+	f(`7.85s`, 7850)
+	f(`-7.85sec`, -7850)
+	f(`-7.85secs`, -7850)
+	f(`5seconds`, 5000)
+	f(`10min`, 10*60*1000)
+	f(`10 mins`, 10*60*1000)
+	f(` 10  mins `, 10*60*1000)
+	f(`10m`, 10*60*1000)
+	f(`-10.5min`, -10.5*60*1000)
+	f(`-10.5m`, -10.5*60*1000)
+	f(`3minutes`, 3*60*1000)
+	f(`3h`, 3*3600*1000)
+	f(`-4.5hour`, -4.5*3600*1000)
+	f(`7hours`, 7*3600*1000)
+	f(`5d`, 5*24*3600*1000)
+	f(`-3.5days`, -3.5*24*3600*1000)
+	f(`0.5w`, 0.5*7*24*3600*1000)
+	f(`10weeks`, 10*7*24*3600*1000)
+	f(`2months`, 2*30*24*3600*1000)
+	f(`2mo`, 2*30*24*3600*1000)
+	f(`1.2y`, 1.2*365*24*3600*1000)
+	f(`-3years`, -3*365*24*3600*1000)
+}
+
+func TestParseIntervalError(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		interval, err := parseInterval(s)
+		if err == nil {
+			t.Fatalf("expecting non-nil error for parseInterval(%q)", s)
+		}
+		if interval != 0 {
+			t.Fatalf("unexpected non-zero interval for parseInterval(%q): %d", s, interval)
+		}
+	}
+	f("")
+	f("foo")
+	f(`'1minute'`)
+	f(`123`)
+}
+
+func TestParseTimeSuccess(t *testing.T) {
+	startTime := time.Now()
+	startTimestamp := startTime.UnixNano() / 1e6
+	f := func(s string, timestampExpected int64) {
+		t.Helper()
+		timestamp, err := parseTime(startTime, s)
+		if err != nil {
+			t.Fatalf("unexpected error from parseTime(%q): %s", s, err)
+		}
+		if timestamp != timestampExpected {
+			t.Fatalf("unexpected timestamp returned from parseTime(%q); got %d; want %d", s, timestamp, timestampExpected)
+		}
+	}
+	f("now", startTimestamp)
+	f("today", startTimestamp-startTimestamp%msecsPerDay)
+	f("yesterday", startTimestamp-(startTimestamp%msecsPerDay)-msecsPerDay)
+	f("1234567890", 1234567890000)
+	f("18:36_20210223", 1614105360000)
+	f("20210223", 1614038400000)
+	f("02/23/21", 1614038400000)
+	f("2021-02-23", 1614038400000)
+	f("2021-02-23T18:36:12Z", 1614105372000)
+	f("-3hours", startTimestamp-3*3600*1000)
+	f("1.5minutes", startTimestamp+1.5*60*1000)
+}
+
+func TestParseTimeFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		timestamp, err := parseTime(time.Now(), s)
+		if err == nil {
+			t.Fatalf("expecting non-nil error for parseTime(%q)", s)
+		}
+		if timestamp != 0 {
+			t.Fatalf("expecting zero value for parseTime(%q); got %d", s, timestamp)
+		}
+	}
+	f("")
+	f("foobar")
+	f("1235aafb")
+}

app/vmselect/graphite/render_response.qtpl

@@ -0,0 +1,59 @@
+{% stripspace %}
+
+{% import (
+	"math"
+	"sort"
+) %}
+
+RenderJSONResponse generates response for /render?format=json .
+See https://graphite.readthedocs.io/en/stable/render_api.html#json
+{% func RenderJSONResponse(nextSeries nextSeriesFunc, jsonp string) %}
+	{% if jsonp != "" %}{%s= jsonp %}({% endif %}
+	{% code ss, err := fetchAllSeries(nextSeries) %}
+	{% if err != nil %}
+		{
+			"error": {%q= err.Error() %}
+		}
+		{% return %}
+	{% endif %}
+	{% code sort.Slice(ss, func(i, j int) bool { return ss[i].Name < ss[j].Name }) %}
+	[
+		{% for i, s := range ss %}
+			{%= renderSeriesJSON(s) %}
+			{% if i+1 < len(ss) %},{% endif %}
+		{% endfor %}
+	]
+	{% if jsonp != "" %}){% endif %}
+{% endfunc %}
+
+{% func renderSeriesJSON(s *series) %}
+	{
+		"target": {%q= s.Name %},
+		"tags":{
+			{% code
+				tagKeys := make([]string, 0, len(s.Tags))
+				for k := range s.Tags {
+					tagKeys = append(tagKeys, k)
+				}
+				sort.Strings(tagKeys)
+			%}
+			{% for i, k := range tagKeys %}
+				{% code v := s.Tags[k] %}
+				{%q= k %}: {%q= v %}
+				{% if i+1 < len(tagKeys) %},{% endif %}
+			{% endfor %}
+		},
+		"datapoints":[
+			{% code timestamps := s.Timestamps %}
+			{% for i, v := range s.Values %}
+				[
+					{% if math.IsNaN(v) %}null{% else %}{%f= v %}{% endif %},
+					{%dl= timestamps[i]/1e3 %}
+				]
+				{% if i+1 < len(timestamps) %},{% endif %}
+			{% endfor %}
+		]
+	}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/render_response.qtpl.go

@@ -0,0 +1,203 @@
+// Code generated by qtc from "render_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/graphite/render_response.qtpl:3
+package graphite
+
+//line app/vmselect/graphite/render_response.qtpl:3
+import (
+	"math"
+	"sort"
+)
+
+// RenderJSONResponse generates response for /render?format=json .See https://graphite.readthedocs.io/en/stable/render_api.html#json
+
+//line app/vmselect/graphite/render_response.qtpl:10
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/render_response.qtpl:10
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/render_response.qtpl:10
+func StreamRenderJSONResponse(qw422016 *qt422016.Writer, nextSeries nextSeriesFunc, jsonp string) {
+//line app/vmselect/graphite/render_response.qtpl:11
+	if jsonp != "" {
+//line app/vmselect/graphite/render_response.qtpl:11
+		qw422016.N().S(jsonp)
+//line app/vmselect/graphite/render_response.qtpl:11
+		qw422016.N().S(`(`)
+//line app/vmselect/graphite/render_response.qtpl:11
+	}
+//line app/vmselect/graphite/render_response.qtpl:12
+	ss, err := fetchAllSeries(nextSeries)
+
+//line app/vmselect/graphite/render_response.qtpl:13
+	if err != nil {
+//line app/vmselect/graphite/render_response.qtpl:13
+		qw422016.N().S(`{"error":`)
+//line app/vmselect/graphite/render_response.qtpl:15
+		qw422016.N().Q(err.Error())
+//line app/vmselect/graphite/render_response.qtpl:15
+		qw422016.N().S(`}`)
+//line app/vmselect/graphite/render_response.qtpl:17
+		return
+//line app/vmselect/graphite/render_response.qtpl:18
+	}
+//line app/vmselect/graphite/render_response.qtpl:19
+	sort.Slice(ss, func(i, j int) bool { return ss[i].Name < ss[j].Name })
+
+//line app/vmselect/graphite/render_response.qtpl:19
+	qw422016.N().S(`[`)
+//line app/vmselect/graphite/render_response.qtpl:21
+	for i, s := range ss {
+//line app/vmselect/graphite/render_response.qtpl:22
+		streamrenderSeriesJSON(qw422016, s)
+//line app/vmselect/graphite/render_response.qtpl:23
+		if i+1 < len(ss) {
+//line app/vmselect/graphite/render_response.qtpl:23
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/render_response.qtpl:23
+		}
+//line app/vmselect/graphite/render_response.qtpl:24
+	}
+//line app/vmselect/graphite/render_response.qtpl:24
+	qw422016.N().S(`]`)
+//line app/vmselect/graphite/render_response.qtpl:26
+	if jsonp != "" {
+//line app/vmselect/graphite/render_response.qtpl:26
+		qw422016.N().S(`)`)
+//line app/vmselect/graphite/render_response.qtpl:26
+	}
+//line app/vmselect/graphite/render_response.qtpl:27
+}
+
+//line app/vmselect/graphite/render_response.qtpl:27
+func WriteRenderJSONResponse(qq422016 qtio422016.Writer, nextSeries nextSeriesFunc, jsonp string) {
+//line app/vmselect/graphite/render_response.qtpl:27
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/render_response.qtpl:27
+	StreamRenderJSONResponse(qw422016, nextSeries, jsonp)
+//line app/vmselect/graphite/render_response.qtpl:27
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/render_response.qtpl:27
+}
+
+//line app/vmselect/graphite/render_response.qtpl:27
+func RenderJSONResponse(nextSeries nextSeriesFunc, jsonp string) string {
+//line app/vmselect/graphite/render_response.qtpl:27
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/render_response.qtpl:27
+	WriteRenderJSONResponse(qb422016, nextSeries, jsonp)
+//line app/vmselect/graphite/render_response.qtpl:27
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/render_response.qtpl:27
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/render_response.qtpl:27
+	return qs422016
+//line app/vmselect/graphite/render_response.qtpl:27
+}
+
+//line app/vmselect/graphite/render_response.qtpl:29
+func streamrenderSeriesJSON(qw422016 *qt422016.Writer, s *series) {
+//line app/vmselect/graphite/render_response.qtpl:29
+	qw422016.N().S(`{"target":`)
+//line app/vmselect/graphite/render_response.qtpl:31
+	qw422016.N().Q(s.Name)
+//line app/vmselect/graphite/render_response.qtpl:31
+	qw422016.N().S(`,"tags":{`)
+//line app/vmselect/graphite/render_response.qtpl:34
+	tagKeys := make([]string, 0, len(s.Tags))
+	for k := range s.Tags {
+		tagKeys = append(tagKeys, k)
+	}
+	sort.Strings(tagKeys)
+
+//line app/vmselect/graphite/render_response.qtpl:40
+	for i, k := range tagKeys {
+//line app/vmselect/graphite/render_response.qtpl:41
+		v := s.Tags[k]
+
+//line app/vmselect/graphite/render_response.qtpl:42
+		qw422016.N().Q(k)
+//line app/vmselect/graphite/render_response.qtpl:42
+		qw422016.N().S(`:`)
+//line app/vmselect/graphite/render_response.qtpl:42
+		qw422016.N().Q(v)
+//line app/vmselect/graphite/render_response.qtpl:43
+		if i+1 < len(tagKeys) {
+//line app/vmselect/graphite/render_response.qtpl:43
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/render_response.qtpl:43
+		}
+//line app/vmselect/graphite/render_response.qtpl:44
+	}
+//line app/vmselect/graphite/render_response.qtpl:44
+	qw422016.N().S(`},"datapoints":[`)
+//line app/vmselect/graphite/render_response.qtpl:47
+	timestamps := s.Timestamps
+
+//line app/vmselect/graphite/render_response.qtpl:48
+	for i, v := range s.Values {
+//line app/vmselect/graphite/render_response.qtpl:48
+		qw422016.N().S(`[`)
+//line app/vmselect/graphite/render_response.qtpl:50
+		if math.IsNaN(v) {
+//line app/vmselect/graphite/render_response.qtpl:50
+			qw422016.N().S(`null`)
+//line app/vmselect/graphite/render_response.qtpl:50
+		} else {
+//line app/vmselect/graphite/render_response.qtpl:50
+			qw422016.N().F(v)
+//line app/vmselect/graphite/render_response.qtpl:50
+		}
+//line app/vmselect/graphite/render_response.qtpl:50
+		qw422016.N().S(`,`)
+//line app/vmselect/graphite/render_response.qtpl:51
+		qw422016.N().DL(timestamps[i] / 1e3)
+//line app/vmselect/graphite/render_response.qtpl:51
+		qw422016.N().S(`]`)
+//line app/vmselect/graphite/render_response.qtpl:53
+		if i+1 < len(timestamps) {
+//line app/vmselect/graphite/render_response.qtpl:53
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/render_response.qtpl:53
+		}
+//line app/vmselect/graphite/render_response.qtpl:54
+	}
+//line app/vmselect/graphite/render_response.qtpl:54
+	qw422016.N().S(`]}`)
+//line app/vmselect/graphite/render_response.qtpl:57
+}
+
+//line app/vmselect/graphite/render_response.qtpl:57
+func writerenderSeriesJSON(qq422016 qtio422016.Writer, s *series) {
+//line app/vmselect/graphite/render_response.qtpl:57
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/render_response.qtpl:57
+	streamrenderSeriesJSON(qw422016, s)
+//line app/vmselect/graphite/render_response.qtpl:57
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/render_response.qtpl:57
+}
+
+//line app/vmselect/graphite/render_response.qtpl:57
+func renderSeriesJSON(s *series) string {
+//line app/vmselect/graphite/render_response.qtpl:57
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/render_response.qtpl:57
+	writerenderSeriesJSON(qb422016, s)
+//line app/vmselect/graphite/render_response.qtpl:57
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/render_response.qtpl:57
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/render_response.qtpl:57
+	return qs422016
+//line app/vmselect/graphite/render_response.qtpl:57
+}

app/vmselect/graphite/transform_test.go

@@ -0,0 +1,81 @@
+package graphite
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestUnmarshalTags(t *testing.T) {
+	f := func(s string, tagsExpected map[string]string) {
+		t.Helper()
+		tags := unmarshalTags(s)
+		if !reflect.DeepEqual(tags, tagsExpected) {
+			t.Fatalf("unexpected tags unmarshaled for s=%q\ngot\n%s\nwant\n%s", s, tags, tagsExpected)
+		}
+	}
+	f("", map[string]string{})
+	f("foo.bar", map[string]string{
+		"name": "foo.bar",
+	})
+	f("foo;bar=baz", map[string]string{
+		"name": "foo",
+		"bar":  "baz",
+	})
+	f("foo.bar;bar;x=aa;baz=aaa;x=y", map[string]string{
+		"name": "foo.bar",
+		"baz":  "aaa",
+		"x":    "y",
+	})
+}
+
+func TestMarshalTags(t *testing.T) {
+	f := func(s, sExpected string) {
+		t.Helper()
+		tags := unmarshalTags(s)
+		sMarshaled := marshalTags(tags)
+		if sMarshaled != sExpected {
+			t.Fatalf("unexpected marshaled tags for s=%q\ngot\n%s\nwant\n%s", s, sMarshaled, sExpected)
+		}
+	}
+	f("", "")
+	f("foo", "foo")
+	f("foo;bar=baz", "foo;bar=baz")
+	f("foo.bar;baz;xx=yy;a=b", "foo.bar;a=b;xx=yy")
+	f("foo.bar;a=bb;a=ccc;d=a.b.c", "foo.bar;a=ccc;d=a.b.c")
+}
+
+func TestGetPathFromName(t *testing.T) {
+	f := func(name, pathExpected string) {
+		t.Helper()
+		path := getPathFromName(name)
+		if path != pathExpected {
+			t.Fatalf("unexpected path extracted from name %q; got %q; want %q", name, path, pathExpected)
+		}
+	}
+	f("", "")
+	f("foo", "foo")
+	f("foo.bar", "foo.bar")
+	f("foo.bar,baz.aa", "foo.bar,baz.aa")
+	f("foo(bar.baz,aa.bb)", "bar.baz")
+	f("foo(1, 'foo', aaa )", "aaa")
+	f("foo|bar(baz)", "foo")
+	f("a(b(c.d.e))", "c.d.e")
+	f("foo()", "foo()")
+	f("123", "123")
+	f("foo(123)", "123")
+	f("fo(bar", "fo(bar")
+}
+
+func TestGraphiteToGolangRegexpReplace(t *testing.T) {
+	f := func(s, replaceExpected string) {
+		t.Helper()
+		replace := graphiteToGolangRegexpReplace(s)
+		if replace != replaceExpected {
+			t.Fatalf("unexpected result for graphiteToGolangRegexpReplace(%q); got %q; want %q", s, replace, replaceExpected)
+		}
+	}
+	f("", "")
+	f("foo", "foo")
+	f(`a\d+`, `a\d+`)
+	f(`\1f\\oo\2`, `$1f\\oo$2`)
+}

app/vmselect/main.go

@@ -224,9 +224,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	if strings.HasPrefix(path, "/functions") {
-		graphiteFunctionsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, "%s", `{}`)
+		funcName := path[len("/functions"):]
+		funcName = strings.TrimPrefix(funcName, "/")
+		if funcName == "" {
+			graphiteFunctionsRequests.Inc()
+			if err := graphite.FunctionsHandler(startTime, w, r); err != nil {
+				graphiteFunctionsErrors.Inc()
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			return true
+		}
+		graphiteFunctionDetailsRequests.Inc()
+		if err := graphite.FunctionDetailsHandler(startTime, funcName, w, r); err != nil {
+			graphiteFunctionDetailsErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
 		return true
 	}
 
@@ -437,6 +451,14 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
+	case "/render":
+		graphiteRenderRequests.Inc()
+		if err := graphite.RenderHandler(startTime, w, r); err != nil {
+			graphiteRenderErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
 	case "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
 		promscrape.WriteMetricRelabelDebug(w, r)
@@ -611,10 +633,17 @@ var (
 	graphiteTagsDelSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/delSeries"}`)
 	graphiteTagsDelSeriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/delSeries"}`)
 
+	graphiteRenderRequests = metrics.NewCounter(`vm_http_requests_total{path="/render"}`)
+	graphiteRenderErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/render"}`)
+
 	promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vm_http_requests_total{path="/metric-relabel-debug"}`)
 	promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vm_http_requests_total{path="/target-relabel-debug"}`)
 
 	graphiteFunctionsRequests = metrics.NewCounter(`vm_http_requests_total{path="/functions"}`)
+	graphiteFunctionsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/functions"}`)
+
+	graphiteFunctionDetailsRequests = metrics.NewCounter(`vm_http_requests_total{path="/functions/<func_name>"}`)
+	graphiteFunctionDetailsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/functions/<func_name>"}`)
 
 	expandWithExprsRequests = metrics.NewCounter(`vm_http_requests_total{path="/expand-with-exprs"}`)
 

app/vmselect/netstorage/netstorage.go

@@ -148,40 +148,31 @@ func timeseriesWorker(qt *querytracer.Tracer, workChs []chan *timeseriesWork, wo
 	// Then help others with the remaining work.
 	rowsProcessed = 0
 	seriesProcessed = 0
-	idx := int(workerID)
-	for {
-		tsw, idxNext := stealTimeseriesWork(workChs, idx)
-		if tsw == nil {
-			// There is no more work
-			break
+	for i := uint(1); i < uint(len(workChs)); i++ {
+		idx := (i + workerID) % uint(len(workChs))
+		ch := workChs[idx]
+		for len(ch) > 0 {
+			// Do not call runtime.Gosched() here in order to give a chance
+			// the real owner of the work to complete it, since it consumes additional CPU
+			// and slows down the code on systems with big number of CPU cores.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966#issuecomment-1483208419
+
+			// It is expected that every channel in the workChs is already closed,
+			// so the next line should return immediately.
+			tsw, ok := <-ch
+			if !ok {
+				break
+			}
+			tsw.err = tsw.do(&tmpResult.rs, workerID)
+			rowsProcessed += tsw.rowsProcessed
+			seriesProcessed++
 		}
-		tsw.err = tsw.do(&tmpResult.rs, workerID)
-		rowsProcessed += tsw.rowsProcessed
-		seriesProcessed++
-		idx = idxNext
 	}
 	qt.Printf("others work processed: series=%d, samples=%d", seriesProcessed, rowsProcessed)
 
 	putTmpResult(tmpResult)
 }
 
-func stealTimeseriesWork(workChs []chan *timeseriesWork, startIdx int) (*timeseriesWork, int) {
-	for i := startIdx; i < startIdx+len(workChs); i++ {
-		// Give a chance other goroutines to perform their work
-		runtime.Gosched()
-
-		idx := i % len(workChs)
-		ch := workChs[idx]
-		// It is expected that every channel in the workChs is already closed,
-		// so the next line should return immediately.
-		tsw, ok := <-ch
-		if ok {
-			return tsw, idx
-		}
-	}
-	return nil, startIdx
-}
-
 func getTmpResult() *result {
 	v := resultPool.Get()
 	if v == nil {
@@ -207,10 +198,17 @@ type result struct {
 
 var resultPool sync.Pool
 
+// MaxWorkers returns the maximum number of workers netstorage can spin when calling RunParallel()
+func MaxWorkers() int {
+	return gomaxprocs
+}
+
+var gomaxprocs = cgroup.AvailableCPUs()
+
 // RunParallel runs f in parallel for all the results from rss.
 //
 // f shouldn't hold references to rs after returning.
-// workerID is the id of the worker goroutine that calls f.
+// workerID is the id of the worker goroutine that calls f. The workerID is in the range [0..MaxWorkers()-1].
 // Data processing is immediately stopped if f returns non-nil error.
 //
 // rss becomes unusable after the call to RunParallel.
@@ -244,7 +242,8 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 		tsw.f = f
 		tsw.mustStop = &mustStop
 	}
-	if gomaxprocs == 1 || tswsLen == 1 {
+	maxWorkers := MaxWorkers()
+	if maxWorkers == 1 || tswsLen == 1 {
 		// It is faster to process time series in the current goroutine.
 		tsw := getTimeseriesWork()
 		tmpResult := getTmpResult()
@@ -280,8 +279,8 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 
 	// Prepare worker channels.
 	workers := len(tsws)
-	if workers > gomaxprocs {
-		workers = gomaxprocs
+	if workers > maxWorkers {
+		workers = maxWorkers
 	}
 	itemsPerWorker := (len(tsws) + workers - 1) / workers
 	workChs := make([]chan *timeseriesWork, workers)
@@ -333,8 +332,6 @@ var (
 	seriesReadPerQuery = metrics.NewHistogram(`vm_series_read_per_query`)
 )
 
-var gomaxprocs = cgroup.AvailableCPUs()
-
 type packedTimeseries struct {
 	metricName string
 	brs        []blockRef
@@ -391,37 +388,25 @@ func unpackWorker(workChs []chan *unpackWork, workerID uint) {
 	}
 
 	// Then help others with their work.
-	idx := int(workerID)
-	for {
-		upw, idxNext := stealUnpackWork(workChs, idx)
-		if upw == nil {
-			// There is no more work
-			break
+	for i := uint(1); i < uint(len(workChs)); i++ {
+		idx := (i + workerID) % uint(len(workChs))
+		ch := workChs[idx]
+		for len(ch) > 0 {
+			// Give a chance other goroutines to perform their work
+			runtime.Gosched()
+			// It is expected that every channel in the workChs is already closed,
+			// so the next line should return immediately.
+			upw, ok := <-ch
+			if !ok {
+				break
+			}
+			upw.unpack(tmpBlock)
 		}
-		upw.unpack(tmpBlock)
-		idx = idxNext
 	}
 
 	putTmpStorageBlock(tmpBlock)
 }
 
-func stealUnpackWork(workChs []chan *unpackWork, startIdx int) (*unpackWork, int) {
-	for i := startIdx; i < startIdx+len(workChs); i++ {
-		// Give a chance other goroutines to perform their work
-		runtime.Gosched()
-
-		idx := i % len(workChs)
-		ch := workChs[idx]
-		// It is expected that every channel in the workChs is already closed,
-		// so the next line should return immediately.
-		upw, ok := <-ch
-		if ok {
-			return upw, idx
-		}
-	}
-	return nil, startIdx
-}
-
 func getTmpStorageBlock() *storage.Block {
 	v := tmpStorageBlockPool.Get()
 	if v == nil {
@@ -1017,7 +1002,6 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 	indexSearchDuration.UpdateDuration(startTime)
 
 	// Start workers that call f in parallel on available CPU cores.
-	gomaxprocs := cgroup.AvailableCPUs()
 	workCh := make(chan *exportWork, gomaxprocs*8)
 	var (
 		errGlobal     error

app/vmselect/netstorage/tmp_blocks_file.go

@@ -142,6 +142,9 @@ func (tbf *tmpBlocksFile) Finalize() error {
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)
+	// Collect local stats in order to improve performance on systems with big number of CPU cores.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966
+	r.SetUseLocalStats()
 	tbf.r = r
 	return nil
 }
@@ -176,14 +179,12 @@ func (tbf *tmpBlocksFile) MustClose() {
 	}
 	fname := tbf.f.Name()
 
-	// Remove the file at first, then close it.
-	// This way the OS shouldn't try to flush file contents to storage
-	// on close.
-	if err := os.Remove(fname); err != nil {
-		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
-	}
 	if err := tbf.f.Close(); err != nil {
 		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
 	}
+	// We cannot remove unclosed at non-posix filesystems, like windows
+	if err := os.Remove(fname); err != nil {
+		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+	}
 	tbf.f = nil
 }

app/vmselect/prometheus/prometheus.go

@@ -747,7 +747,8 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	} else {
 		queryOffset = 0
 	}
-	ec := promql.EvalConfig{
+	qs := &promql.QueryStats{}
+	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 start,
 		Step:                step,
@@ -762,8 +763,10 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
+
+		QueryStats: qs,
 	}
-	result, err := promql.Exec(qt, &ec, query, true)
+	result, err := promql.Exec(qt, ec, query, true)
 	if err != nil {
 		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %w", query, start, step, err)
 	}
@@ -786,7 +789,8 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	qtDone := func() {
 		qt.Donef("query=%s, time=%d: series=%d", query, start, len(result))
 	}
-	WriteQueryResponse(bw, result, qt, qtDone)
+
+	WriteQueryResponse(bw, result, qt, qtDone, qs)
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot flush query response to remote client: %w", err)
 	}
@@ -851,7 +855,8 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		start, end = promql.AdjustStartEnd(start, end, step)
 	}
 
-	ec := promql.EvalConfig{
+	qs := &promql.QueryStats{}
+	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 end,
 		Step:                step,
@@ -866,8 +871,10 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
+
+		QueryStats: qs,
 	}
-	result, err := promql.Exec(qt, &ec, query, false)
+	result, err := promql.Exec(qt, ec, query, false)
 	if err != nil {
 		return err
 	}
@@ -891,7 +898,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	qtDone := func() {
 		qt.Donef("start=%d, end=%d, step=%d, query=%q: series=%d", start, end, step, query, len(result))
 	}
-	WriteQueryRangeResponse(bw, result, qt, qtDone)
+	WriteQueryRangeResponse(bw, result, qt, qtDone, qs)
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot send query range response to remote client: %w", err)
 	}

app/vmselect/prometheus/query_range_response.qtpl

@@ -1,12 +1,13 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
 {% stripspace %}
 QueryRangeResponse generates response for /api/v1/query_range.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
-{% func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %}
+{% func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) %}
 {
 	{% code
 		seriesCount := len(rs)
@@ -26,6 +27,9 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 				{% endfor %}
 			{% endif %}
 		]
+	},
+	"stats":{
+	    "seriesFetched": "{%d qs.SeriesFetched %}"
 	}
 	{% code
 		qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)

app/vmselect/prometheus/query_range_response.qtpl.go

@@ -7,133 +7,138 @@ package prometheus
 //line app/vmselect/prometheus/query_range_response.qtpl:1
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 )
 
 // QueryRangeResponse generates response for /api/v1/query_range.See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 
-//line app/vmselect/prometheus/query_range_response.qtpl:9
+//line app/vmselect/prometheus/query_range_response.qtpl:10
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/query_range_response.qtpl:9
+//line app/vmselect/prometheus/query_range_response.qtpl:10
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/query_range_response.qtpl:9
-func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_range_response.qtpl:9
+//line app/vmselect/prometheus/query_range_response.qtpl:10
+func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_range_response.qtpl:10
 	qw422016.N().S(`{`)
-//line app/vmselect/prometheus/query_range_response.qtpl:12
+//line app/vmselect/prometheus/query_range_response.qtpl:13
 	seriesCount := len(rs)
 	pointsCount := 0
 
-//line app/vmselect/prometheus/query_range_response.qtpl:14
+//line app/vmselect/prometheus/query_range_response.qtpl:15
 	qw422016.N().S(`"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vmselect/prometheus/query_range_response.qtpl:19
-	if len(rs) > 0 {
 //line app/vmselect/prometheus/query_range_response.qtpl:20
-		streamqueryRangeLine(qw422016, &rs[0])
+	if len(rs) > 0 {
 //line app/vmselect/prometheus/query_range_response.qtpl:21
+		streamqueryRangeLine(qw422016, &rs[0])
+//line app/vmselect/prometheus/query_range_response.qtpl:22
 		pointsCount += len(rs[0].Values)
 
-//line app/vmselect/prometheus/query_range_response.qtpl:22
+//line app/vmselect/prometheus/query_range_response.qtpl:23
 		rs = rs[1:]
 
-//line app/vmselect/prometheus/query_range_response.qtpl:23
-		for i := range rs {
-//line app/vmselect/prometheus/query_range_response.qtpl:23
-			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/query_range_response.qtpl:24
-			streamqueryRangeLine(qw422016, &rs[i])
+		for i := range rs {
+//line app/vmselect/prometheus/query_range_response.qtpl:24
+			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/query_range_response.qtpl:25
+			streamqueryRangeLine(qw422016, &rs[i])
+//line app/vmselect/prometheus/query_range_response.qtpl:26
 			pointsCount += len(rs[i].Values)
 
-//line app/vmselect/prometheus/query_range_response.qtpl:26
+//line app/vmselect/prometheus/query_range_response.qtpl:27
 		}
-//line app/vmselect/prometheus/query_range_response.qtpl:27
+//line app/vmselect/prometheus/query_range_response.qtpl:28
 	}
-//line app/vmselect/prometheus/query_range_response.qtpl:27
-	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:31
+//line app/vmselect/prometheus/query_range_response.qtpl:28
+	qw422016.N().S(`]},"stats":{"seriesFetched": "`)
+//line app/vmselect/prometheus/query_range_response.qtpl:32
+	qw422016.N().D(qs.SeriesFetched)
+//line app/vmselect/prometheus/query_range_response.qtpl:32
+	qw422016.N().S(`"}`)
+//line app/vmselect/prometheus/query_range_response.qtpl:35
 	qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)
 	qtDone()
 
-//line app/vmselect/prometheus/query_range_response.qtpl:34
+//line app/vmselect/prometheus/query_range_response.qtpl:38
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/query_range_response.qtpl:34
-	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-}
-
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-func WriteQueryRangeResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	StreamQueryRangeResponse(qw422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-}
-
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string {
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	WriteQueryRangeResponse(qb422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-	return qs422016
-//line app/vmselect/prometheus/query_range_response.qtpl:36
-}
-
 //line app/vmselect/prometheus/query_range_response.qtpl:38
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+}
+
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+func WriteQueryRangeResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	StreamQueryRangeResponse(qw422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+}
+
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+func QueryRangeResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) string {
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	WriteQueryRangeResponse(qb422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+	return qs422016
+//line app/vmselect/prometheus/query_range_response.qtpl:40
+}
+
+//line app/vmselect/prometheus/query_range_response.qtpl:42
 func streamqueryRangeLine(qw422016 *qt422016.Writer, r *netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:38
+//line app/vmselect/prometheus/query_range_response.qtpl:42
 	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:40
+//line app/vmselect/prometheus/query_range_response.qtpl:44
 	streammetricNameObject(qw422016, &r.MetricName)
-//line app/vmselect/prometheus/query_range_response.qtpl:40
+//line app/vmselect/prometheus/query_range_response.qtpl:44
 	qw422016.N().S(`,"values":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:41
+//line app/vmselect/prometheus/query_range_response.qtpl:45
 	streamvaluesWithTimestamps(qw422016, r.Values, r.Timestamps)
-//line app/vmselect/prometheus/query_range_response.qtpl:41
+//line app/vmselect/prometheus/query_range_response.qtpl:45
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 func writequeryRangeLine(qq422016 qtio422016.Writer, r *netstorage.Result) {
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	streamqueryRangeLine(qw422016, r)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 }
 
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 func queryRangeLine(r *netstorage.Result) string {
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	writequeryRangeLine(qb422016, r)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 	return qs422016
-//line app/vmselect/prometheus/query_range_response.qtpl:43
+//line app/vmselect/prometheus/query_range_response.qtpl:47
 }

app/vmselect/prometheus/query_response.qtpl

@@ -1,12 +1,13 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
 {% stripspace %}
 QueryResponse generates response for /api/v1/query.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
-{% func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %}
+{% func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) %}
 {
 	{% code seriesCount := len(rs) %}
 	"status":"success",
@@ -28,6 +29,9 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 				{% endfor %}
 			{% endif %}
 		]
+	},
+	"stats":{
+	    "seriesFetched": "{%d qs.SeriesFetched %}"
 	}
 	{% code
 		qt.Printf("generate /api/v1/query response for series=%d", seriesCount)

app/vmselect/prometheus/query_response.qtpl.go

@@ -7,102 +7,107 @@ package prometheus
 //line app/vmselect/prometheus/query_response.qtpl:1
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 )
 
 // QueryResponse generates response for /api/v1/query.See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 
-//line app/vmselect/prometheus/query_response.qtpl:9
+//line app/vmselect/prometheus/query_response.qtpl:10
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/query_response.qtpl:9
+//line app/vmselect/prometheus/query_response.qtpl:10
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/query_response.qtpl:9
-func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_response.qtpl:9
+//line app/vmselect/prometheus/query_response.qtpl:10
+func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_response.qtpl:10
 	qw422016.N().S(`{`)
-//line app/vmselect/prometheus/query_response.qtpl:11
+//line app/vmselect/prometheus/query_response.qtpl:12
 	seriesCount := len(rs)
 
-//line app/vmselect/prometheus/query_response.qtpl:11
+//line app/vmselect/prometheus/query_response.qtpl:12
 	qw422016.N().S(`"status":"success","data":{"resultType":"vector","result":[`)
-//line app/vmselect/prometheus/query_response.qtpl:16
+//line app/vmselect/prometheus/query_response.qtpl:17
 	if len(rs) > 0 {
-//line app/vmselect/prometheus/query_response.qtpl:16
+//line app/vmselect/prometheus/query_response.qtpl:17
 		qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/query_response.qtpl:18
+//line app/vmselect/prometheus/query_response.qtpl:19
 		streammetricNameObject(qw422016, &rs[0].MetricName)
-//line app/vmselect/prometheus/query_response.qtpl:18
+//line app/vmselect/prometheus/query_response.qtpl:19
 		qw422016.N().S(`,"value":`)
-//line app/vmselect/prometheus/query_response.qtpl:19
+//line app/vmselect/prometheus/query_response.qtpl:20
 		streammetricRow(qw422016, rs[0].Timestamps[0], rs[0].Values[0])
-//line app/vmselect/prometheus/query_response.qtpl:19
+//line app/vmselect/prometheus/query_response.qtpl:20
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_response.qtpl:21
+//line app/vmselect/prometheus/query_response.qtpl:22
 		rs = rs[1:]
 
-//line app/vmselect/prometheus/query_response.qtpl:22
-		for i := range rs {
 //line app/vmselect/prometheus/query_response.qtpl:23
+		for i := range rs {
+//line app/vmselect/prometheus/query_response.qtpl:24
 			r := &rs[i]
 
-//line app/vmselect/prometheus/query_response.qtpl:23
+//line app/vmselect/prometheus/query_response.qtpl:24
 			qw422016.N().S(`,{"metric":`)
-//line app/vmselect/prometheus/query_response.qtpl:25
+//line app/vmselect/prometheus/query_response.qtpl:26
 			streammetricNameObject(qw422016, &r.MetricName)
-//line app/vmselect/prometheus/query_response.qtpl:25
+//line app/vmselect/prometheus/query_response.qtpl:26
 			qw422016.N().S(`,"value":`)
-//line app/vmselect/prometheus/query_response.qtpl:26
+//line app/vmselect/prometheus/query_response.qtpl:27
 			streammetricRow(qw422016, r.Timestamps[0], r.Values[0])
-//line app/vmselect/prometheus/query_response.qtpl:26
+//line app/vmselect/prometheus/query_response.qtpl:27
 			qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_response.qtpl:28
+//line app/vmselect/prometheus/query_response.qtpl:29
 		}
-//line app/vmselect/prometheus/query_response.qtpl:29
+//line app/vmselect/prometheus/query_response.qtpl:30
 	}
-//line app/vmselect/prometheus/query_response.qtpl:29
-	qw422016.N().S(`]}`)
-//line app/vmselect/prometheus/query_response.qtpl:33
+//line app/vmselect/prometheus/query_response.qtpl:30
+	qw422016.N().S(`]},"stats":{"seriesFetched": "`)
+//line app/vmselect/prometheus/query_response.qtpl:34
+	qw422016.N().D(qs.SeriesFetched)
+//line app/vmselect/prometheus/query_response.qtpl:34
+	qw422016.N().S(`"}`)
+//line app/vmselect/prometheus/query_response.qtpl:37
 	qt.Printf("generate /api/v1/query response for series=%d", seriesCount)
 	qtDone()
 
-//line app/vmselect/prometheus/query_response.qtpl:36
+//line app/vmselect/prometheus/query_response.qtpl:40
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/query_response.qtpl:36
+//line app/vmselect/prometheus/query_response.qtpl:40
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:38
-func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) {
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
-	StreamQueryResponse(qw422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+	StreamQueryResponse(qw422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:38
-func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string {
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) string {
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_response.qtpl:38
-	WriteQueryResponse(qb422016, rs, qt, qtDone)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
+	WriteQueryResponse(qb422016, rs, qt, qtDone, qs)
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 	return qs422016
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:42
 }

app/vmselect/promql/aggr_incremental.go

@@ -3,8 +3,9 @@ package promql
 import (
 	"math"
 	"strings"
-	"sync"
+	"unsafe"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/metricsql"
 )
@@ -63,31 +64,36 @@ var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{
 	},
 }
 
+type incrementalAggrContextMap struct {
+	m map[string]*incrementalAggrContext
+
+	// The padding prevents false sharing on widespread platforms with
+	// 128 mod (cache line size) = 0 .
+	_ [128 - unsafe.Sizeof(map[string]*incrementalAggrContext{})%128]byte
+}
+
 type incrementalAggrFuncContext struct {
 	ae *metricsql.AggrFuncExpr
 
-	m sync.Map
+	byWorkerID []incrementalAggrContextMap
 
 	callbacks *incrementalAggrFuncCallbacks
 }
 
 func newIncrementalAggrFuncContext(ae *metricsql.AggrFuncExpr, callbacks *incrementalAggrFuncCallbacks) *incrementalAggrFuncContext {
 	return &incrementalAggrFuncContext{
-		ae:        ae,
-		callbacks: callbacks,
+		ae:         ae,
+		byWorkerID: make([]incrementalAggrContextMap, netstorage.MaxWorkers()),
+		callbacks:  callbacks,
 	}
 }
 
 func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, workerID uint) {
-	v, ok := iafc.m.Load(workerID)
-	if !ok {
-		// It is safe creating and storing m in iafc.m without locking,
-		// since it is guaranteed that only a single goroutine can execute
-		// code for the given workerID at a time.
-		v = make(map[string]*incrementalAggrContext, 1)
-		iafc.m.Store(workerID, v)
+	v := &iafc.byWorkerID[workerID]
+	if v.m == nil {
+		v.m = make(map[string]*incrementalAggrContext, 1)
 	}
-	m := v.(map[string]*incrementalAggrContext)
+	m := v.m
 
 	ts := tsOrig
 	keepOriginal := iafc.callbacks.keepOriginal
@@ -128,9 +134,9 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 func (iafc *incrementalAggrFuncContext) finalizeTimeseries() []*timeseries {
 	mGlobal := make(map[string]*incrementalAggrContext)
 	mergeAggrFunc := iafc.callbacks.mergeAggrFunc
-	iafc.m.Range(func(k, v interface{}) bool {
-		m := v.(map[string]*incrementalAggrContext)
-		for k, iac := range m {
+	byWorkerID := iafc.byWorkerID
+	for i := range byWorkerID {
+		for k, iac := range byWorkerID[i].m {
 			iacGlobal := mGlobal[k]
 			if iacGlobal == nil {
 				if iafc.ae.Limit > 0 && len(mGlobal) >= iafc.ae.Limit {
@@ -142,8 +148,7 @@ func (iafc *incrementalAggrFuncContext) finalizeTimeseries() []*timeseries {
 			}
 			mergeAggrFunc(iacGlobal, iac)
 		}
-		return true
-	})
+	}
 	tss := make([]*timeseries, 0, len(mGlobal))
 	finalizeAggrFunc := iafc.callbacks.finalizeAggrFunc
 	for _, iac := range mGlobal {

app/vmselect/promql/aggr_incremental_test.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/metricsql"
 )
 
@@ -99,7 +100,7 @@ func TestIncrementalAggr(t *testing.T) {
 }
 
 func testIncrementalParallelAggr(iafc *incrementalAggrFuncContext, tssSrc, tssExpected []*timeseries) error {
-	const workersCount = 3
+	workersCount := netstorage.MaxWorkers()
 	tsCh := make(chan *timeseries)
 	var wg sync.WaitGroup
 	wg.Add(workersCount)

app/vmselect/promql/eval.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
@@ -131,6 +132,12 @@ type EvalConfig struct {
 	// The request URI isn't stored here because its' construction may take non-trivial amounts of CPU.
 	GetRequestURI func() string
 
+	// QueryStats contains various stats for the currently executed query.
+	//
+	// The caller must initialize the QueryStats if it needs the stats.
+	// Otherwise the stats isn't collected.
+	QueryStats *QueryStats
+
 	timestamps     []int64
 	timestampsOnce sync.Once
 }
@@ -149,11 +156,24 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
 	ec.GetRequestURI = src.GetRequestURI
+	ec.QueryStats = src.QueryStats
 
 	// do not copy src.timestamps - they must be generated again.
 	return &ec
 }
 
+// QueryStats contains various stats for the query.
+type QueryStats struct {
+	// SeriesFetched contains the number of series fetched from storage during the query evaluation.
+	SeriesFetched int
+}
+
+func (qs *QueryStats) addSeriesFetched(n int) {
+	if qs != nil {
+		qs.SeriesFetched += n
+	}
+}
+
 func (ec *EvalConfig) validate() {
 	if ec.Start > ec.End {
 		logger.Panicf("BUG: start cannot exceed end; got %d vs %d", ec.Start, ec.End)
@@ -904,31 +924,34 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	if err != nil {
 		return nil, err
 	}
-	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
-	var tssLock sync.Mutex
+
 	var samplesScannedTotal uint64
 	keepMetricNames := getKeepMetricNames(expr)
-	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) {
+	tsw := getTimeseriesByWorkerID()
+	seriesByWorkerID := tsw.byWorkerID
+	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64, workerID uint) ([]float64, []int64) {
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
 		preFunc(values, timestamps)
 		for _, rc := range rcs {
 			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
 				samplesScanned := rc.DoTimeseriesMap(tsm, values, timestamps)
 				atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-				tssLock.Lock()
-				tss = tsm.AppendTimeseriesTo(tss)
-				tssLock.Unlock()
+				seriesByWorkerID[workerID].tss = tsm.AppendTimeseriesTo(seriesByWorkerID[workerID].tss)
 				continue
 			}
 			var ts timeseries
 			samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps)
 			atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-			tssLock.Lock()
-			tss = append(tss, &ts)
-			tssLock.Unlock()
+			seriesByWorkerID[workerID].tss = append(seriesByWorkerID[workerID].tss, &ts)
 		}
 		return values, timestamps
 	})
+	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
+	for i := range seriesByWorkerID {
+		tss = append(tss, seriesByWorkerID[i].tss...)
+	}
+	putTimeseriesByWorkerID(tsw)
+
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal))
 	qt.Printf("rollup %s() over %d series returned by subquery: series=%d, samplesScanned=%d", funcName, len(tssSQ), len(tss), samplesScannedTotal)
 	return tss, nil
@@ -952,28 +975,36 @@ func getKeepMetricNames(expr metricsql.Expr) bool {
 	return false
 }
 
-func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64) ([]float64, []int64)) {
-	concurrency := cgroup.AvailableCPUs()
-	if concurrency > len(tss) {
-		concurrency = len(tss)
+func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64, workerID uint) ([]float64, []int64)) {
+	workers := netstorage.MaxWorkers()
+	if workers > len(tss) {
+		workers = len(tss)
 	}
-	workCh := make(chan *timeseries, concurrency)
+	seriesPerWorker := (len(tss) + workers - 1) / workers
+	workChs := make([]chan *timeseries, workers)
+	for i := range workChs {
+		workChs[i] = make(chan *timeseries, seriesPerWorker)
+	}
+	for i, ts := range tss {
+		idx := i % len(workChs)
+		workChs[idx] <- ts
+	}
+	for _, workCh := range workChs {
+		close(workCh)
+	}
+
 	var wg sync.WaitGroup
-	wg.Add(concurrency)
-	for i := 0; i < concurrency; i++ {
-		go func() {
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func(workerID uint) {
 			defer wg.Done()
 			var tmpValues []float64
 			var tmpTimestamps []int64
-			for ts := range workCh {
-				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps)
+			for ts := range workChs[workerID] {
+				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, workerID)
 			}
-		}()
+		}(uint(i))
 	}
-	for _, ts := range tss {
-		workCh <- ts
-	}
-	close(workCh)
 	wg.Wait()
 }
 
@@ -1065,6 +1096,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 		tss := mergeTimeseries(tssCached, nil, start, ec)
 		return tss, nil
 	}
+	ec.QueryStats.addSeriesFetched(rssLen)
 
 	// Verify timeseries fit available memory after the rollup.
 	// Take into account points from tssCached.
@@ -1195,9 +1227,11 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
-	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
-	var tssLock sync.Mutex
+
 	var samplesScannedTotal uint64
+	tsw := getTimeseriesByWorkerID()
+	seriesByWorkerID := tsw.byWorkerID
+	seriesLen := rss.Len()
 	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
@@ -1205,23 +1239,25 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil {
 				samplesScanned := rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
 				atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-				tssLock.Lock()
-				tss = tsm.AppendTimeseriesTo(tss)
-				tssLock.Unlock()
+				seriesByWorkerID[workerID].tss = tsm.AppendTimeseriesTo(seriesByWorkerID[workerID].tss)
 				continue
 			}
 			var ts timeseries
 			samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
 			atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-			tssLock.Lock()
-			tss = append(tss, &ts)
-			tssLock.Unlock()
+			seriesByWorkerID[workerID].tss = append(seriesByWorkerID[workerID].tss, &ts)
 		}
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
+	tss := make([]*timeseries, 0, seriesLen*len(rcs))
+	for i := range seriesByWorkerID {
+		tss = append(tss, seriesByWorkerID[i].tss...)
+	}
+	putTimeseriesByWorkerID(tsw)
+
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal))
 	qt.Printf("samplesScanned=%d", samplesScannedTotal)
 	return tss, nil
@@ -1243,6 +1279,42 @@ func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConf
 	return samplesScanned
 }
 
+type timeseriesWithPadding struct {
+	tss []*timeseries
+
+	// The padding prevents false sharing on widespread platforms with
+	// 128 mod (cache line size) = 0 .
+	_ [128 - unsafe.Sizeof([]*timeseries{})%128]byte
+}
+
+type timeseriesByWorkerID struct {
+	byWorkerID []timeseriesWithPadding
+}
+
+func (tsw *timeseriesByWorkerID) reset() {
+	byWorkerID := tsw.byWorkerID
+	for i := range byWorkerID {
+		byWorkerID[i].tss = nil
+	}
+}
+
+func getTimeseriesByWorkerID() *timeseriesByWorkerID {
+	v := timeseriesByWorkerIDPool.Get()
+	if v == nil {
+		return &timeseriesByWorkerID{
+			byWorkerID: make([]timeseriesWithPadding, netstorage.MaxWorkers()),
+		}
+	}
+	return v.(*timeseriesByWorkerID)
+}
+
+func putTimeseriesByWorkerID(tsw *timeseriesByWorkerID) {
+	tsw.reset()
+	timeseriesByWorkerIDPool.Put(tsw)
+}
+
+var timeseriesByWorkerIDPool sync.Pool
+
 var bbPool bytesutil.ByteBufferPool
 
 func evalNumber(ec *EvalConfig, n float64) []*timeseries {

app/vmselect/promql/eval_test.go

@@ -76,3 +76,21 @@ func TestValidateMaxPointsPerSeriesSuccess(t *testing.T) {
 	f(1659962171908, 1659966077742, 5000, 800)
 	f(1659962150000, 1659966070000, 10000, 393)
 }
+
+func TestQueryStats_addSeriesFetched(t *testing.T) {
+	qs := &QueryStats{}
+	ec := &EvalConfig{
+		QueryStats: qs,
+	}
+	ec.QueryStats.addSeriesFetched(1)
+
+	if qs.SeriesFetched != 1 {
+		t.Fatalf("expected to get 1; got %d instead", qs.SeriesFetched)
+	}
+
+	ecNew := copyEvalConfig(ec)
+	ecNew.QueryStats.addSeriesFetched(3)
+	if qs.SeriesFetched != 4 {
+		t.Fatalf("expected to get 4; got %d instead", qs.SeriesFetched)
+	}
+}

app/vmselect/promql/rollup_result_cache.go

@@ -441,7 +441,7 @@ func mustSaveRollupResultCacheKeyPrefix(path string) {
 var tooBigRollupResults = metrics.NewCounter("vm_too_big_rollup_results_total")
 
 // Increment this value every time the format of the cache changes.
-const rollupResultCacheVersion = 8
+const rollupResultCacheVersion = 9
 
 func marshalRollupResultCacheKey(dst []byte, expr metricsql.Expr, window, step int64, etfs [][]storage.TagFilter) []byte {
 	dst = append(dst, rollupResultCacheVersion)

app/vmselect/promql/timeseries_test.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"reflect"
 	"testing"
+	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -24,6 +25,26 @@ func TestMarshalTimeseriesFast(t *testing.T) {
 		if !reflect.DeepEqual(tss, tss2) {
 			t.Fatalf("unexpected timeseries unmarshaled\ngot\n%#v\nwant\n%#v", tss2[0], tss[0])
 		}
+
+		// Check 8-byte alignment.
+		// This prevents from SIGBUS error on arm architectures.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+		for _, ts := range tss2 {
+			if len(ts.Values) == 0 {
+				continue
+			}
+
+			// check float64 alignment
+			addr := uintptr(unsafe.Pointer(&ts.Values[0]))
+			if mod := addr % unsafe.Alignof(ts.Values[0]); mod != 0 {
+				t.Fatalf("mis-aligned; &ts.Values[0]=%p; mod=%d", &ts.Values[0], mod)
+			}
+			// check int64 alignment
+			addr = uintptr(unsafe.Pointer(&ts.Timestamps[0]))
+			if mod := addr % unsafe.Alignof(ts.Timestamps[0]); mod != 0 {
+				t.Fatalf("mis-aligned; &ts.Timestamps[0]=%p; mod=%d", &ts.Timestamps[0], mod)
+			}
+		}
 	}
 
 	// Single series

app/vmselect/promql/transform.go

@@ -556,18 +556,28 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 		for _, xs := range xss {
 			ts := xs.ts
 			if isZeroTS(ts) {
-				// Skip time series with zeros. They are substituted by xssNew below.
-				xsPrev = xs
+				// Skip buckets with zero values - they will be merged into a single bucket
+				// when the next non-zero bucket appears.
+
+				// Do not store xs in xsPrev in order to properly create `le` time series
+				// for zero buckets.
+				// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4021
 				continue
 			}
-			if xs.start != xsPrev.end && uniqTs[xs.startStr] == nil {
-				uniqTs[xs.startStr] = xs.ts
-				xssNew = append(xssNew, x{
-					endStr: xs.startStr,
-					end:    xs.start,
-					ts:     copyTS(ts, xs.startStr),
-				})
+			if xs.start != xsPrev.end {
+				// There is a gap between the previous bucket and the current bucket
+				// or the previous bucket is skipped because it was zero.
+				// Fill it with a time series with le=xs.start.
+				if uniqTs[xs.startStr] == nil {
+					uniqTs[xs.startStr] = xs.ts
+					xssNew = append(xssNew, x{
+						endStr: xs.startStr,
+						end:    xs.start,
+						ts:     copyTS(ts, xs.startStr),
+					})
+				}
 			}
+			// Convert the current time series to a time series with le=xs.end
 			ts.MetricName.AddTag("le", xs.endStr)
 			prevTs := uniqTs[xs.endStr]
 			if prevTs != nil {
@@ -579,7 +589,7 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 			}
 			xsPrev = xs
 		}
-		if !math.IsInf(xsPrev.end, 1) && !isZeroTS(xsPrev.ts) {
+		if xsPrev.ts != nil && !math.IsInf(xsPrev.end, 1) && !isZeroTS(xsPrev.ts) {
 			xssNew = append(xssNew, x{
 				endStr: "+Inf",
 				end:    math.Inf(1),

app/vmselect/promql/transform_test.go

@@ -78,6 +78,36 @@ foo{le="+Inf"} 1.23 456`,
 foo{le="+Inf"} 5.3 0`,
 	)
 
+	// Adjacent empty vmrange bucket
+	f(
+		`foo{vmrange="7.743e+05...8.799e+05"} 5 123
+foo{vmrange="6.813e+05...7.743e+05"} 0 123`,
+		`foo{le="7.743e+05"} 0 123
+foo{le="8.799e+05"} 5 123
+foo{le="+Inf"} 5 123`,
+	)
+
+	// Multiple adjacent empty vmrange bucket
+	f(
+		`foo{vmrange="7.743e+05...8.799e+05"} 5 123
+foo{vmrange="6.813e+05...7.743e+05"} 0 123
+foo{vmrange="5.813e+05...6.813e+05"} 0 123
+`,
+		`foo{le="7.743e+05"} 0 123
+foo{le="8.799e+05"} 5 123
+foo{le="+Inf"} 5 123`,
+	)
+	f(
+		`foo{vmrange="8.799e+05...9.813e+05"} 0 123
+foo{vmrange="7.743e+05...8.799e+05"} 5 123
+foo{vmrange="6.813e+05...7.743e+05"} 0 123
+foo{vmrange="5.813e+05...6.813e+05"} 0 123
+`,
+		`foo{le="7.743e+05"} 0 123
+foo{le="8.799e+05"} 5 123
+foo{le="+Inf"} 5 123`,
+	)
+
 	// Multiple non-empty vmrange buckets
 	f(
 		`foo{vmrange="4.084e+02...4.642e+02"} 2 123

app/vmselect/vmui/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
-    "main.css": "./static/css/main.8d8c45cf.css",
-    "main.js": "./static/js/main.d5e360af.js",
+    "main.css": "./static/css/main.0d9f8101.css",
+    "main.js": "./static/js/main.ba695a31.js",
     "static/js/27.c1ccfd29.chunk.js": "./static/js/27.c1ccfd29.chunk.js",
     "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
     "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.8d8c45cf.css",
-    "static/js/main.d5e360af.js"
+    "static/css/main.0d9f8101.css",
+    "static/js/main.ba695a31.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.d5e360af.js"></script><link href="./static/css/main.8d8c45cf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.ba695a31.js"></script><link href="./static/css/main.0d9f8101.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.20.2 as build-web-stage
+FROM golang:1.20.3 as build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.17.2
+FROM alpine:3.17.3
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/package-lock.json

@@ -18,6 +18,7 @@
         "@types/marked": "^4.0.2",
         "@types/node": "^17.0.21",
         "@types/qs": "^6.9.7",
+        "@types/react-input-mask": "^3.0.2",
         "@types/react-router-dom": "^5.3.3",
         "@types/webpack-env": "^1.16.3",
         "classnames": "^2.3.2",
@@ -28,6 +29,7 @@
         "marked": "^4.0.14",
         "preact": "^10.7.1",
         "qs": "^6.10.3",
+        "react-input-mask": "^2.0.4",
         "react-router-dom": "^6.3.0",
         "sass": "^1.56.0",
         "typescript": "~4.6.2",
@@ -4392,6 +4394,14 @@
         "@types/react": "*"
       }
     },
+    "node_modules/@types/react-input-mask": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/@types/react-input-mask/-/react-input-mask-3.0.2.tgz",
+      "integrity": "sha512-WTli3kUyvUqqaOLYG/so2pLqUvRb+n4qnx2He5klfqZDiQmRyD07jVIt/bco/1BrcErkPMtpOm+bHii4Oed6cQ==",
+      "dependencies": {
+        "@types/react": "*"
+      }
+    },
     "node_modules/@types/react-router": {
       "version": "5.1.20",
       "resolved": "https://registry.npmjs.org/@types/react-router/-/react-router-5.1.20.tgz",
@@ -10236,6 +10246,14 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/invariant": {
+      "version": "2.2.4",
+      "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz",
+      "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==",
+      "dependencies": {
+        "loose-envify": "^1.0.0"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.0.1.tgz",
@@ -16403,6 +16421,19 @@
       "dev": true,
       "peer": true
     },
+    "node_modules/react-input-mask": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/react-input-mask/-/react-input-mask-2.0.4.tgz",
+      "integrity": "sha512-1hwzMr/aO9tXfiroiVCx5EtKohKwLk/NT8QlJXHQ4N+yJJFyUuMT+zfTpLBwX/lK3PkuMlievIffncpMZ3HGRQ==",
+      "dependencies": {
+        "invariant": "^2.2.4",
+        "warning": "^4.0.2"
+      },
+      "peerDependencies": {
+        "react": ">=0.14.0",
+        "react-dom": ">=0.14.0"
+      }
+    },
     "node_modules/react-is": {
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
@@ -18811,6 +18842,14 @@
         "makeerror": "1.0.12"
       }
     },
+    "node_modules/warning": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/warning/-/warning-4.0.3.tgz",
+      "integrity": "sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==",
+      "dependencies": {
+        "loose-envify": "^1.0.0"
+      }
+    },
     "node_modules/watchpack": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz",
@@ -18851,9 +18890,9 @@
       }
     },
     "node_modules/webpack": {
-      "version": "5.75.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz",
-      "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==",
+      "version": "5.76.2",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.2.tgz",
+      "integrity": "sha512-Th05ggRm23rVzEOlX8y67NkYCHa9nTNcwHPBhdg+lKG+mtiW7XgggjAeeLnADAe7mLjJ6LUNfgHAuRRh+Z6J7w==",
       "dev": true,
       "peer": true,
       "dependencies": {

app/vmui/packages/vmui/package.json

@@ -14,6 +14,7 @@
     "@types/marked": "^4.0.2",
     "@types/node": "^17.0.21",
     "@types/qs": "^6.9.7",
+    "@types/react-input-mask": "^3.0.2",
     "@types/react-router-dom": "^5.3.3",
     "@types/webpack-env": "^1.16.3",
     "classnames": "^2.3.2",
@@ -24,6 +25,7 @@
     "marked": "^4.0.14",
     "preact": "^10.7.1",
     "qs": "^6.10.3",
+    "react-input-mask": "^2.0.4",
     "react-router-dom": "^6.3.0",
     "sass": "^1.56.0",
     "typescript": "~4.6.2",

app/vmui/packages/vmui/src/api/tsdb.ts

@@ -1,6 +1,5 @@
 export interface CardinalityRequestsParams {
   topN: number,
-  extraLabel: string | null,
   match: string | null,
   date: string | null,
   focusLabel: string | null,

app/vmui/packages/vmui/src/components/Chart/GraphTips/GraphTips.tsx

@@ -0,0 +1,53 @@
+import React, { FC } from "preact/compat";
+import "./style.scss";
+import Button from "../../Main/Button/Button";
+import { TipIcon } from "../../Main/Icons";
+import Tooltip from "../../Main/Tooltip/Tooltip";
+import Modal from "../../Main/Modal/Modal";
+import useBoolean from "../../../hooks/useBoolean";
+import tips from "./contants/tips";
+
+const GraphTips: FC = () => {
+  const {
+    value: showTips,
+    setFalse: handleCloseTips,
+    setTrue: handleOpenTips
+  } = useBoolean(false);
+
+  return (
+    <>
+      <Tooltip title={"Show tips on working with the graph"}>
+        <Button
+          variant="text"
+          color={"gray"}
+          startIcon={<TipIcon/>}
+          onClick={handleOpenTips}
+        />
+      </Tooltip>
+      {showTips && (
+        <Modal
+          title={"Tips on working with the graph and the legend"}
+          onClose={handleCloseTips}
+        >
+          <div className="fc-graph-tips">
+            {tips.map(({ title, description }) => (
+              <div
+                className="fc-graph-tips-item"
+                key={title}
+              >
+                <h4 className="fc-graph-tips-item__action">
+                  {title}
+                </h4>
+                <p className="fc-graph-tips-item__description">
+                  {description}
+                </p>
+              </div>
+            ))}
+          </div>
+        </Modal>
+      )}
+    </>
+  );
+};
+
+export default GraphTips;

app/vmui/packages/vmui/src/components/Chart/GraphTips/contants/tips.tsx

@@ -0,0 +1,68 @@
+import React from "react";
+import { isMacOs } from "../../../../utils/detect-device";
+import { DragIcon, SettingsIcon } from "../../../Main/Icons";
+
+const metaKey = <code>{isMacOs() ? "Cmd" : "Ctrl"}</code>;
+
+const graphTips = [
+  {
+    title: "Zoom in",
+    description: <>
+      To zoom in, hold down the {metaKey} + <code>scroll up</code>, or press the <code>+</code>.
+      Also, you can zoom in on a range on the graph by holding down your mouse button and selecting the range.
+    </>,
+  },
+  {
+    title: "Zoom out",
+    description: <>
+      To zoom out, hold down the {metaKey} + <code>scroll down</code>, or press the <code>-</code>.
+    </>,
+  },
+  {
+    title: "Move horizontal axis",
+    description: <>
+      To move the graph, hold down the {metaKey} + <code>drag</code> the graph to the right or left.
+    </>,
+  },
+  {
+    title: "Fixing a tooltip",
+    description: <>
+      To fix the tooltip, <code>click</code> mouse when it&#39;s open.
+      Then, you can drag the fixed tooltip by <code>clicking</code> and <code>dragging</code> on the <DragIcon/> icon.
+    </>
+  },
+  {
+    title: "Set a custom range for the vertical axis",
+    description: <>
+      To set a custom range for the vertical axis,
+      click on the <SettingsIcon/> icon located in the upper right corner of the graph,
+      activate the toggle, and set the values.
+    </>
+  },
+];
+
+const legendTips = [
+  {
+    title: "Show/hide a legend item",
+    description: <>
+      <code>click</code> on a legend item to isolate it on the graph.
+      {metaKey} + <code>click</code> on a legend item to remove it from the graph.
+      To revert to the previous state, click again.
+    </>
+  },
+  {
+    title: "Copy label key-value pairs",
+    description: <>
+      <code>click</code> on a label key-value pair to save it to the clipboard.
+    </>
+  },
+  {
+    title: "Collapse/Expand the legend group",
+    description: <>
+      <code>click</code> on the group name (e.g. <b>Query 1: &#123;__name__!=&#34;&#34;&#125;</b>)
+      to collapse or expand the legend.
+    </>
+  },
+];
+
+export default graphTips.concat(legendTips);

app/vmui/packages/vmui/src/components/Chart/GraphTips/style.scss

@@ -0,0 +1,49 @@
+@use "src/styles/variables" as *;
+
+.fc-graph-tips {
+  max-width: 520px;
+  display: grid;
+  gap: $padding-global;
+
+  &-item {
+    display: grid;
+    gap: $padding-small;
+    line-height: 1.3;
+    padding-bottom: $padding-global;
+    border-bottom: $border-divider;
+
+    &__action {
+      color: $color-text-secondary;
+      font-weight: bold;
+    }
+
+    &__description {
+      display: inline-block;
+      line-height: 20px;
+
+      svg,
+      code {
+        min-height: 20px;
+        min-width: 20px;
+        display: inline-flex;
+        align-items: center;
+        justify-content: center;
+        padding: 0 4px;
+        font-size: $font-size-small;
+        color: $color-text;
+        background-color: $color-background-body;
+        border: $border-divider;
+        border-radius: $border-radius-small;
+        margin: 0 2px 2px;
+      }
+    }
+
+    svg {
+      width: 18px;
+      color: $color-primary;
+      padding: 2px;
+      margin-top: -8px;
+      transform: translateY(8px);
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Chart/Heatmap/ChartTooltipHeatmap/ChartTooltipHeatmap.tsx

@@ -0,0 +1,152 @@
+import React, { FC, useEffect, useMemo, useRef, useState } from "preact/compat";
+import uPlot from "uplot";
+import ReactDOM from "react-dom";
+import Button from "../../../Main/Button/Button";
+import { CloseIcon, DragIcon } from "../../../Main/Icons";
+import classNames from "classnames";
+import { MouseEvent as ReactMouseEvent } from "react";
+import "../../Line/ChartTooltip/style.scss";
+
+export interface TooltipHeatmapProps  {
+  cursor: {left: number, top: number}
+  startDate: string,
+  endDate: string,
+  bucket: string,
+  value: number,
+  valueFormat: string
+}
+
+export interface ChartTooltipHeatmapProps extends TooltipHeatmapProps {
+  id: string,
+  u: uPlot,
+  unit?: string,
+  isSticky?: boolean,
+  tooltipOffset: { left: number, top: number },
+  onClose?: (id: string) => void
+}
+
+const ChartTooltipHeatmap: FC<ChartTooltipHeatmapProps> = ({
+  u,
+  id,
+  unit = "",
+  cursor,
+  tooltipOffset,
+  isSticky,
+  onClose,
+  startDate,
+  endDate,
+  bucket,
+  valueFormat,
+  value
+}) => {
+  const tooltipRef = useRef<HTMLDivElement>(null);
+
+  const [position, setPosition] = useState({ top: -999, left: -999 });
+  const [moving, setMoving] = useState(false);
+  const [moved, setMoved] = useState(false);
+
+  const targetPortal = useMemo(() => u.root.querySelector(".u-wrap"), [u]);
+
+  const handleClose = () => {
+    onClose && onClose(id);
+  };
+
+  const handleMouseDown = (e: ReactMouseEvent<HTMLButtonElement, MouseEvent>) => {
+    setMoved(true);
+    setMoving(true);
+    const { clientX, clientY } = e;
+    setPosition({ top: clientY, left: clientX });
+  };
+
+  const handleMouseMove = (e: MouseEvent) => {
+    if (!moving) return;
+    const { clientX, clientY } = e;
+    setPosition({ top: clientY, left: clientX });
+  };
+
+  const handleMouseUp = () => {
+    setMoving(false);
+  };
+
+  const calcPosition = () => {
+    if (!tooltipRef.current) return;
+
+    const topOnChart = cursor.top;
+    const leftOnChart = cursor.left;
+    const { width: tooltipWidth, height: tooltipHeight } = tooltipRef.current.getBoundingClientRect();
+    const { width, height } = u.over.getBoundingClientRect();
+
+    const margin = 10;
+    const overflowX = leftOnChart + tooltipWidth >= width ? tooltipWidth + (2 * margin) : 0;
+    const overflowY = topOnChart + tooltipHeight >= height ? tooltipHeight + (2 * margin) : 0;
+
+    setPosition({
+      top: topOnChart + tooltipOffset.top + margin - overflowY,
+      left: leftOnChart + tooltipOffset.left + margin - overflowX
+    });
+  };
+
+  useEffect(calcPosition, [u, cursor, tooltipOffset, tooltipRef]);
+
+  useEffect(() => {
+    if (moving) {
+      document.addEventListener("mousemove", handleMouseMove);
+      document.addEventListener("mouseup", handleMouseUp);
+    }
+
+    return () => {
+      document.removeEventListener("mousemove", handleMouseMove);
+      document.removeEventListener("mouseup", handleMouseUp);
+    };
+  }, [moving]);
+
+  if (!targetPortal || !cursor.left || !cursor.top || !value) return null;
+
+  return ReactDOM.createPortal((
+    <div
+      className={classNames({
+        "vm-chart-tooltip": true,
+        "vm-chart-tooltip_sticky": isSticky,
+        "vm-chart-tooltip_moved": moved
+
+      })}
+      ref={tooltipRef}
+      style={position}
+    >
+      <div className="vm-chart-tooltip-header">
+        <div className="vm-chart-tooltip-header__date vm-chart-tooltip-header__date_range">
+          <span>{startDate}</span>
+          <span>{endDate}</span>
+        </div>
+        {isSticky && (
+          <>
+            <Button
+              className="vm-chart-tooltip-header__drag"
+              variant="text"
+              size="small"
+              startIcon={<DragIcon/>}
+              onMouseDown={handleMouseDown}
+            />
+            <Button
+              className="vm-chart-tooltip-header__close"
+              variant="text"
+              size="small"
+              startIcon={<CloseIcon/>}
+              onClick={handleClose}
+            />
+          </>
+        )}
+      </div>
+      <div className="vm-chart-tooltip-data">
+        <p>
+          value: <b className="vm-chart-tooltip-data__value">{valueFormat}</b>{unit}
+        </p>
+      </div>
+      <div className="vm-chart-tooltip-info">
+        {bucket}
+      </div>
+    </div>
+  ), targetPortal);
+};
+
+export default ChartTooltipHeatmap;

app/vmui/packages/vmui/src/components/Chart/Heatmap/HeatmapChart/HeatmapChart.tsx

@@ -0,0 +1,376 @@
+import React, { FC, useCallback, useEffect, useMemo, useRef, useState } from "preact/compat";
+import uPlot, {
+  AlignedData as uPlotData,
+  Options as uPlotOptions,
+  Range
+} from "uplot";
+import { defaultOptions, sizeAxis } from "../../../../utils/uplot/helpers";
+import { dragChart } from "../../../../utils/uplot/events";
+import { getAxes } from "../../../../utils/uplot/axes";
+import { MetricResult } from "../../../../api/types";
+import { dateFromSeconds, formatDateForNativeInput, limitsDurations } from "../../../../utils/time";
+import throttle from "lodash.throttle";
+import useResize from "../../../../hooks/useResize";
+import { TimeParams } from "../../../../types";
+import { YaxisState } from "../../../../state/graph/reducer";
+import "uplot/dist/uPlot.min.css";
+import classNames from "classnames";
+import dayjs from "dayjs";
+import { useAppState } from "../../../../state/common/StateContext";
+import { heatmapPaths } from "../../../../utils/uplot/heatmap";
+import { DATE_FULL_TIMEZONE_FORMAT } from "../../../../constants/date";
+import ChartTooltipHeatmap, {
+  ChartTooltipHeatmapProps,
+  TooltipHeatmapProps
+} from "../ChartTooltipHeatmap/ChartTooltipHeatmap";
+
+export interface HeatmapChartProps {
+  metrics: MetricResult[];
+  data: uPlotData;
+  period: TimeParams;
+  yaxis: YaxisState;
+  unit?: string;
+  setPeriod: ({ from, to }: {from: Date, to: Date}) => void;
+  container: HTMLDivElement | null;
+  height?: number;
+  onChangeLegend: (val: TooltipHeatmapProps) => void;
+}
+
+enum typeChartUpdate {xRange = "xRange", yRange = "yRange"}
+
+const HeatmapChart: FC<HeatmapChartProps> = ({
+  data,
+  metrics = [],
+  period,
+  yaxis,
+  unit,
+  setPeriod,
+  container,
+  height,
+  onChangeLegend,
+}) => {
+  const { isDarkTheme } = useAppState();
+
+  const uPlotRef = useRef<HTMLDivElement>(null);
+  const [isPanning, setPanning] = useState(false);
+  const [xRange, setXRange] = useState({ min: period.start, max: period.end });
+  const [uPlotInst, setUPlotInst] = useState<uPlot>();
+  const [startTouchDistance, setStartTouchDistance] = useState(0);
+  const layoutSize = useResize(container);
+
+  const [tooltipProps, setTooltipProps] = useState<TooltipHeatmapProps | null>(null);
+  const [tooltipOffset, setTooltipOffset] = useState({ left: 0, top: 0 });
+  const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipHeatmapProps[]>([]);
+  const tooltipId = useMemo(() => {
+    return `${tooltipProps?.bucket}_${tooltipProps?.startDate}`;
+  }, [tooltipProps]);
+
+  const setScale = ({ min, max }: { min: number, max: number }): void => {
+    if (isNaN(min) || isNaN(max)) return;
+    setPeriod({
+      from: dayjs(min * 1000).toDate(),
+      to: dayjs(max * 1000).toDate()
+    });
+  };
+  const throttledSetScale = useCallback(throttle(setScale, 500), []);
+  const setPlotScale = ({ u, min, max }: { u: uPlot, min: number, max: number }) => {
+    const delta = (max - min) * 1000;
+    if ((delta < limitsDurations.min) || (delta > limitsDurations.max)) return;
+    u.setScale("x", { min, max });
+    setXRange({ min, max });
+    throttledSetScale({ min, max });
+  };
+
+  const onReadyChart = (u: uPlot) => {
+    const factor = 0.9;
+    setTooltipOffset({
+      left: parseFloat(u.over.style.left),
+      top: parseFloat(u.over.style.top)
+    });
+
+    u.over.addEventListener("mousedown", e => {
+      const { ctrlKey, metaKey, button } = e;
+      const leftClick = button === 0;
+      const leftClickWithMeta = leftClick && (ctrlKey || metaKey);
+      if (leftClickWithMeta) {
+        // drag pan
+        dragChart({ u, e, setPanning, setPlotScale, factor });
+      }
+    });
+
+    u.over.addEventListener("touchstart", e => {
+      dragChart({ u, e, setPanning, setPlotScale, factor });
+    });
+
+    u.over.addEventListener("wheel", e => {
+      if (!e.ctrlKey && !e.metaKey) return;
+      e.preventDefault();
+      const { width } = u.over.getBoundingClientRect();
+      const zoomPos = u.cursor.left && u.cursor.left > 0 ? u.cursor.left : 0;
+      const xVal = u.posToVal(zoomPos, "x");
+      const oxRange = (u.scales.x.max || 0) - (u.scales.x.min || 0);
+      const nxRange = e.deltaY < 0 ? oxRange * factor : oxRange / factor;
+      const min = xVal - (zoomPos / width) * nxRange;
+      const max = min + nxRange;
+      u.batch(() => setPlotScale({ u, min, max }));
+    });
+  };
+
+  const handleKeyDown = (e: KeyboardEvent) => {
+    const { target, ctrlKey, metaKey, key } = e;
+    const isInput = target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement;
+    if (!uPlotInst || isInput) return;
+    const minus = key === "-";
+    const plus = key === "+" || key === "=";
+    if ((minus || plus) && !(ctrlKey || metaKey)) {
+      e.preventDefault();
+      const factor = (xRange.max - xRange.min) / 10 * (plus ? 1 : -1);
+      setPlotScale({
+        u: uPlotInst,
+        min: xRange.min + factor,
+        max: xRange.max - factor
+      });
+    }
+  };
+
+  const handleClick = () => {
+    if (!tooltipProps) return;
+    const id = `${tooltipProps?.bucket}_${tooltipProps?.startDate}`;
+    const props = {
+      id,
+      unit,
+      tooltipOffset,
+      ...tooltipProps
+    };
+
+    if (!stickyTooltips.find(t => t.id === id)) {
+      const res = JSON.parse(JSON.stringify(props));
+      setStickyToolTips(prev => [...prev, res]);
+    }
+  };
+
+  const handleUnStick = (id:string) => {
+    setStickyToolTips(prev => prev.filter(t => t.id !== id));
+  };
+
+
+  const setCursor = (u: uPlot) => {
+    const left = u.cursor.left && u.cursor.left > 0 ? u.cursor.left : 0;
+    const top = u.cursor.top && u.cursor.top > 0 ? u.cursor.top : 0;
+
+    const xArr = (u.data[1][0] || []) as number[];
+    if (!Array.isArray(xArr)) return;
+    const xVal = u.posToVal(left, "x");
+    const yVal = u.posToVal(top, "y");
+    const xIdx = xArr.findIndex((t, i) => xVal >= t && xVal < xArr[i + 1]) || -1;
+    const second = xArr[xIdx + 1];
+
+    const result = metrics[Math.round(yVal)];
+    if (!result) {
+      setTooltipProps(null);
+      return;
+    }
+
+    const [endTime = 0, value = ""] = result.values.find(v => v[0] === second) || [];
+    const valueFormat = `${+value}%`;
+    const startTime = xArr[xIdx];
+    const startDate = dayjs(startTime * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT);
+    const endDate = dayjs(endTime * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT);
+
+    setTooltipProps({
+      cursor: { left, top },
+      startDate,
+      endDate,
+      bucket: result?.metric?.vmrange || "",
+      value: +value,
+      valueFormat: valueFormat,
+    });
+  };
+
+  const getRangeX = (): Range.MinMax => [xRange.min, xRange.max];
+
+  const axes = getAxes( [{}], unit);
+  const options: uPlotOptions = {
+    ...defaultOptions,
+    mode: 2,
+    tzDate: ts => dayjs(formatDateForNativeInput(dateFromSeconds(ts))).local().toDate(),
+    series: [
+      {},
+      {
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore
+        paths: heatmapPaths(),
+        facets: [
+          {
+            scale: "x",
+            auto: true,
+            sorted: 1,
+          },
+          {
+            scale: "y",
+            auto: true,
+          },
+        ],
+      },
+    ],
+    axes: [
+      ...axes,
+      {
+        scale: "y",
+        stroke: axes[0].stroke,
+        font: axes[0].font,
+        size: sizeAxis,
+        splits: metrics.map((m, i) => i),
+        values: metrics.map(m => m.metric.vmrange),
+      }
+    ],
+    scales: {
+      x: {
+        time: true,
+      },
+      y: {
+        log: 2,
+        time: false,
+        range: (self, initMin, initMax) => [initMin - 1, initMax + 1]
+      }
+    },
+    width: layoutSize.width || 400,
+    height: height || 500,
+    plugins: [{ hooks: { ready: onReadyChart, setCursor } }],
+    hooks: {
+      setSelect: [
+        (u) => {
+          const min = u.posToVal(u.select.left, "x");
+          const max = u.posToVal(u.select.left + u.select.width, "x");
+          setPlotScale({ u, min, max });
+        }
+      ]
+    },
+  };
+
+  const updateChart = (type: typeChartUpdate): void => {
+    if (!uPlotInst) return;
+    switch (type) {
+      case typeChartUpdate.xRange:
+        uPlotInst.scales.x.range = getRangeX;
+        break;
+    }
+    if (!isPanning) uPlotInst.redraw();
+  };
+
+  useEffect(() => setXRange({ min: period.start, max: period.end }), [period]);
+
+  useEffect(() => {
+    setStickyToolTips([]);
+    setTooltipProps(null);
+    if (!uPlotRef.current || !layoutSize.width || !layoutSize.height) return;
+    const u = new uPlot(options, data, uPlotRef.current);
+    setUPlotInst(u);
+    setXRange({ min: period.start, max: period.end });
+    return u.destroy;
+  }, [uPlotRef.current, layoutSize, height, isDarkTheme, data]);
+
+  useEffect(() => {
+    window.addEventListener("keydown", handleKeyDown);
+
+    return () => {
+      window.removeEventListener("keydown", handleKeyDown);
+    };
+  }, [xRange]);
+
+  const handleTouchStart = (e: TouchEvent) => {
+    if (e.touches.length !== 2) return;
+    e.preventDefault();
+
+    const dx = e.touches[0].clientX - e.touches[1].clientX;
+    const dy = e.touches[0].clientY - e.touches[1].clientY;
+    setStartTouchDistance(Math.sqrt(dx * dx + dy * dy));
+  };
+
+  const handleTouchMove = (e: TouchEvent) => {
+    if (e.touches.length !== 2 || !uPlotInst) return;
+    e.preventDefault();
+
+    const dx = e.touches[0].clientX - e.touches[1].clientX;
+    const dy = e.touches[0].clientY - e.touches[1].clientY;
+    const endTouchDistance = Math.sqrt(dx * dx + dy * dy);
+    const diffDistance = startTouchDistance - endTouchDistance;
+
+    const max = (uPlotInst.scales.x.max || xRange.max);
+    const min = (uPlotInst.scales.x.min || xRange.min);
+    const dur = max - min;
+    const dir = (diffDistance > 0 ? -1 : 1);
+
+    const zoomFactor = dur / 50 * dir;
+    uPlotInst.batch(() => setPlotScale({
+      u: uPlotInst,
+      min: min + zoomFactor,
+      max: max - zoomFactor
+    }));
+  };
+
+  useEffect(() => {
+    window.addEventListener("touchmove", handleTouchMove);
+    window.addEventListener("touchstart", handleTouchStart);
+
+    return () => {
+      window.removeEventListener("touchmove", handleTouchMove);
+      window.removeEventListener("touchstart", handleTouchStart);
+    };
+  }, [uPlotInst, startTouchDistance]);
+
+  useEffect(() => updateChart(typeChartUpdate.xRange), [xRange]);
+  useEffect(() => updateChart(typeChartUpdate.yRange), [yaxis]);
+
+  useEffect(() => {
+    const show = !!tooltipProps?.value;
+    if (show) window.addEventListener("click", handleClick);
+
+    return () => {
+      window.removeEventListener("click", handleClick);
+    };
+  }, [tooltipProps, stickyTooltips]);
+
+  useEffect(() => {
+    if (tooltipProps) onChangeLegend(tooltipProps);
+  }, [tooltipProps]);
+
+  return (
+    <div
+      className={classNames({
+        "vm-line-chart": true,
+        "vm-line-chart_panning": isPanning
+      })}
+      style={{
+        minWidth: `${layoutSize.width || 400}px`,
+        minHeight: `${height || 500}px`
+      }}
+    >
+      <div
+        className="vm-line-chart__u-plot"
+        ref={uPlotRef}
+      />
+      {uPlotInst && tooltipProps && (
+        <ChartTooltipHeatmap
+          {...tooltipProps}
+          unit={unit}
+          u={uPlotInst}
+          tooltipOffset={tooltipOffset}
+          id={tooltipId}
+        />
+      )}
+
+      {uPlotInst && stickyTooltips.map(t => (
+        <ChartTooltipHeatmap
+          {...t}
+          isSticky
+          u={uPlotInst}
+          key={t.id}
+          onClose={handleUnStick}
+        />
+      ))}
+    </div>
+  );
+};
+
+export default HeatmapChart;

app/vmui/packages/vmui/src/components/Chart/Heatmap/LegendHeatmap/LegendHeatmap.tsx

@@ -0,0 +1,68 @@
+import React, { FC, useEffect, useState } from "preact/compat";
+import { gradMetal16 } from "../../../../utils/uplot/heatmap";
+import "./style.scss";
+import { TooltipHeatmapProps } from "../ChartTooltipHeatmap/ChartTooltipHeatmap";
+import { SeriesItem } from "../../../../utils/uplot/series";
+import LegendItem from "../../Line/Legend/LegendItem/LegendItem";
+import { LegendItemType } from "../../../../utils/uplot/types";
+
+interface LegendHeatmapProps {
+  min: number
+  max: number
+  legendValue: TooltipHeatmapProps | null,
+  series: SeriesItem[]
+}
+
+const LegendHeatmap: FC<LegendHeatmapProps> = (
+  {
+    min,
+    max,
+    legendValue,
+    series,
+  }
+) => {
+
+  const [percent, setPercent] = useState(0);
+  const [valueFormat, setValueFormat] = useState("");
+  const [minFormat, setMinFormat] = useState("");
+  const [maxFormat, setMaxFormat] = useState("");
+
+  useEffect(() => {
+    const value = legendValue?.value || 0;
+    setPercent(value ? (value - min) / (max - min) * 100 : 0);
+    setValueFormat(value ? `${value}%` : "");
+    setMinFormat(`${min}%`);
+    setMaxFormat(`${max}%`);
+  }, [legendValue, min, max]);
+
+  return (
+    <div className="vm-legend-heatmap__wrapper">
+      <div className="vm-legend-heatmap">
+        <div
+          className="vm-legend-heatmap-gradient"
+          style={{ background: `linear-gradient(to right, ${gradMetal16.join(", ")})` }}
+        >
+          {!!legendValue?.value && (
+            <div
+              className="vm-legend-heatmap-gradient__value"
+              style={{ left: `${percent}%` }}
+            >
+              <span>{valueFormat}</span>
+            </div>
+          )}
+        </div>
+        <div className="vm-legend-heatmap__value">{minFormat}</div>
+        <div className="vm-legend-heatmap__value">{maxFormat}</div>
+      </div>
+      {series[1] && (
+        <LegendItem
+          key={series[1]?.label}
+          legend={series[1] as LegendItemType}
+          isHeatmap
+        />
+      )}
+    </div>
+  );
+};
+
+export default LegendHeatmap;

app/vmui/packages/vmui/src/components/Chart/Heatmap/LegendHeatmap/style.scss

@@ -0,0 +1,67 @@
+@use "src/styles/variables" as *;
+
+.vm-legend-heatmap {
+  display: inline-grid;
+  grid-template-columns: auto auto;
+  align-items: center;
+  justify-content: space-between;
+  gap: 4px;
+
+  &__wrapper {
+    display: flex;
+    align-items: flex-start;
+    justify-content: space-between;
+    gap: $padding-global;
+    flex-wrap: wrap;
+  }
+
+  &__value {
+    color: $color-text;
+    font-size: $font-size-small;
+
+    &:last-child {
+      text-align: right;
+    }
+  }
+
+  &-gradient {
+    $gradient-height: $font-size-small;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    position: relative;
+    grid-column: 1/-1;
+    height: $gradient-height;
+    width: 200px;
+
+    &__value {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      position: absolute;
+      top: -2px;
+      border-radius: 50%;
+      border: 2px solid $color-text;
+      width: calc($gradient-height + 4px);
+      height: calc($gradient-height + 4px);
+
+      transform: translateX(calc(($gradient-height/-2) - 2px));
+      transition: left 100ms ease;
+
+      span {
+        position: absolute;
+        top: calc($gradient-height + 6px);
+        left: auto;
+        padding: 4px 8px;
+        color: $color-text;
+        font-size: $font-size-small;
+        background-color: $color-background-block;
+        box-shadow: $box-shadow;
+      }
+    }
+  }
+
+  &__labels {
+    word-break: break-all;
+  }
+}

app/vmui/packages/vmui/src/components/Chart/Legend/Legend.tsx

@@ -1,44 +0,0 @@
-import React, { FC, useMemo } from "preact/compat";
-import { LegendItemType } from "../../../utils/uplot/types";
-import LegendItem from "./LegendItem/LegendItem";
-import "./style.scss";
-
-interface LegendProps {
-  labels: LegendItemType[];
-  query: string[];
-  onChange: (item: LegendItemType, metaKey: boolean) => void;
-}
-
-const Legend: FC<LegendProps> = ({ labels, query, onChange }) => {
-  const groups = useMemo(() => {
-    return Array.from(new Set(labels.map(l => l.group)));
-  }, [labels]);
-  const showQueryNum = groups.length > 1;
-
-  return <>
-    <div className="vm-legend">
-      {groups.map((group) => <div
-        className="vm-legend-group"
-        key={group}
-      >
-        <div className="vm-legend-group-title">
-          {showQueryNum && (
-            <span className="vm-legend-group-title__count">Query {group}: </span>
-          )}
-          <span className="vm-legend-group-title__query">{query[group - 1]}</span>
-        </div>
-        <div>
-          {labels.filter(l => l.group === group).map((legendItem: LegendItemType) =>
-            <LegendItem
-              key={legendItem.label}
-              legend={legendItem}
-              onChange={onChange}
-            />
-          )}
-        </div>
-      </div>)}
-    </div>
-  </>;
-};
-
-export default Legend;

app/vmui/packages/vmui/src/components/Chart/Line/ChartTooltip/ChartTooltip.tsx

@@ -1,17 +1,17 @@
 import React, { FC, useEffect, useMemo, useRef, useState } from "preact/compat";
 import uPlot from "uplot";
-import { MetricResult } from "../../../api/types";
-import { formatPrettyNumber } from "../../../utils/uplot/helpers";
+import { MetricResult } from "../../../../api/types";
+import { formatPrettyNumber } from "../../../../utils/uplot/helpers";
 import dayjs from "dayjs";
-import { DATE_FULL_TIMEZONE_FORMAT } from "../../../constants/date";
+import { DATE_FULL_TIMEZONE_FORMAT } from "../../../../constants/date";
 import ReactDOM from "react-dom";
 import get from "lodash.get";
-import Button from "../../Main/Button/Button";
-import { CloseIcon, DragIcon } from "../../Main/Icons";
+import Button from "../../../Main/Button/Button";
+import { CloseIcon, DragIcon } from "../../../Main/Icons";
 import classNames from "classnames";
 import { MouseEvent as ReactMouseEvent } from "react";
 import "./style.scss";
-import { SeriesItem } from "../../../utils/uplot/series";
+import { SeriesItem } from "../../../../utils/uplot/series";
 
 export interface ChartTooltipProps {
   id: string,

app/vmui/packages/vmui/src/components/Chart/Line/ChartTooltip/style.scss

@@ -51,6 +51,13 @@ $chart-tooltip-y: -1 * ($padding-small + $chart-tooltip-half-icon);
       color: $color-white;
       cursor: move;
     }
+
+    &__date {
+      &_range {
+        display: grid;
+        gap: 2px;
+      }
+    }
   }
 
   &-data {
@@ -71,5 +78,6 @@ $chart-tooltip-y: -1 * ($padding-small + $chart-tooltip-half-icon);
     display: grid;
     grid-gap: 4px;
     word-break: break-all;
+    white-space: pre-wrap;
   }
 }

app/vmui/packages/vmui/src/components/Chart/Line/Legend/Legend.tsx

@@ -0,0 +1,53 @@
+import React, { FC, useMemo } from "preact/compat";
+import { LegendItemType } from "../../../../utils/uplot/types";
+import LegendItem from "./LegendItem/LegendItem";
+import Accordion from "../../../Main/Accordion/Accordion";
+import "./style.scss";
+
+interface LegendProps {
+  labels: LegendItemType[];
+  query: string[];
+  onChange: (item: LegendItemType, metaKey: boolean) => void;
+}
+
+const Legend: FC<LegendProps> = ({ labels, query, onChange }) => {
+  const groups = useMemo(() => {
+    return Array.from(new Set(labels.map(l => l.group)));
+  }, [labels]);
+  const showQueryNum = groups.length > 1;
+
+  return <>
+    <div className="vm-legend">
+      {groups.map((group) => (
+        <div
+          className="vm-legend-group"
+          key={group}
+        >
+          <Accordion
+            defaultExpanded={true}
+            title={(
+              <div className="vm-legend-group-title">
+                {showQueryNum && (
+                  <span className="vm-legend-group-title__count">Query {group}: </span>
+                )}
+                <span className="vm-legend-group-title__query">{query[group - 1]}</span>
+              </div>
+            )}
+          >
+            <div>
+              {labels.filter(l => l.group === group).map((legendItem: LegendItemType) =>
+                <LegendItem
+                  key={legendItem.label}
+                  legend={legendItem}
+                  onChange={onChange}
+                />
+              )}
+            </div>
+          </Accordion>
+        </div>
+      ))}
+    </div>
+  </>;
+};
+
+export default Legend;

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -1,20 +1,25 @@
 import React, { FC, useState, useMemo } from "preact/compat";
 import { MouseEvent } from "react";
-import { LegendItemType } from "../../../../utils/uplot/types";
+import { LegendItemType } from "../../../../../utils/uplot/types";
 import "./style.scss";
 import classNames from "classnames";
-import Tooltip from "../../../Main/Tooltip/Tooltip";
+import Tooltip from "../../../../Main/Tooltip/Tooltip";
 import { getFreeFields } from "./helpers";
 
 interface LegendItemProps {
   legend: LegendItemType;
-  onChange: (item: LegendItemType, metaKey: boolean) => void;
+  onChange?: (item: LegendItemType, metaKey: boolean) => void;
+  isHeatmap?: boolean;
 }
 
-const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
+const LegendItem: FC<LegendItemProps> = ({ legend, onChange, isHeatmap }) => {
   const [copiedValue, setCopiedValue] = useState("");
-  const freeFormFields = useMemo(() => getFreeFields(legend), [legend]);
+  const freeFormFields = useMemo(() => {
+    const result = getFreeFields(legend);
+    return isHeatmap ? result.filter(f => f.key !== "vmrange") : result;
+  }, [legend, isHeatmap]);
   const calculations = legend.calculations;
+  const showCalculations = Object.values(calculations).some(v => v);
 
   const handleClickFreeField = async (val: string, id: string) => {
     await navigator.clipboard.writeText(val);
@@ -23,7 +28,7 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
   };
 
   const createHandlerClick = (legend: LegendItemType) => (e: MouseEvent<HTMLDivElement>) => {
-    onChange(legend, e.ctrlKey || e.metaKey);
+    onChange && onChange(legend, e.ctrlKey || e.metaKey);
   };
 
   const createHandlerCopy = (freeField: string, id: string) => (e: MouseEvent<HTMLDivElement>) => {
@@ -36,18 +41,21 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
       className={classNames({
         "vm-legend-item": true,
         "vm-legend-row": true,
-        "vm-legend-item_hide": !legend.checked,
+        "vm-legend-item_hide": !legend.checked && !isHeatmap,
+        "vm-legend-item_static": isHeatmap,
       })}
       onClick={createHandlerClick(legend)}
     >
-      <div
-        className="vm-legend-item__marker"
-        style={{ backgroundColor: legend.color }}
-      />
+      {!isHeatmap && (
+        <div
+          className="vm-legend-item__marker"
+          style={{ backgroundColor: legend.color }}
+        />
+      )}
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
           {legend.freeFormFields["__name__"]}
-          &#123;
+          {!!freeFormFields.length && <>&#123;</>}
           {freeFormFields.map((f, i) => (
             <Tooltip
               key={f.id}
@@ -65,12 +73,14 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
               </span>
             </Tooltip>
           ))}
-          &#125;
+          {!!freeFormFields.length && <>&#125;</>}
         </span>
       </div>
-      <div className="vm-legend-item-values">
-        median:{calculations.median}, min:{calculations.min}, max:{calculations.max}, last:{calculations.last}
-      </div>
+      {!isHeatmap && showCalculations && (
+        <div className="vm-legend-item-values">
+          median:{calculations.median}, min:{calculations.min}, max:{calculations.max}, last:{calculations.last}
+        </div>
+      )}
     </div>
   );
 };

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/helpers.ts

@@ -1,4 +1,4 @@
-import { LegendItemType } from "../../../../utils/uplot/types";
+import { LegendItemType } from "../../../../../utils/uplot/types";
 
 export const getFreeFields = (legend: LegendItemType) => {
   const keys = Object.keys(legend.freeFormFields).filter(f => f !== "__name__");

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/style.scss

@@ -21,6 +21,17 @@
     opacity: 0.5;
   }
 
+  &_static {
+    grid-template-columns: 1fr;
+    margin: 0;
+    padding: 0;
+    cursor: default;
+
+    &:hover {
+      background-color: $color-background-block;
+    }
+  }
+
   &__marker {
     width: 14px;
     height: 14px;

app/vmui/packages/vmui/src/components/Chart/Line/Legend/style.scss

@@ -4,7 +4,6 @@
   position: relative;
   display: flex;
   flex-wrap: wrap;
-  margin-top: $padding-medium;
   cursor: default;
 
   &-group {