go.mod: remove outdated replacement

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1736

.github/dependabot.yml

@@ -4,3 +4,23 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "bundler"
+    directory: "/docs"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/app/vmui/packages/vmui/web"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/app/vmui"
+    schedule:
+      interval: "weekly"

.github/workflows/main.yml

@@ -60,7 +60,7 @@ jobs:
           GOOS=darwin go build -mod=vendor ./app/vmctl
           CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent
       - name: Publish coverage
-        uses: codecov/codecov-action@v1.5.2
+        uses: codecov/codecov-action@v2.1.0
         with:
           file: ./coverage.txt
 

.gitignore

@@ -16,3 +16,6 @@
 /package/*.deb
 /package/*.rpm
 .DS_store
+Gemfile.lock
+/_site
+_site

Makefile

@@ -261,7 +261,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.40.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.42.1
 
 install-wwhrd:
 	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd

README.md

@@ -12,24 +12,23 @@
 
 VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.
 
-It is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
-[docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap package](https://snapcraft.io/victoriametrics)
-and in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics and see [how to start it](#how-to-start-victoriametrics).
-If you use Ubuntu, then just run `snap install victoriametrics` in order to install and run it.
+VictoriaMetrics is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
+in [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), in [Snap packages](https://snapcraft.io/victoriametrics)
+and in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just download VictoriaMetrics follow [these instructions](#how-to-start-victoriametrics).
 Then read [Prometheus setup](#prometheus-setup) and [Grafana setup](#grafana-setup) docs.
 
-Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+Cluster version of VictoriaMetrics is available [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
 
-See additional docs at our [Wiki](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki).
-
-[Contact us](mailto:info@victoriametrics.com) if you need paid enterprise support for VictoriaMetrics.
-See [features available for enterprise customers](https://victoriametrics.com/enterprise.html).
+[Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics.
+See [features available in enterprise package](https://victoriametrics.com/enterprise.html).
+Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 
 ## Case studies and talks
 
 Case studies:
 
+* [AbiosGaming](https://docs.victoriametrics.com/CaseStudies.html#abiosgaming)
 * [adidas](https://docs.victoriametrics.com/CaseStudies.html#adidas)
 * [Adsterra](https://docs.victoriametrics.com/CaseStudies.html#adsterra)
 * [ARNES](https://docs.victoriametrics.com/CaseStudies.html#arnes)
@@ -37,11 +36,16 @@ Case studies:
 * [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
 * [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
 * [Dreamteam](https://docs.victoriametrics.com/CaseStudies.html#dreamteam)
+* [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
 * [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
+* [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
 * [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
 * [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
 * [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
+* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
+* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
 * [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia)
+* [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets)
 * [Synthesio](https://docs.victoriametrics.com/CaseStudies.html#synthesio)
 * [Wedos.com](https://docs.victoriametrics.com/CaseStudies.html#wedoscom)
 * [Wix.com](https://docs.victoriametrics.com/CaseStudies.html#wixcom)
@@ -53,188 +57,101 @@ See also [articles and slides about VictoriaMetrics from our users](https://docs
 
 ## Prominent features
 
-* VictoriaMetrics can be used as long-term storage for Prometheus or for [vmagent](https://docs.victoriametrics.com/vmagent.html).
-  See [these docs](#prometheus-setup) for details.
-* VictoriaMetrics supports [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/), so it can be used as Prometheus drop-in replacement in Grafana.
-* VictoriaMetrics implements [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) query language backwards compatible with PromQL.
-* VictoriaMetrics provides global query view. Multiple Prometheus instances or any other data sources may ingest data into VictoriaMetrics.
-  Later this data may be queried via a single query.
-* High performance and good scalability for both [inserts](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
-  and [selects](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
-  [Outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
-* [Uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893)
-  and [up to 7x less RAM than Prometheus, Thanos or Cortex](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f)
-  when dealing with millions of unique time series (aka high cardinality).
-* Optimized for time series with high churn rate. Think about [prometheus-operator](https://github.com/coreos/prometheus-operator) metrics from frequent deployments in Kubernetes.
-* High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
-  may be crammed into limited storage comparing to TimescaleDB
-  and [up to 7x less storage space is required comparing to Prometheus, Thanos or Cortex](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f).
-* Optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc).
-  See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
-* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB.
-  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae),
-  [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
-  and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk
-  from [PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
-* Easy operation:
+VictoriaMetrics has the following prominent features:
+
+* It can be used as long-term storage for Prometheus. See [these docs](#prometheus-setup) for details.
+* It can be used as drop-in replacement for Prometheus in Grafana, because it supports [Prometheus querying API](#prometheus-querying-api-usage).
+* It can be used as drop-in replacement for Graphite in Grafana, because it supports [Graphite API](#graphite-api-usage).
+* It features easy setup and operation:
   * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
   * All the data is stored in a single directory pointed by `-storageDataPath` command-line flag.
-  * Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
-  to S3 or GCS with [vmbackup](https://docs.victoriametrics.com/vmbackup.html) / [vmrestore](https://docs.victoriametrics.com/vmrestore.html).
-  See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
-* Storage is protected from corruption on unclean shutdown (i.e. OOM, hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
-* Supports metrics' scraping, ingestion and [backfilling](#backfilling) via the following protocols:
-  * [Metrics from Prometheus exporters](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
-  such as [node_exporter](https://github.com/prometheus/node_exporter). See [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter) for details.
-  * [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
-  * [InfluxDB line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) over HTTP, TCP and UDP.
-  * [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon)
-    if `-graphiteListenAddr` is set.
-  * [OpenTSDB put message](#sending-data-via-telnet-put-protocol) if `-opentsdbListenAddr` is set.
-  * [HTTP OpenTSDB /api/put requests](#sending-opentsdb-data-via-http-apiput-requests) if `-opentsdbHTTPListenAddr` is set.
-  * [JSON line format](#how-to-import-data-in-json-line-format).
-  * [Native binary format](#how-to-import-data-in-native-format).
+  * Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) to S3 or GCS can be done with [vmbackup](https://docs.victoriametrics.com/vmbackup.html) / [vmrestore](https://docs.victoriametrics.com/vmrestore.html) tools. See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+* It implements PromQL-based query language - [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which provides improved functionality on top of PromQL.
+* It provides global query view. Multiple Prometheus instances or any other data sources may ingest data into VictoriaMetrics. Later this data may be queried via a single query.
+* It provides high performance and good vertical and horizontal scalability for both [data ingestion](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b) and [data querying](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4). It [outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+* It [uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) and [up to 7x less RAM than Prometheus, Thanos or Cortex](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f) when dealing with millions of unique time series (aka [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality)).
+* It is optimized for time series with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).
+* It provides high data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) may be crammed into limited storage comparing to TimescaleDB and [up to 7x less storage space is required compared to Prometheus, Thanos or Cortex](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f).
+* It is optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [disk IO graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
+* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB. See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae), [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683) and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk from [PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
+* It protects the storage from data corruption on unclean shutdown (i.e. OOM, hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+* It supports metrics' scraping, ingestion and [backfilling](#backfilling) via the following protocols:
+  * [Metrics scraping from Prometheus exporters](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
+  * [Prometheus remote write API](#prometheus-setup).
   * [Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format).
+  * [InfluxDB line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) over HTTP, TCP and UDP.
+  * [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon).
+  * [OpenTSDB put message](#sending-data-via-telnet-put-protocol).
+  * [HTTP OpenTSDB /api/put requests](#sending-opentsdb-data-via-http-apiput-requests).
+  * [JSON line format](#how-to-import-data-in-json-line-format).
   * [Arbitrary CSV data](#how-to-import-csv-data).
-* Supports metrics' relabeling. See [these docs](#relabeling) for details.
-* Can deal with high cardinality and high churn rate issues using [series limiter](#cardinality-limiter).
-* Ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various Enterprise workloads.
-* Has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
-* See also technical [Articles about VictoriaMetrics](https://docs.victoriametrics.com/Articles.html).
+  * [Native binary format](#how-to-import-data-in-native-format).
+* It supports metrics' relabeling. See [these docs](#relabeling) for details.
+* It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
+* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/enterprise.html).
+* It has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
+See also [various Articles about VictoriaMetrics](https://docs.victoriametrics.com/Articles.html).
 
 
 ## Operation
 
-### Table of contents
-
-* [How to start VictoriaMetrics](#how-to-start-victoriametrics)
-  * [Environment variables](#environment-variables)
-  * [Configuration with snap package](#configuration-with-snap-package)
-* [Prometheus setup](#prometheus-setup)
-* [Grafana setup](#grafana-setup)
-* [How to upgrade VictoriaMetrics](#how-to-upgrade-victoriametrics)
-* [How to apply new config to VictoriaMetrics](#how-to-apply-new-config-to-victoriametrics)
-* [How to scrape Prometheus exporters such as node_exporter](#how-to-scrape-prometheus-exporters-such-as-node-exporter)
-* [How to send data from InfluxDB-compatible agents such as Telegraf](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
-* [How to send data from Graphite-compatible agents such as StatsD](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
-* [Querying Graphite data](#querying-graphite-data)
-* [How to send data from OpenTSDB-compatible agents](#how-to-send-data-from-opentsdb-compatible-agents)
-* [Prometheus querying API usage](#prometheus-querying-api-usage)
-  * [Prometheus querying API enhancements](#prometheus-querying-api-enhancements)
-* [Graphite API usage](#graphite-api-usage)
-  * [Graphite Render API usage](#graphite-render-api-usage)
-  * [Graphite Metrics API usage](#graphite-metrics-api-usage)
-  * [Graphite Tags API usage](#graphite-tags-api-usage)
-* [How to build from sources](#how-to-build-from-sources)
-  * [Development build](#development-build)
-  * [Production build](#production-build)
-  * [ARM build](#arm-build)
-  * [Pure Go build (CGO_ENABLED=0)](#pure-go-build-cgo_enabled0)
-  * [Building docker images](#building-docker-images)
-* [Start with docker-compose](#start-with-docker-compose)
-* [Setting up service](#setting-up-service)
-* [How to work with snapshots](#how-to-work-with-snapshots)
-* [How to delete time series](#how-to-delete-time-series)
-* [Forced merge](#forced-merge)
-* [How to export time series](#how-to-export-time-series)
-  * [How to export data in native format](#how-to-export-data-in-native-format)
-  * [How to export data in JSON line format](#how-to-export-data-in-json-line-format)
-  * [How to export CSV data](#how-to-export-csv-data)
-* [How to import time series data](#how-to-import-time-series-data)
-  * [How to import data in native format](#how-to-import-data-in-native-format)
-  * [How to import data in json line format](#how-to-import-data-in-json-line-format)
-  * [How to import CSV data](#how-to-import-csv-data)
-  * [How to import data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format)
-* [Relabeling](#relabeling)
-* [Federation](#federation)
-* [Capacity planning](#capacity-planning)
-* [High availability](#high-availability)
-* [Deduplication](#deduplication)
-* [Retention](#retention)
-* [Multiple retentions](#multiple-retentions)
-* [Downsampling](#downsampling)
-* [Multi-tenancy](#multi-tenancy)
-* [Scalability and cluster version](#scalability-and-cluster-version)
-* [Alerting](#alerting)
-* [Security](#security)
-* [Tuning](#tuning)
-* [Monitoring](#monitoring)
-* [TSDB stats](#tsdb-stats)
-* [Cardinality limiter](#cardinality-limiter)
-* [Troubleshooting](#troubleshooting)
-* [Data migration](#data-migration)
-* [Backfilling](#backfilling)
-* [Data updates](#data-updates)
-* [Replication](#replication)
-* [Backups](#backups)
-* [Profiling](#profiling)
-* [Integrations](#integrations)
-* [Third-party contributions](#third-party-contributions)
-* [Contacts](#contacts)
-* [Community and contributions](#community-and-contributions)
-* [Reporting bugs](#reporting-bugs)
-* [VictoriaMetrics Logo](#victoria-metrics-logo)
-  * [Logo Usage Guidelines](#logo-usage-guidelines)
-    * [Font used](#font-used)
-    * [Color Palette](#color-palette)
-  * [We kindly ask](#we-kindly-ask)
-* [List of command-line flags](#list-of-command-line-flags)
-
-
 ## How to start VictoriaMetrics
 
-Start VictoriaMetrics [executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-or [docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) with the desired command-line flags.
+Just download [VictoriaMetrics executable](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or [Docker image](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and start it with the desired command-line flags.
 
 The following command-line flags are used the most:
 
-* `-storageDataPath` - path to data directory. VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in the current working directory.
+* `-storageDataPath` - VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in the current working directory.
 * `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 1 month. See [these docs](#retention) for more details.
 
 Other flags have good enough default values, so set them only if you really need this. Pass `-help` to see [all the available flags with description and default values](#list-of-command-line-flags).
 
-See how to [ingest data to VictoriaMetrics](#how-to-import-time-series-data), how to [query VictoriaMetrics](#grafana-setup)
-and how to [handle alerts](#alerting).
+See how to [ingest data to VictoriaMetrics](#how-to-import-time-series-data), how to [query VictoriaMetrics via Grafana](#grafana-setup), how to [query VictoriaMetrics via Graphite API](#graphite-api-usage) and how to [handle alerts](#alerting).
+
 VictoriaMetrics accepts [Prometheus querying API requests](#prometheus-querying-api-usage) on port `8428` by default.
 
 It is recommended setting up [monitoring](#monitoring) for VictoriaMetrics.
 
+
 ### Environment variables
 
 Each flag value can be set via environment variables according to these rules:
 
-* The `-envflag.enable` flag must be set
-* Each `.` char in flag name must be substituted by `_` (for example `-insert.maxQueueDuration <duration>` will translate to `insert_maxQueueDuration=<duration>`)
-* For repeating flags an alternative syntax can be used by joining the different values into one using `,` char as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`)
-* It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`
+* The `-envflag.enable` flag must be set.
+* Each `.` char in flag name must be substituted with `_` (for example `-insert.maxQueueDuration <duration>` will translate to `insert_maxQueueDuration=<duration>`).
+* For repeating flags an alternative syntax can be used by joining the different values into one using `,` char as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`).
+* Environment var prefix can be set via `-envflag.prefix` flag. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`.
+
 
 ### Configuration with snap package
 
 
- Command-line flags can be changed with following command:
+Snap package for VictoriaMetrics is available [here](https://snapcraft.io/victoriametrics).
+
+Command-line flags for Snap package can be set with following command:
 
 ```text
 echo 'FLAGS="-selfScrapeInterval=10s -search.logSlowQueryDuration=20s"' > $SNAP_DATA/var/snap/victoriametrics/current/extra_flags
 snap restart victoriametrics
 ```
-  Or add needed command-line flags to the file `$SNAP_DATA/var/snap/victoriametrics/current/extra_flags`.
 
- Note you cannot change value for `-storageDataPath` flag, for safety snap package has limited access to host system.
+Do not change value for `-storageDataPath` flag, because snap package has limited access to host filesystem.
 
 
- Changing scrape configuration is possible with text editor:
-    ```text
-    vi $SNAP_DATA/var/snap/victoriametrics/current/etc/victoriametrics-scrape-config.yaml
-    ```
- After changes was made, trigger config re-read with command `curl 127.0.0.1:8248/-/reload`.
+Changing scrape configuration is possible with text editor:
+
+```text
+vi $SNAP_DATA/var/snap/victoriametrics/current/etc/victoriametrics-scrape-config.yaml
+```
+
+After changes were made, trigger config re-read with the command `curl 127.0.0.1:8248/-/reload`.
 
 
 ## Prometheus setup
 
-Prometheus must be configured with [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
-in order to send data to VictoriaMetrics. Add the following lines
-to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`):
+Add the following lines to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`) in order to send data to VictoriaMetrics:
 
 ```yml
 remote_write:
@@ -252,7 +169,7 @@ Prometheus writes incoming data to local storage and replicates it to remote sto
 This means that data remains available in local storage for `--storage.tsdb.retention.time` duration
 even if remote storage is unavailable.
 
-If you plan to send data to VictoriaMetrics from multiple Prometheus instances, then add the following lines into `global` section
+If you plan sending data to VictoriaMetrics from multiple Prometheus instances, then add the following lines into `global` section
 of [Prometheus config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file):
 
 ```yml
@@ -261,11 +178,11 @@ global:
     datacenter: dc-123
 ```
 
-This instructs Prometheus to add `datacenter=dc-123` label to each time series sent to remote storage.
+This instructs Prometheus to add `datacenter=dc-123` label to each sample before sending it to remote storage.
 The label name can be arbitrary - `datacenter` is just an example. The label value must be unique
-across Prometheus instances, so those time series may be filtered and grouped by this label.
+across Prometheus instances, so time series could be filtered and grouped by this label.
 
-For highly loaded Prometheus instances (400k+ samples per second) the following tuning may be applied:
+For highly loaded Prometheus instances (200k+ samples per second) the following tuning may be applied:
 
 ```yaml
 remote_write:
@@ -276,14 +193,13 @@ remote_write:
       max_shards: 30
 ```
 
-Using remote write increases memory usage for Prometheus up to ~25% and depends on the shape of data. If you are experiencing issues with
-too high memory consumption try to lower `max_samples_per_send` and `capacity` params (keep in mind that these two params are tightly connected).
+Using remote write increases memory usage for Prometheus by up to ~25%. If you are experiencing issues with
+too high memory consumption of Prometheus, then try to lower `max_samples_per_send` and `capacity` params. Keep in mind that these two params are tightly connected.
 Read more about tuning remote write for Prometheus [here](https://prometheus.io/docs/practices/remote_write).
 
 It is recommended upgrading Prometheus to [v2.12.0](https://github.com/prometheus/prometheus/releases) or newer, since previous versions may have issues with `remote_write`.
 
-Take a look also at [vmagent](https://docs.victoriametrics.com/vmagent.html)
-and [vmalert](https://docs.victoriametrics.com/vmalert.html),
+Take a look also at [vmagent](https://docs.victoriametrics.com/vmagent.html) and [vmalert](https://docs.victoriametrics.com/vmalert.html),
 which can be used as faster and less resource-hungry alternative to Prometheus.
 
 
@@ -297,27 +213,22 @@ http://<victoriametrics-addr>:8428
 
 Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaMetrics.
 
-Then build graphs with the created datasource using [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/)
-or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). VictoriaMetrics supports [Prometheus querying API](#prometheus-querying-api-usage),
-which is used by Grafana.
+Then build graphs and dashboards for the created datasource using [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html).
 
 
 ## How to upgrade VictoriaMetrics
 
-It is safe upgrading VictoriaMetrics to new versions unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-say otherwise. It is safe skipping multiple versions during the upgrade unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) say otherwise.
-It is recommended performing regular upgrades to the latest version, since it may contain important bug fixes, performance optimizations or new features.
+It is safe upgrading VictoriaMetrics to new versions unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) say otherwise. It is safe skipping multiple versions during the upgrade unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) say otherwise. It is recommended performing regular upgrades to the latest version, since it may contain important bug fixes, performance optimizations or new features.
 
-It is also safe downgrading to the previous version unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) say otherwise.
+It is also safe downgrading to older versions unless [release notes](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) say otherwise.
 
-The following steps must be performed during the upgrade / downgrade:
+The following steps must be performed during the upgrade / downgrade procedure:
 
 * Send `SIGINT` signal to VictoriaMetrics process in order to gracefully stop it.
 * Wait until the process stops. This can take a few seconds.
 * Start the upgraded VictoriaMetrics.
 
-Prometheus doesn't drop data during VictoriaMetrics restart.
-See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
+Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/vmagent.html).
 
 
 ## How to apply new config to VictoriaMetrics
@@ -328,15 +239,12 @@ VictoriaMetrics is configured via command-line flags, so it must be restarted wh
 * Wait until the process stops. This can take a few seconds.
 * Start VictoriaMetrics with the new command-line flags.
 
-Prometheus doesn't drop data during VictoriaMetrics restart.
-See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details.
+Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details. The same applies alos to [vmagent](https://docs.victoriametrics.com/vmagent.html).
 
 
 ## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
 
-VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file).
-Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets.
-Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported:
+VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported:
 
 * [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config)
 * [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)
@@ -353,7 +261,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
 * [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config)
 
 
-Other `*_sd_config` types will be supported in the future.
+File a [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need support for other `*_sd_config` types.
 
 The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 
@@ -362,6 +270,52 @@ VictoriaMetrics also supports [importing data in Prometheus exposition format](#
 See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be used as drop-in replacement for Prometheus.
 
 
+## How to send data from DataDog agent
+
+VictoriaMetrics accepts data from [DataDog agent](https://docs.datadoghq.com/agent/) or [DogStatsD]() via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) at `/datadog/api/v1/series` path.
+
+Run DataDog agent with `DD_DD_URL=http://victoriametrics-host:8428/datadog` environment variable in order to write data to VictoriaMetrics at `victoriametrics-host` host. Another option is to set `dd_url` param at [DataDog agent configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) to `http://victoriametrics-host:8428/datadog`.
+
+Example on how to send data to VictoriaMetrics via DataDog "submit metrics" API from command line:
+
+```bash
+echo '
+{
+  "series": [
+    {
+      "host": "test.example.com",
+      "interval": 20,
+      "metric": "system.load.1",
+      "points": [[
+        0,
+        0.5
+      ]],
+      "tags": [
+        "environment:test"
+      ],
+      "type": "rate"
+    }
+  ]
+}
+' | curl -X POST --data-binary @- http://localhost:8428/datadog/api/v1/series
+```
+
+The imported data can be read via [export API](https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format):
+
+```bash
+curl http://localhost:8428/api/v1/export -d 'match[]=system.load.1'
+```
+
+This command should return the following output if everything is OK:
+
+```
+{"metric":{"__name__":"system.load.1","environment":"test","host":"test.example.com"},"values":[0.5],"timestamps":[1632833641000]}
+```
+
+Extra labels may be added to all the written time series by passing `extra_label=name=value` query args.
+For example, `/datadog/api/v1/series?extra_label=foo=bar` would add `{foo="bar"}` label to all the ingested metrics.
+
+
 ## How to send data from InfluxDB-compatible agents such as [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)
 
 Use `http://<victoriametric-addr>:8428` url instead of InfluxDB url in agents' configs.
@@ -372,21 +326,18 @@ For instance, put the following lines into `Telegraf` config, so it sends data t
   urls = ["http://<victoriametrics-addr>:8428"]
 ```
 
-Another option is to enable TCP and UDP receiver for Influx line protocol via `-influxListenAddr` command-line flag
-and stream plain Influx line protocol data to the configured TCP and/or UDP addresses.
+Another option is to enable TCP and UDP receiver for InfluxDB line protocol via `-influxListenAddr` command-line flag
+and stream plain InfluxDB line protocol data to the configured TCP and/or UDP addresses.
 
-VictoriaMetrics maps Influx data using the following rules:
+VictoriaMetrics performs the following transformations to the ingested InfluxDB data:
 
 * [`db` query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db` label value
-  unless `db` tag exists in the Influx line.
-* Field names are mapped to time series names prefixed with `{measurement}{separator}` value,
-  where `{separator}` equals to `_` by default. It can be changed with `-influxMeasurementFieldSeparator` command-line flag.
-  See also `-influxSkipSingleField` command-line flag.
-  If `{measurement}` is empty or `-influxSkipMeasurement` command-line flag is set, then time series names correspond to field names.
+  unless `db` tag exists in the InfluxDB line.
+* Field names are mapped to time series names prefixed with `{measurement}{separator}` value, where `{separator}` equals to `_` by default. It can be changed with `-influxMeasurementFieldSeparator` command-line flag. See also `-influxSkipSingleField` command-line flag. If `{measurement}` is empty or if `-influxSkipMeasurement` command-line flag is set, then time series names correspond to field names.
 * Field values are mapped to time series values.
 * Tags are mapped to Prometheus labels as-is.
 
-For example, the following Influx line:
+For example, the following InfluxDB line:
 
 ```raw
 foo,tag1=value1,tag2=value2 field1=12,field2=40
@@ -399,7 +350,7 @@ foo_field1{tag1="value1", tag2="value2"} 12
 foo_field2{tag1="value1", tag2="value2"} 40
 ```
 
-Example for writing data with [Influx line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
+Example for writing data with [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
 to local VictoriaMetrics using `curl`:
 
 ```bash
@@ -420,7 +371,7 @@ The `/api/v1/export` endpoint should return the following response:
 {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[1.23],"timestamps":[1560272508147]}
 ```
 
-Note that Influx line protocol expects [timestamps in *nanoseconds* by default](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/#timestamp),
+Note that InfluxDB line protocol expects [timestamps in *nanoseconds* by default](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/#timestamp),
 while VictoriaMetrics stores them with *milliseconds* precision.
 
 Extra labels may be added to all the written time series by passing `extra_label=name=value` query args.
@@ -579,7 +530,7 @@ By default, VictoriaMetrics returns time series for the last 5 minutes from `/ap
 
 Additionally VictoriaMetrics provides the following handlers:
 
-* `/vmui` - Basic Web UI
+* `/vmui` - Basic Web UI. See [these docs](#vmui).
 * `/api/v1/series/count` - returns the total number of time series in the database. Some notes:
   * the handler scans all the inverted index, so it can be slow if the database contains tens of millions of time series;
   * the handler may count [deleted time series](#how-to-delete-time-series) additionally to normal time series due to internal implementation restrictions;
@@ -618,7 +569,8 @@ and it is easier to use when migrating from Graphite to VictoriaMetrics.
 
 [VictoriaMetrics Enterprise](https://victoriametrics.com/enterprise.html) supports [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) subset
 at `/render` endpoint, which is used by [Graphite datasource in Grafana](https://grafana.com/docs/grafana/latest/datasources/graphite/).
-It supports `Storage-Step` http request header, which must be set to a step between data points stored in VictoriaMetrics when configuring Graphite datasource in Grafana.
+When configuring Graphite datasource in Grafana, the `Storage-Step` http request header must be set to a step between Graphite data points stored in VictoriaMetrics. For example, `Storage-Step: 10s` would mean 10 seconds distance between Graphite datapoints stored in VictoriaMetrics.
+Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 
 ### Graphite Metrics API usage
@@ -649,6 +601,21 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/
 * [/tags/delSeries](https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb)
 
 
+## vmui
+
+VictoriaMetrics provides UI for query troubleshooting and exploration. The UI is available at `http://victoriametrics:8428/vmui`.
+The UI allows exploring query results via graphs and tables. Graphs support scrolling and zooming:
+
+* Drag the graph to the left / right in order to move the displayed time range into the past / future.
+* Hold `Ctrl` (or `Cmd` on MacOS) and scroll up / down in order to zoom in / out the graph.
+
+Query history can be navigated by holding `Ctrl` (or `Cmd` on MacOS) and pressing `up` or `down` arrows on the keyboard while the cursor is located in the query input field.
+
+When querying the [backfilled data](https://docs.victoriametrics.com/#backfilling), it may be useful disabling response cache by clicking `Enable cache` checkbox.
+
+See the [example VMUI at VictoriaMetrics playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/?g0.expr=100%20*%20sum(rate(process_cpu_seconds_total))%20by%20(job)&g0.range_input=1d).
+
+
 ## How to build from sources
 
 We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or
@@ -758,7 +725,7 @@ Note that background merges may never occur for data from previous months, so st
 In this case [forced merge](#forced-merge) may help freeing up storage space.
 
 It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
-before actually deleting the metrics.  By default this query will only scan active series in the past 5 minutes, so you may need to
+before actually deleting the metrics.  By default this query will only scan series in the past 5 minutes, so you may need to
 adjust `start` and `end` to a suitable range to achieve match hits.
 
 The `/api/v1/admin/tsdb/delete_series` handler may be protected with `authKey` if `-deleteAuthKey` command-line flag is set.
@@ -889,7 +856,8 @@ The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv
 Time series data can be imported via any supported ingestion protocol:
 
 * [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). See [these docs](#prometheus-setup) for details.
-* Influx line protocol. See [these docs](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details.
+* DataDog `submit metrics` API. See [these docs](#how-to-send-data-from-datadog-agent) for details.
+* InfluxDB line protocol. See [these docs](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details.
 * Graphite plaintext protocol. See [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
 * OpenTSDB telnet put protocol. See [these docs](#sending-data-via-telnet-put-protocol) for details.
 * OpenTSDB http `/api/put` protocol. See [these docs](#sending-opentsdb-data-via-http-apiput-requests) for details.
@@ -1068,14 +1036,7 @@ Example contents for `-relabelConfig` file:
   regex: true
 ```
 
-VictoriaMetrics provides the following extra actions for relabeling rules:
-
-* `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
-* `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
-* `keep_if_equal`: keeps the entry if all label values from `source_labels` are equal.
-* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
-
-See also [relabeling in vmagent](https://docs.victoriametrics.com/vmagent.html#relabeling).
+See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details about relabeling in VictoriaMetrics.
 
 
 ## Federation
@@ -1094,7 +1055,7 @@ with scrape intervals exceeding `5m`.
 
 VictoriaMetrics uses lower amounts of CPU, RAM and storage space on production workloads compared to competing solutions (Prometheus, Thanos, Cortex, TimescaleDB, InfluxDB, QuestDB, M3DB) according to [our case studies](https://docs.victoriametrics.com/CaseStudies.html).
 
-VictoriaMetrics capacity scales linearly with the available resources. The needed amounts of CPU and RAM highly depends on the workload - the number of active time series, series churn rate, query types, query qps, etc. It is recommended setting up a test VictoriaMetrics for your production workload and iteratively scaling CPU and RAM resources until it becomes stable according to [troubleshooting docs](#troubleshooting). A single-node VictoriaMetrics works perfectly with the following production workload according to [our case studies](https://docs.victoriametrics.com/CaseStudies.html):
+VictoriaMetrics capacity scales linearly with the available resources. The needed amounts of CPU and RAM highly depends on the workload - the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series), series [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate), query types, query qps, etc. It is recommended setting up a test VictoriaMetrics for your production workload and iteratively scaling CPU and RAM resources until it becomes stable according to [troubleshooting docs](#troubleshooting). A single-node VictoriaMetrics works perfectly with the following production workload according to [our case studies](https://docs.victoriametrics.com/CaseStudies.html):
 
 * Ingestion rate: 1.5+ million samples per second
 * Active time series: 50+ million
@@ -1203,6 +1164,7 @@ There is no downsampling support at the moment, but:
   in [this article](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
 * VictoriaMetrics has good compression for on-disk data. See [this article](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
   for details.
+* The downsampling doesn't improve query performance on a long time range if the time range contains big number of time series due to [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). The query performance depends on the number of unique time series on the selected time range, while downsampling doesn't reduce the number of unique time series in the database - it can reduce only the number of samples per each time series.
 
 These properties reduce the need of downsampling. We plan to implement downsampling in the future.
 See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/36) for details.
@@ -1214,7 +1176,7 @@ only a single data point out of 20 initial data points per each 5m interval.
 
 ## Multi-tenancy
 
-Single-node VictoriaMetrics doesn't support multi-tenancy. Use [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) instead.
+Single-node VictoriaMetrics doesn't support multi-tenancy. Use [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) instead.
 
 
 ## Scalability and cluster version
@@ -1225,7 +1187,7 @@ such as Thanos, Uber M3, InfluxDB or TimescaleDB. See [vertical scalability benc
 
 So try single-node VictoriaMetrics at first and then [switch to cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster) if you still need
 horizontally scalable long-term remote storage for really large Prometheus deployments.
-[Contact us](mailto:info@victoriametrics.com) for paid support.
+[Contact us](mailto:info@victoriametrics.com) for enterprise support.
 
 
 ## Alerting
@@ -1264,8 +1226,11 @@ or similar auth proxy.
 * There is no need for VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
   which are automatically adjusted for the available CPU and RAM resources.
 * There is no need for Operating System tuning since VictoriaMetrics is optimized for default OS settings.
-  The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a),
-  so Prometheus instances could establish more connections to VictoriaMetrics.
+  The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a).
+  The recommendation is not specific for VictoriaMetrics only but also for any service which handles many HTTP connections and stores data on disk.
+* VictoriaMetrics is a write-heavy application and its performance depends on disk performance. So be careful with other
+  applications or utilities (like [fstrim](http://manpages.ubuntu.com/manpages/bionic/man8/fstrim.8.html))
+  which could [exhaust disk resources](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1521).
 * The recommended filesystem is `ext4`, the recommended persistent storage is [persistent HDD-based disk on GCP](https://cloud.google.com/compute/docs/disks/#pdspecs),
   since it is protected from hardware failures via internal replication and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).
   If you plan to store more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
@@ -1291,18 +1256,18 @@ It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.co
 The most interesting metrics are:
 
 * `vm_cache_entries{type="storage/hour_metric_ids"}` - the number of time series with new data points during the last hour
-  aka active time series.
-* `increase(vm_new_timeseries_created_total[1h])` - time series churn rate during the previous hour.
+  aka [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
+* `increase(vm_new_timeseries_created_total[1h])` - time series [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) during the previous hour.
 * `sum(vm_rows{type=~"storage/.*"})` - total number of `(timestamp, value)` data points in the database.
 * `sum(rate(vm_rows_inserted_total[5m]))` - ingestion rate, i.e. how many samples are inserted int the database per second.
 * `vm_free_disk_space_bytes` - free space left at `-storageDataPath`.
 * `sum(vm_data_size_bytes)` - the total size of data on disk.
 * `increase(vm_slow_row_inserts_total[5m])` - the number of slow inserts during the last 5 minutes.
   If this number remains high during extended periods of time, then it is likely more RAM is needed for optimal handling
-  of the current number of active time series.
+  of the current number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
 * `increase(vm_slow_metric_name_loads_total[5m])` - the number of slow loads of metric names during the last 5 minutes.
   If this number remains high during extended periods of time, then it is likely more RAM is needed for optimal handling
-  of the current number of active time series.
+  of the current number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
 
 VictoriaMetrics also exposes currently running queries with their execution times at `/api/v1/status/active_queries` page.
 
@@ -1322,8 +1287,8 @@ VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way simi
 
 By default VictoriaMetrics doesn't limit the number of stored time series. The limit can be enforced by setting the following command-line flags:
 
-* `-storage.maxHourlySeries` - limits the number of time series that can be added during the last hour. Useful for limiting the number of active time series.
-* `-storage.maxDailySeries` - limits the number of time series that can be added during the last day. Useful for limiting daily churn rate.
+* `-storage.maxHourlySeries` - limits the number of time series that can be added during the last hour. Useful for limiting the number of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series).
+* `-storage.maxDailySeries` - limits the number of time series that can be added during the last day. Useful for limiting daily [churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).
 
 Both limits can be set simultaneously. If any of these limits is reached, then incoming samples for new time series are dropped. A sample of dropped series is put in the log with `WARNING` level.
 
@@ -1334,6 +1299,8 @@ The exceeded limits can be [monitored](#monitoring) with the following metrics:
 
 These limits are approximate, so VictoriaMetrics can underflow/overflow the limit by a small percentage (usually less than 1%).
 
+See also more advanced [cardinality limiter in vmagent](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter).
+
 
 ## Troubleshooting
 
@@ -1364,7 +1331,7 @@ These limits are approximate, so VictoriaMetrics can underflow/overflow the limi
     See [this article for technical details](https://valyala.medium.com/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
 
 * If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
-  then it is likely you have too many active time series for the current amount of RAM.
+  then it is likely you have too many [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-active-time-series) for the current amount of RAM.
   VictoriaMetrics [exposes](#monitoring) `vm_slow_*` metrics such as `vm_slow_row_inserts_total` and `vm_slow_metric_name_loads_total`, which could be used
   as an indicator of low amounts of RAM. It is recommended increasing the amount of RAM on the node with VictoriaMetrics in order to improve
   ingestion and query performance in this case.
@@ -1391,7 +1358,7 @@ These limits are approximate, so VictoriaMetrics can underflow/overflow the limi
   It may be needed in order to suppress default gap filling algorithm used by VictoriaMetrics - by default it assumes
   each time series is continuous instead of discrete, so it fills gaps between real samples with regular intervals.
 
-* Metrics and labels leading to high cardinality or high churn rate can be determined at `/api/v1/status/tsdb` page. See [these docs](#tsdb-stats) for details.
+* Metrics and labels leading to [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) or [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) can be determined at `/api/v1/status/tsdb` page. See [these docs](#tsdb-stats) for details.
 
 * New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics.
 
@@ -1417,6 +1384,7 @@ Use [vmctl](https://docs.victoriametrics.com/vmctl.html) for data migration. It
 * From Prometheus to VictoriaMetrics
 * From InfluxDB to VictoriaMetrics
 * From VictoriaMetrics to VictoriaMetrics
+* From OpenTSDB to VictoriaMetrics
 
 See [vmctl docs](https://docs.victoriametrics.com/vmctl.html) for more details.
 
@@ -1460,7 +1428,8 @@ See also [high availability docs](#high-availability) and [backup docs](#backups
 
 VictoriaMetrics supports backups via [vmbackup](https://docs.victoriametrics.com/vmbackup.html)
 and [vmrestore](https://docs.victoriametrics.com/vmrestore.html) tools.
-We also provide `vmbackupmanager` tool for paid enterprise subscribers - see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) for details.
+We also provide [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html) tool for enterprise subscribers.
+Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 
 ## Profiling
@@ -1582,6 +1551,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	The maximum number of CPU cores to use for big merges. Default value is used if set to 0
   -csvTrimTimestamp duration
     	Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+  -datadog.maxInsertRequestSize size
+    	The maximum size in bytes of a single DataDog POST request to /api/v1/series
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
   -dedup.minScrapeInterval duration
     	Leave only the first sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details
   -deleteAuthKey string
@@ -1593,7 +1565,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -enableTCP6
     	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -finalMergeDelay duration
@@ -1633,18 +1605,18 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
     	Supports an array of values separated by comma or specified via multiple flags.
   -influx.maxLineSize size
-    	The maximum size in bytes for a single Influx line during parsing
+    	The maximum size in bytes for a single InfluxDB line during parsing
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
   -influxListenAddr string
-    	TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write
+    	TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write
   -influxMeasurementFieldSeparator string
-    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol (default "_")
+    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
   -influxSkipMeasurement
     	Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'
   -influxSkipSingleField
-    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field
+    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field
   -influxTrimTimestamp duration
-    	Trim timestamps for Influx line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+    	Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
     	The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
   -logNewSeries
@@ -1747,8 +1719,15 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.maxScrapeSize size
     	The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
+  -promscrape.minResponseSizeForStreamParse size
+    	The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 1000000)
+  -promscrape.noStaleMarkers
+    	Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
     	Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details (default 30s)
+  -promscrape.seriesLimitPerTarget int
+    	Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
     	Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
@@ -1756,14 +1735,16 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.suppressScrapeErrors
     	Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
   -relabelConfig string
-    	Optional path to a file with relabeling rules, which are applied to all the ingested metrics. See https://docs.victoriametrics.com/#relabeling for details
+    	Optional path to a file with relabeling rules, which are applied to all the ingested metrics. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
   -relabelDebug
     	Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs
   -retentionPeriod value
     	Data with timestamps outside the retentionPeriod is automatically deleted
     	The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1)
   -search.cacheTimestampOffset duration
-    	The maximum duration since the current time for response data, which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses due to time synchronization issues between VictoriaMetrics and data sources (default 5m0s)
+    	The maximum duration since the current time for response data, which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset (default 5m0s)
+  -search.disableAutoCacheReset
+    	Whether to disable automatic response cache reset if a sample with timestamp outside -search.cacheTimestampOffset is inserted into VictoriaMetrics
   -search.disableCache
     	Whether to disable response caching. This may be useful during data backfilling
   -search.latencyOffset duration
@@ -1785,6 +1766,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16384)
   -search.maxQueueDuration duration
     	The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached; see also -search.maxQueryDuration (default 10s)
+  -search.maxSamplesPerQuery int
+    	The maximum number of raw samples a single query can process across all time series. This protects from heavy queries, which select unexpectedly high number of raw samples. See also -search.maxSamplesPerSeries (default 1000000000)
+  -search.maxSamplesPerSeries int
+    	The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage (default 30000000)
   -search.maxStalenessInterval duration
     	The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons
   -search.maxStatusRequestDuration duration
@@ -1798,13 +1783,15 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxTagValues int
     	The maximum number of tag values returned from /api/v1/label/<label_name>/values (default 100000)
   -search.maxUniqueTimeseries int
-    	The maximum number of unique time series each search can scan (default 300000)
+    	The maximum number of unique time series each search can scan. This option allows limiting memory usage (default 300000)
   -search.minStalenessInterval duration
     	The minimum interval for staleness calculations. This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. See also '-search.maxStalenessInterval'
+  -search.noStaleMarkers
+    	Set this flag to true if the database doesn't contain Prometheus stale markers, so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets
   -search.queryStats.lastQueriesCount int
     	Query stats for /api/v1/status/top_queries is tracked on this number of last queries. Zero value disables query stats tracking (default 20000)
   -search.queryStats.minQueryDuration duration
-    	The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats
+    	The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats (default 1ms)
   -search.resetCacheAuthKey string
     	Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call
   -search.treatDotsAsIsInRegexps
@@ -1825,6 +1812,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See also -storage.maxHourlySeries
   -storage.maxHourlySeries int
     	The maximum number of unique series can be added to the storage during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See also -storage.maxDailySeries
+  -storage.minFreeDiskSpaceBytes size
+    	The minimum free disk space at -storageDataPath after which the storage stops accepting new data
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 10000000)
   -storageDataPath string
     	Path to storage data (default "victoria-metrics-data")
   -tls

app/victoria-metrics/main.go

@@ -93,7 +93,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"/vmui", "Web UI"},
 			{"/targets", "discovered targets list"},
 			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"/config", "-promscrape.config contents"},
 			{"/metrics", "available service metrics"},
+			{"/flags", "command-line flags"},
 			{"/api/v1/status/tsdb", "tsdb status page"},
 			{"/api/v1/status/top_queries", "top queries"},
 			{"/api/v1/status/active_queries", "active queries"},

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -11,6 +11,6 @@
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[
-	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"]]}
+	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"], ["{TIME_S}","0.5"]]}
       ]}}
 }

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -19,6 +19,7 @@
 	      ["{TIME_S-110s}","3"],
 	      ["{TIME_S-100s}","3"],
 	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-80s}","3"],
 	      ["{TIME_S-60s}","2"],
 	      ["{TIME_S-50s}","2"],
 	      ["{TIME_S-40s}","2"],

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -13,6 +13,6 @@
     "data":{"resultType":"matrix",
       "result":[
 	      {"metric":{"__name__":"not_nan_as_missing_data","item":"x"},"values":[["{TIME_S-2m}","2"]]},
-	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"]]}
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"],["{TIME_S}", "3"]]}
       ]}}
 }

app/vmagent/README.md

@@ -17,17 +17,19 @@ to `vmagent` such as the ability to push metrics instead of pulling them. We did
 
 ## Features
 
-* Can be used as a drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
-  See [Quick Start](#quick-start) for details.
+* Can be used as a drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter). See [Quick Start](#quick-start) for details.
+* Can read data from Kafka. See [these docs](#reading-metrics-from-kafka).
+* Can write data to Kafka. See [these docs](#writing-metrics-to-kafka).
 * Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
 * Accepts data via all ingestion protocols supported by VictoriaMetrics:
-  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * DataDog "submit metrics" API. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent).
+  * InfluxDB line protocol via `http://<vmagent>:8429/write`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
   * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
   * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
   * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format).
   * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-native-format).
-  * Data in Prometheus exposition format. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details.
+  * Prometheus exposition format via `http://<vmagent>:8429/api/v1/import/prometheus`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details.
   * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
 * Works smoothly in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
@@ -36,8 +38,8 @@ to `vmagent` such as the ability to push metrics instead of pulling them. We did
 * Uses lower amounts of RAM, CPU, disk IO and network bandwidth compared with Prometheus.
 * Scrape targets can be spread among multiple `vmagent` instances when big number of targets must be scraped. See [these docs](#scraping-big-number-of-targets).
 * Can efficiently scrape targets that expose millions of time series such as [/federate endpoint in Prometheus](https://prometheus.io/docs/prometheus/latest/federation/). See [these docs](#stream-parsing-mode).
-* Can deal with high cardinality and high churn rate issues by limiting the number of unique time series sent to remote storage systems. See [these docs](#cardinality-limiter).
-
+* Can deal with [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues by limiting the number of unique time series at scrape time and before sending them to remote storage systems. See [these docs](#cardinality-limiter).
+* Can load scrape configs from multiple files. See [these docs](#loading-scrape-configs-from-multiple-files).
 
 ## Quick Start
 
@@ -53,13 +55,13 @@ Example command line:
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-If you only need to collect Influx data, then the following command is sufficient:
+If you only need to collect InfluxDB data, then the following command is sufficient:
 
 ```
 /path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
-Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+Then send InfluxDB data to `http://vmagent-host:8429`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
 
 `vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags).
 
@@ -135,7 +137,11 @@ Also, Basic Auth can be enabled for the incoming `remote_write` requests with `-
 
 ### remote_write for clustered version
 
-While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx, Prometheus, Graphite) and scrape data from various targets, writes are always peformed in Promethes remote_write protocol. Therefore for the [clustered version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html), `-remoteWrite.url` the command-line flag should be configured as `<schema>://<vminsert-host>:8480/insert/<customer-id>/prometheus/api/v1/write`
+While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx, Prometheus, Graphite) and scrape data from various targets, writes are always peformed in Promethes remote_write protocol. Therefore for the [clustered version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html), `-remoteWrite.url` the command-line flag should be configured as `<schema>://<vminsert-host>:8480/insert/<accountID>/prometheus/api/v1/write` according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). There is also support for multitenant writes. See [these docs](#multitenancy).
+
+## Multitenancy
+
+By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`. But it can accept multitenant data if `-remoteWrite.multitenantURL` is set. In this case it accepts multitenant data at `http://vmagent:8429/insert/<accountID>/...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert/<accountID>/prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants.
 
 
 ## How to collect metrics in Prometheus format
@@ -192,7 +198,12 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM
   to save network bandwidth.
 * `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis.
   By default, `vmagent` uses keep-alive connections to scrape targets to reduce overhead on connection re-establishing.
+* `series_limit: N` - for limiting the number of unique time series a single scrape target can expose. See [these docs](#cardinality-limiter).
 * `stream_parse: true` - for scraping targets in a streaming manner. This may be useful for targets exporting big number of metrics. See [these docs](#stream-parsing-mode).
+* `scrape_align_interval: duration` - for aligning scrapes to the given interval instead of using random offset in the range `[0 ... scrape_interval]` for scraping each target. The random offset helps spreading scrapes evenly in time.
+* `scrape_offset: duration` - for specifying the exact offset for scraping instead of using random offset in the range `[0 ... scrape_interval]`.
+* `relabel_debug: true` - for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling).
+* `metric_relabel_debug: true` - for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling).
 
 Note that `vmagent` doesn't support `refresh_interval` option for these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
 command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
@@ -201,6 +212,30 @@ entries to 60s. Run `vmagent -help` in order to see default values for the `-pro
 The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values.
 
 
+## Loading scrape configs from multiple files
+
+`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory plus a `single_scrape_config.yml` file:
+
+```yml
+scrape_config_files:
+- configs/*.yml
+- single_scrape_config.yml
+```
+
+Every referred file can contain arbitrary number of any [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example:
+
+```yml
+- job_name: foo
+  static_configs:
+  - targets: ["vmagent:8429"]
+- job_name: bar
+  kubernetes_sd_configs:
+  - role: pod
+```
+
+`vmagent` dynamically reloads these files on `SIGHUP` signal or on the request to `http://vmagent:8429/-/reload`.
+
+
 ## Adding labels to metrics
 
 Labels can be added to metrics by the following mechanisms:
@@ -215,13 +250,30 @@ Labels can be added to metrics by the following mechanisms:
 
 ## Relabeling
 
-`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
-and also provides the following actions:
+`vmagent` and VictoriaMetrics support Prometheus-compatible relabeling.
+They provide the following additional actions on top of actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config):
 
 * `replace_all`: replaces all of the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`.
 * `labelmap_all`: replaces all of the occurences of `regex` in all the label names with the `replacement`.
 * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal.
 * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
+* `keep_metrics`: keeps all the metrics with names matching the given `regex`.
+* `drop_metrics`: drops all the metrics with names matching the given `regex`.
+
+The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent:
+
+```yaml
+- action: keep_metrics
+  regex: "metric_a|metric_b|foo_.+"
+```
+
+```yaml
+- action: keep_metrics
+  regex:
+  - "metric_a"
+  - "metric_b"
+  - "foo_.+"
+```
 
 The relabeling can be defined in the following places:
 
@@ -240,25 +292,46 @@ You can read more about relabeling in the following articles:
 * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
 
 
+## Prometheus staleness markers
+
+`vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) to `-remoteWrite.url` in the following cases:
+
+* If they are passed to `vmagent` via [Prometheus remote_write protocol](#prometheus-remote_write-proxy).
+* If the metric disappears from the list of scraped metrics, then stale marker is sent to this particular metric.
+* If the scrape target becomes temporarily unavailable, then stale markers are sent for all the metrics scraped from this target.
+* If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target.
+* Stale markers are sent for all the scraped metrics on graceful shutdown of `vmagent`.
+
+Prometheus staleness markers' tracking needs additional memory, since it must store the previous response body per each scrape target in order to compare it to the current response body. The memory usage may be reduced by passing `-promscrape.noStaleMarkers` command-line flag to `vmagent`. This disables staleness tracking. This also disables tracking the number of new time series per each scrape with the auto-generated `scrape_series_added` metric. See [these docs](https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series) for details.
+
+
 ## Stream parsing mode
 
-By default `vmagent` reads the full response from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics. Stream parsing mode may be enabled either globally for all of the scrape targets by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
+By default `vmagent` reads the full response body from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics.
 
-  ```yml
-  scrape_configs:
-  - job_name: 'big-federate'
-    stream_parse: true
-    static_configs:
-    - targets:
-      - big-prometeus1
-      - big-prometeus2
-    honor_labels: true
-    metrics_path: /federate
-    params:
-      'match[]': ['{__name__!=""}']
-  ```
+Stream parsing mode is automatically enabled for scrape targets returning response bodies with sizes bigger than the `-promscrape.minResponseSizeForStreamParse` command-line flag value. Additionally, the stream parsing mode can be explicitly enabled in the following places:
 
-Note that `sample_limit` option doesn't prevent from data push to remote storage if stream parsing is enabled because the parsed data is pushed to remote storage as soon as it is parsed.
+- Via `-promscrape.streamParse` command-line flag. In this case all the scrape targets defined in the file pointed by `-promscrape.config` are scraped in stream parsing mode.
+- Via `stream_parse: true` option at `scrape_configs` section. In this case all the scrape targets defined in this section are scraped in stream parsing mode.
+- Via `__stream_parse__=true` label, which can be set via [relabeling](#relabeling) at `relabel_configs` section. In this case stream parsing mode is enabled for the corresponding scrape targets. Typical use case: to set the label via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets exposing big number of metrics.
+
+Examples:
+
+```yml
+scrape_configs:
+- job_name: 'big-federate'
+  stream_parse: true
+  static_configs:
+  - targets:
+    - big-prometeus1
+    - big-prometeus2
+  honor_labels: true
+  metrics_path: /federate
+  params:
+    'match[]': ['{__name__!=""}']
+```
+
+Note that `sample_limit` and `series_limit` options cannot be used in stream parsing mode because the parsed data is pushed to remote storage as soon as it is parsed.
 
 
 ## Scraping big number of targets
@@ -326,6 +399,16 @@ scrape_configs:
 
 ## Cardinality limiter
 
+By default `vmagent` doesn't limit the number of time series each scrape target can expose. The limit can be enforced in the following places:
+
+- Via `-promscrape.seriesLimitPerTarget` command-line option. This limit is applied individually to all the scrape targets defined in the file pointed by `-promscrape.config`.
+- Via `series_limit` config option at `scrape_config` section. This limit is applied individually to all the scrape targets defined in the given `scrape_config`.
+- Via `__series_limit__` label, which can be set with [relabeling](#relabeling) at `relabel_configs` section. This limit is applied to the corresponding scrape targets. Typical use case: to set the limit via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets, which may expose too high number of time series.
+
+All the scraped metrics are dropped for time series exceeding the given limit. The exceeded limit can be [monitored](#monitoring) via `promscrape_series_limit_rows_dropped_total` metric.
+
+See also `sample_limit` option at [scrape_config section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).
+
 By default `vmagent` doesn't limit the number of time series written to remote storage systems specified at `-remoteWrite.url`. The limit can be enforced by setting the following command-line flags:
 
 * `-remoteWrite.maxHourlySeries` - limits the number of unique time series `vmagent` can write to remote storage systems during the last hour. Useful for limiting the number of active time series.
@@ -368,7 +451,8 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
   as `vmagent` establishes at least a single TCP connection per target.
 
 * If `vmagent` uses too big amounts of memory, then the following options can help:
-  * Enabling stream parsing. See [these docs](#stream-parsing-mode).
+  * Disabling staleness tracking with `-promscrape.noStaleMarkers` option. See [these docs](#prometheus-staleness-markers).
+  * Enabling stream parsing mode. See [these docs](#stream-parsing-mode).
   * Reducing the number of output queues with `-remoteWrite.queues` command-line option.
   * Reducing the amounts of RAM vmagent can use for in-memory buffering with `-memory.allowedPercent` or `-memory.allowedBytes` command-line option. Another option is to reduce memory limits in Docker and/or Kuberntes if `vmagent` runs under these systems.
   * Reducing the number of CPU cores vmagent can use by passing `GOMAXPROCS=N` environment variable to `vmagent`, where `N` is the desired limit on CPU cores. Another option is to reduce CPU limits in Docker or Kubernetes if `vmagent` runs under these systems.
@@ -394,7 +478,7 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
 
 * `vmagent` drops data blocks if remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses. The number of dropped blocks can be monitored via `vmagent_remotewrite_packets_dropped_total` metric exported at [/metrics page](#monitoring).
 
-* Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling). Such storage systems include Prometheus, Cortex and Thanos.
+* Use `-remoteWrite.queues=1` when `-remoteWrite.url` points to remote storage, which doesn't accept out-of-order samples (aka data backfilling). Such storage systems include Prometheus, Cortex and Thanos, which typically emit `out of order sample` errors. The best solution is to use remote storage with [backfilling support](https://docs.victoriametrics.com/#backfilling).
 
 * `vmagent` buffers scraped data at the `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
@@ -441,6 +525,108 @@ It may be useful to perform `vmagent` rolling update without any scrape loss.
     regex: true
   ```
 
+## Kafka integration
+
+[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read and write metrics from / to Kafka:
+
+* [Reading metrics from Kafka](#reading-metrics-from-kafka)
+* [Writing metrics to Kafka](#writing-metrics-to-kafka)
+
+The enterprise version of vmagent is available for evaluation at [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page in `vmutils-*-enteprise.tar.gz` archives and in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+
+
+### Reading metrics from Kafka
+
+[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` can read metrics in various formats from Kafka messages. These formats can be configured with `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` command-line options. The following formats are supported:
+
+* `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). Messages in this format can be sent by vmagent - see [these docs](#writing-metrics-to-kafka).
+* `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/).
+* `prometheus` - [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format) and [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md).
+* `graphite` - [Graphite plaintext format](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol).
+* `jsonline` - [JSON line format](https://docs.victoriametrics.com/#how-to-import-data-in-json-line-format).
+
+Every Kafka message may contain multiple lines in `influx`, `prometheus`, `graphite` and `jsonline` format delimited by `\n`.
+
+`vmagent` consumes messages from Kafka topics specified by `-kafka.consumer.topic` command-line flag. Multiple topics can be specified by passing multiple `-kafka.consumer.topic` command-line flags to `vmagent`.
+
+`vmagent` consumes messages from Kafka brokers specified by `-kafka.consumer.topic.brokers` command-line flag. Multiple brokers can be specified per each `-kafka.consumer.topic` by passing a list of brokers delimited by `;`. For example, `-kafka.consumer.topic.brokers=host1:9092;host2:9092`.
+
+The following command starts `vmagent`, which reads metrics in InfluxDB line protocol format from Kafka broker at `localhost:9092` from the topic `metrics-by-telegraf` and sends them to remote storage at `http://localhost:8428/api/v1/write`:
+
+```bash
+./bin/vmagent -remoteWrite.url=http://localhost:8428/api/v1/write \
+       -kafka.consumer.topic.brokers=localhost:9092 \
+       -kafka.consumer.topic.format=influx \
+       -kafka.consumer.topic=metrics-by-telegraf \
+       -kafka.consumer.topic.groupID=some-id
+```
+
+It is expected that [Telegraf](https://github.com/influxdata/telegraf) sends metrics to the `metrics-by-telegraf` topic with the following config:
+
+```yaml
+[[outputs.kafka]]
+brokers = ["localhost:9092"]
+topic = "influx"
+data_format = "influx"
+```
+
+
+#### Command-line flags for Kafka consumer
+
+These command-line flags are available only in [enterprise](https://victoriametrics.com/enterprise.html) version of `vmagent`, which can be downloaded for evaluation from [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page (see `vmutils-*-enteprise.tar.gz` archives) and from [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags) with tags containing `enterprise` suffix.
+
+```
+  -kafka.consumer.topic array
+        Kafka topic names for data consumption.
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.password array
+        Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.basicAuth.username array
+        Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN'
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.brokers array
+        List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.defaultFormat string
+        Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite")
+  -kafka.consumer.topic.format array
+        data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.groupID array
+        Defines group.id for topic
+        Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.isGzipped array
+        Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages.
+        Supports array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.options array
+        Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+        Supports an array of values separated by comma or specified via multiple flags.
+```
+
+### Writing metrics to Kafka
+
+[Enterprise version](https://victoriametrics.com/enterprise.html) of `vmagent` writes data to Kafka with `at-least-once` semantics if `-remoteWrite.url` contains e.g. Kafka url. For example, if `vmagent` is started with `-remoteWrite.url=kafka://localhost:9092/?topic=prom-rw`, then it would send Prometheus remote_write messages to Kafka bootstrap server at `localhost:9092` with the topic `prom-rw`. These messages can be read later from Kafka by another `vmagent` - see [these docs](#reading-metrics-from-kafka) for details.
+
+Additional Kafka options can be passed as query params to `-remoteWrite.url`. For instance, `kafka://localhost:9092/?topic=prom-rw&client.id=my-favorite-id` sets `client.id` Kafka option to `my-favorite-id`. The full list of Kafka options is available [here](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
+
+
+#### Kafka broker authorization and authentication
+
+Two types of auth are supported:
+
+* sasl with username and password:
+
+```bash
+./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SASL_SSL&sasl.mechanisms=PLAIN -remoteWrite.basicAuth.username=user -remoteWrite.basicAuth.password=password
+```
+
+* tls certificates:
+
+```bash
+./bin/vmagent -remoteWrite.url=kafka://localhost:9092/?topic=prom-rw&security.protocol=SSL -remoteWrite.tlsCAFile=/opt/ca.pem -remoteWrite.tlsCertFile=/opt/cert.pem -remoteWrite.tlsKeyFile=/opt/key.pem
+```
+
 
 ## How to build from sources
 
@@ -523,12 +709,15 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
 
   -csvTrimTimestamp duration
     	Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+  -datadog.maxInsertRequestSize size
+    	The maximum size in bytes of a single DataDog POST request to /api/v1/series
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864)
   -dryRun
     	Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse
   -enableTCP6
     	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
@@ -562,18 +751,18 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
     	Supports an array of values separated by comma or specified via multiple flags.
   -influx.maxLineSize size
-    	The maximum size in bytes for a single Influx line during parsing
+    	The maximum size in bytes for a single InfluxDB line during parsing
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 262144)
   -influxListenAddr string
-    	TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
+    	TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
   -influxMeasurementFieldSeparator string
-    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol (default "_")
+    	Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
   -influxSkipMeasurement
     	Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'
   -influxSkipSingleField
-    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field
+    	Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field
   -influxTrimTimestamp duration
-    	Trim timestamps for Influx line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
+    	Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
     	The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
   -loggerDisableTimestamps
@@ -634,7 +823,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.consulSDCheckInterval duration
     	Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s)
   -promscrape.digitaloceanSDCheckInterval duration
-        Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config for details (default 1m0s)    	
+    	Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config for details (default 1m0s)
   -promscrape.disableCompression
     	Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.disableKeepAlive
@@ -645,6 +834,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
   -promscrape.dnsSDCheckInterval duration
     	Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config for details (default 30s)
+  -promscrape.dockerSDCheckInterval duration
+    	Interval for checking for changes in docker. This works only if docker_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config for details (default 30s)
   -promscrape.dockerswarmSDCheckInterval duration
     	Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details (default 30s)
   -promscrape.dropOriginalLabels
@@ -658,7 +849,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.httpSDCheckInterval duration
-        Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)    	   	
+    	Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)
   -promscrape.kubernetes.apiServerTimeout duration
     	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration
@@ -668,8 +859,15 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.maxScrapeSize size
     	The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216)
+  -promscrape.minResponseSizeForStreamParse size
+    	The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 1000000)
+  -promscrape.noStaleMarkers
+    	Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
     	Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details (default 30s)
+  -promscrape.seriesLimitPerTarget int
+    	Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info
   -promscrape.streamParse
     	Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is posible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control
   -promscrape.suppressDuplicateScrapeTargetErrors
@@ -700,12 +898,15 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	The maximum size in bytes of unpacked request to send to remote storage. It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 8388608)
   -remoteWrite.maxDailySeries int
-    	The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See also -remoteWrite.maxHourlySeries
+    	The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
   -remoteWrite.maxDiskUsagePerURL size
     	The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. Disk usage is unlimited if the value is set to 0
     	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -remoteWrite.maxHourlySeries int
-    	The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See also -remoteWrite.maxDailySeries
+    	The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter
+  -remoteWrite.multitenantURL array
+    	Base path for multitenant remote storage URL to write data to. See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.oauth2.clientID array
     	Optional OAuth2 clientID to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
     	Supports an array of values separated by comma or specified via multiple flags.
@@ -725,7 +926,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Optional proxy URL for writing data to -remoteWrite.url. Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.queues int
-    	The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage (default 2 * numberOfAvailableCPUs)
+    	The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs (default 8)
   -remoteWrite.rateLimit array
     	Optional rate limit in bytes per second for data sent to -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
     	Supports array of values separated by comma or specified via multiple flags.
@@ -762,7 +963,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.tmpDataPath string
     	Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data")
   -remoteWrite.url array
-    	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems
+    	Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
     	Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
     	Optional path to relabel config for the corresponding -remoteWrite.url

app/vmagent/csvimport/request_handler.go

@@ -5,32 +5,35 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="csvimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
 )
 
 // InsertHandler processes csv data from req.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(at, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -64,8 +67,11 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(len(rows))
+	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
 }

app/vmagent/datadog/request_handler.go

@@ -0,0 +1,99 @@
+package datadog
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		ce := req.Header.Get("Content-Encoding")
+		return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+			return insertRows(at, series, extraLabels)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "host",
+			Value: ss.Host,
+		})
+		for _, tag := range ss.Tags {
+			n := strings.IndexByte(tag, ':')
+			if n < 0 {
+				return fmt.Errorf("cannot find ':' in tag %q", tag)
+			}
+			name := tag[:n]
+			value := tag[n+1:]
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp(),
+				Value:     pt.Value(),
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -8,34 +8,37 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="influx"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
 )
 
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(r io.Reader) error {
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, nil)
+		return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+			return insertRows(nil, db, rows, nil)
 		})
 	})
 }
@@ -43,7 +46,7 @@ func InsertHandlerForReader(r io.Reader) error {
 // InsertHandlerForHTTP processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
-func InsertHandlerForHTTP(req *http.Request) error {
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
@@ -55,12 +58,12 @@ func InsertHandlerForHTTP(req *http.Request) error {
 		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 		db := q.Get("db")
 		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, extraLabels)
+			return insertRows(at, db, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 
@@ -130,8 +133,11 @@ func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label)
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	remotewrite.Push(&ctx.ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 
 	return nil

app/vmagent/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
@@ -19,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -40,7 +43,7 @@ var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
@@ -92,7 +95,9 @@ func main() {
 	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			return influx.InsertHandlerForReader(r, false)
+		})
 	}
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
@@ -154,16 +159,19 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"/targets", "discovered targets list"},
 			{"/api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"/config", "-promscrape.config contents"},
 			{"/metrics", "available service metrics"},
+			{"/flags", "command-line flags"},
 			{"/-/reload", "reload configuration"},
 		})
 		return true
 	}
+
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := promremotewrite.InsertHandler(r); err != nil {
+		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -172,7 +180,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import":
 		vmimportRequests.Inc()
-		if err := vmimport.InsertHandler(r); err != nil {
+		if err := vmimport.InsertHandler(nil, r); err != nil {
 			vmimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -181,7 +189,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import/csv":
 		csvimportRequests.Inc()
-		if err := csvimport.InsertHandler(r); err != nil {
+		if err := csvimport.InsertHandler(nil, r); err != nil {
 			csvimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -190,7 +198,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import/prometheus":
 		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(r); err != nil {
+		if err := prometheusimport.InsertHandler(nil, r); err != nil {
 			prometheusimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -199,7 +207,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/import/native":
 		nativeimportRequests.Inc()
-		if err := native.InsertHandler(r); err != nil {
+		if err := native.InsertHandler(nil, r); err != nil {
 			nativeimportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -208,7 +216,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
-		if err := influx.InsertHandlerForHTTP(r); err != nil {
+		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
 			influxWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -219,10 +227,45 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "/datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
+	case "/config":
+		promscrapeConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		promscrape.WriteConfigData(w)
+		return true
 	case "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
@@ -245,9 +288,121 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	}
+	if remotewrite.MultitenancyEnabled() {
+		return processMultitenantRequest(w, r, path)
+	}
 	return false
 }
 
+func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		// Cannot parse multitenant path. Skip it - probably it will be parsed later.
+		return false
+	}
+	if p.Prefix != "insert" {
+		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
+		return true
+	}
+	at, err := auth.NewToken(p.AuthToken)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
+		return true
+	}
+	switch p.Suffix {
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(at, r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(at, r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(at, r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(at, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(at, r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/write", "influx/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(at, r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/query":
+		influxQueryRequests.Inc()
+		influxutils.WriteDatabaseNames(w)
+		return true
+	case "datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, `{}`)
+		return true
+	default:
+		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
+		return true
+	}
+}
+
 var (
 	prometheusWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
 	prometheusWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/write", protocol="promremotewrite"}`)
@@ -264,14 +419,23 @@ var (
 	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 
-	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
-	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
+	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
 
 	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
+	promscrapeConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
 

app/vmagent/native/request_handler.go

@@ -5,36 +5,39 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="native"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
 		return parser.ParseStream(req, func(block *parser.Block) error {
-			return insertRows(block, extraLabels)
+			return insertRows(at, block, extraLabels)
 		})
 	})
 }
 
-func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -42,6 +45,9 @@ func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	// since relabeling can prevent from inserting the rows.
 	rowsLen := len(block.Values)
 	rowsInserted.Add(rowsLen)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsLen)
+	}
 	rowsPerInsert.Update(float64(rowsLen))
 
 	tssDst := ctx.WriteRequest.Timeseries[:0]
@@ -80,6 +86,6 @@ func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/prometheusimport/request_handler.go

@@ -1,24 +1,28 @@
 package prometheusimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
@@ -30,12 +34,21 @@ func InsertHandler(req *http.Request) error {
 	return writeconcurrencylimiter.Do(func() error {
 		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+			return insertRows(at, rows, extraLabels)
 		}, nil)
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+// InsertHandlerForReader processes metrics from given reader with optional gzip format
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		}, nil)
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -69,8 +82,11 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(len(rows))
+	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
 }

app/vmagent/promremotewrite/request_handler.go

@@ -1,38 +1,51 @@
 package promremotewrite
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(tss []prompb.TimeSeries) error {
-			return insertRows(tss, extraLabels)
+		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, extraLabels)
 		})
 	})
 }
 
-func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, nil)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -68,8 +81,11 @@ func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Labe
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmagent/remotewrite/client.go

@@ -67,7 +67,8 @@ type client struct {
 	fq             *persistentqueue.FastQueue
 	hc             *http.Client
 
-	authCfg *promauth.Config
+	sendBlock func(block []byte) bool
+	authCfg   *promauth.Config
 
 	rl rateLimiter
 
@@ -78,12 +79,13 @@ type client struct {
 	errorsCount     *metrics.Counter
 	packetsDropped  *metrics.Counter
 	retriesCount    *metrics.Counter
+	sendDuration    *metrics.FloatCounter
 
 	wg     sync.WaitGroup
 	stopCh chan struct{}
 }
 
-func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
+func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
 		logger.Panicf("FATAL: cannot initialize auth config: %s", err)
@@ -120,6 +122,11 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		},
 		stopCh: make(chan struct{}),
 	}
+	c.sendBlock = c.sendBlockHTTP
+	return c
+}
+
+func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
@@ -133,6 +140,7 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
 	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
+	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -141,7 +149,6 @@ func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqu
 		}()
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
-	return c
 }
 
 func (c *client) MustStop() {
@@ -204,7 +211,9 @@ func (c *client) runWorker() {
 			return
 		}
 		go func() {
+			startTime := time.Now()
 			ch <- c.sendBlock(block)
+			c.sendDuration.Add(time.Since(startTime).Seconds())
 		}()
 		select {
 		case ok := <-ch:
@@ -233,9 +242,9 @@ func (c *client) runWorker() {
 	}
 }
 
-// sendBlock returns false only if c.stopCh is closed.
+// sendBlockHTTP returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlock(block []byte) bool {
+func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
 	retryDuration := time.Second
 	retriesCount := 0

app/vmagent/remotewrite/relabel.go

@@ -42,11 +42,11 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		}
 		rcs.global = global
 	}
-	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
-		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*relabelConfigPaths), len(*remoteWriteURLs))
+	if len(*relabelConfigPaths) > (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)) {
+		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url or -remoteWrite.multitenantURL args: %d",
+			len(*relabelConfigPaths), (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	}
-	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
 			// Skip empty relabel config.

app/vmagent/remotewrite/remotewrite.go

@@ -3,11 +3,13 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
@@ -18,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 	xxhash "github.com/cespare/xxhash/v2"
 )
@@ -25,11 +28,14 @@ import (
 var (
 	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
-		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
+		"Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
+	remoteWriteMultitenantURLs = flagutil.NewArray("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
+		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
+		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
 		"See also -remoteWrite.maxDiskUsagePerURL")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. Default value if 2 * numberOfAvailableCPUs")
+		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
@@ -48,12 +54,27 @@ var (
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
 		`Enabled sorting for labels can slow down ingestion performance a bit`)
 	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
-		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See also -remoteWrite.maxDailySeries")
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
-		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See also -remoteWrite.maxHourlySeries")
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 )
 
-var rwctxs []*remoteWriteCtx
+var (
+	// rwctxsDefault contains statically populated entries when -remoteWrite.url is specified.
+	rwctxsDefault []*remoteWriteCtx
+
+	// rwctxsMap contains dynamically populated entries when -remoteWrite.multitenantURL is specified.
+	rwctxsMap     = make(map[tenantmetrics.TenantID][]*remoteWriteCtx)
+	rwctxsMapLock sync.Mutex
+
+	// Data without tenant id is written to defaultAuthToken if -remoteWrite.multitenantURL is specified.
+	defaultAuthToken = &auth.Token{}
+)
+
+// MultitenancyEnabled returns true if -remoteWrite.multitenantURL is specified.
+func MultitenancyEnabled() bool {
+	return len(*remoteWriteMultitenantURLs) > 0
+}
 
 // Contains the current relabelConfigs.
 var allRelabelConfigs atomic.Value
@@ -76,8 +97,11 @@ func InitSecretFlags() {
 //
 // Stop must be called for graceful shutdown.
 func Init() {
-	if len(*remoteWriteURLs) == 0 {
-		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
+	if len(*remoteWriteURLs) == 0 && len(*remoteWriteMultitenantURLs) == 0 {
+		logger.Fatalf("at least one `-remoteWrite.url` or `-remoteWrite.multitenantURL` command-line flag must be set")
+	}
+	if len(*remoteWriteURLs) > 0 && len(*remoteWriteMultitenantURLs) > 0 {
+		logger.Fatalf("cannot set both `-remoteWrite.url` and `-remoteWrite.multitenantURL` command-line flags")
 	}
 	if *maxHourlySeries > 0 {
 		hourlySeriesLimiter = bloomfilter.NewLimiter(*maxHourlySeries, time.Hour)
@@ -116,23 +140,8 @@ func Init() {
 	}
 	allRelabelConfigs.Store(rcs)
 
-	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
-	if maxInmemoryBlocks > 400 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 400
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
-	for i, remoteWriteURL := range *remoteWriteURLs {
-		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
-		if *showRemoteWriteURL {
-			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
-		}
-		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
-		rwctxs = append(rwctxs, rwctx)
+	if len(*remoteWriteURLs) > 0 {
+		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 	}
 
 	// Start config reloader.
@@ -157,6 +166,41 @@ func Init() {
 	}()
 }
 
+func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
+	if len(urls) == 0 {
+		logger.Panicf("BUG: urls must be non-empty")
+	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(urls) / maxRowsPerBlock / 100
+	if maxInmemoryBlocks > 400 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 400
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
+	rwctxs := make([]*remoteWriteCtx, len(urls))
+	for i, remoteWriteURLRaw := range urls {
+		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
+		if err != nil {
+			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		}
+		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
+		if at != nil {
+			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
+			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
+			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
+		}
+		if *showRemoteWriteURL {
+			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
+		}
+		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+	}
+	return rwctxs
+}
+
 var stopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
@@ -167,16 +211,62 @@ func Stop() {
 	close(stopCh)
 	configReloaderWG.Wait()
 
-	for _, rwctx := range rwctxs {
+	for _, rwctx := range rwctxsDefault {
 		rwctx.MustStop()
 	}
-	rwctxs = nil
+	rwctxsDefault = nil
+
+	// There is no need in locking rwctxsMapLock here, since nobody should call Push during the Stop call.
+	for _, rwctxs := range rwctxsMap {
+		for _, rwctx := range rwctxs {
+			rwctx.MustStop()
+		}
+	}
+	rwctxsMap = nil
+
+	if sl := hourlySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
+	if sl := dailySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
 }
 
 // Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
 // Note that wr may be modified by Push due to relabeling and rounding.
 func Push(wr *prompbmarshal.WriteRequest) {
+	PushWithAuthToken(nil, wr)
+}
+
+// PushWithAuthToken sends wr to remote storage systems set via `-remoteWrite.multitenantURL`.
+//
+// Note that wr may be modified by Push due to relabeling and rounding.
+func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
+		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
+		at = defaultAuthToken
+	}
+	var rwctxs []*remoteWriteCtx
+	if at == nil {
+		rwctxs = rwctxsDefault
+	} else {
+		if len(*remoteWriteMultitenantURLs) == 0 {
+			logger.Panicf("BUG: remoteWriteMultitenantURLs must be non-empty for non-nil at")
+		}
+		rwctxsMapLock.Lock()
+		tenantID := tenantmetrics.TenantID{
+			AccountID: at.AccountID,
+			ProjectID: at.ProjectID,
+		}
+		rwctxs = rwctxsMap[tenantID]
+		if rwctxs == nil {
+			rwctxs = newRemoteWriteCtxs(at, *remoteWriteMultitenantURLs)
+			rwctxsMap[tenantID] = rwctxs
+		}
+		rwctxsMapLock.Unlock()
+	}
+
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load().(*relabelConfigs)
 	pcsGlobal := rcs.global
@@ -318,17 +408,29 @@ type remoteWriteCtx struct {
 	relabelMetricsDropped *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
-	h := xxhash.Sum64([]byte(remoteWriteURL))
-	path := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
-	fq := persistentqueue.MustOpenFastQueue(path, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+	// strip query params, otherwise changing params resets pq
+	pqURL := *remoteWriteURL
+	pqURL.RawQuery = ""
+	pqURL.Fragment = ""
+	h := xxhash.Sum64([]byte(pqURL.String()))
+	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	c := newClient(argIdx, remoteWriteURL, sanitizedURL, fq, *queues)
+	var c *client
+	switch remoteWriteURL.Scheme {
+	case "http", "https":
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+	default:
+		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
+	}
+	c.init(argIdx, *queues, sanitizedURL)
+
 	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
 	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
 	pssLen := *queues
@@ -347,7 +449,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int,
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, sanitizedURL)),
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
 }
 

app/vmagent/vmimport/request_handler.go

@@ -1,40 +1,54 @@
 package vmimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="vmimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(req *http.Request) error {
+func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+			return insertRows(at, rows, extraLabels)
 		})
 	})
 }
 
-func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -74,8 +88,11 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	remotewrite.Push(&ctx.WriteRequest)
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmalert/Makefile

@@ -56,6 +56,7 @@ test-vmalert:
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
+	go test -v -race -cover ./app/vmalert/remotewrite
 
 run-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \

app/vmalert/README.md

@@ -24,7 +24,6 @@ may fail;
 * by default, rules execution is sequential within one group, but persisting of execution results to remote
 storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
 recording rule is reused in next one;
-* `vmalert` has no UI, just an API for getting groups and rules statuses.
 
 ## QuickStart
 
@@ -95,6 +94,13 @@ name: <string>
 extra_filter_labels:
   [ <labelname>: <labelvalue> ... ]
 
+# Optional list of labels added to every rule within a group.
+# It has priority over the external labels.
+# Labels are commonly used for adding environment
+# or tenant-specific tag.
+labels:
+  [ <labelname>: <labelvalue> ... ]
+
 rules:
   [ - <rule> ... ]
 ```
@@ -107,11 +113,11 @@ expression and then act according to the Rule type.
 
 There are two types of Rules:
 * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) -
-Alerting rules allows to define alert conditions via `expr` field and to send notifications 
+Alerting rules allows to define alert conditions via `expr` field and to send notifications
 [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty.
 * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -
 Recording rules allows to define `expr` which result will be than backfilled to configured
-`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally 
+`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
 expensive expressions and save their result as a new set of time series.
 
 `vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels
@@ -189,26 +195,26 @@ The state stored to the configured address on every rule evaluation.
 from configured address by querying time series with name `ALERTS_FOR_STATE`.
 
 Both flags are required for the proper state restoring. Restore process may fail if time series are missing
-in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`) 
+in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
 or received state doesn't match current `vmalert` rules configuration.
 
 
 ### Multitenancy
 
-There are the following approaches for alerting and recording rules across 
+There are the following approaches for alerting and recording rules across
 [multiple tenants](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy):
 
-* To run a separate `vmalert` instance per each tenant. 
-  The corresponding tenant must be specified in `-datasource.url` command-line flag 
-  according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). 
+* To run a separate `vmalert` instance per each tenant.
+  The corresponding tenant must be specified in `-datasource.url` command-line flag
+  according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format).
   For example, `/path/to/vmalert -datasource.url=http://vmselect:8481/select/123/prometheus`
-  would run alerts against `AccountID=123`. For recording rules the `-remoteWrite.url` command-line 
-  flag must contain the url for the specific tenant as well. 
-  For example, `-remoteWrite.url=http://vminsert:8480/insert/123/prometheus` would write recording 
+  would run alerts against `AccountID=123`. For recording rules the `-remoteWrite.url` command-line
+  flag must contain the url for the specific tenant as well.
+  For example, `-remoteWrite.url=http://vminsert:8480/insert/123/prometheus` would write recording
   rules to `AccountID=123`.
 
-* To specify `tenant` parameter per each alerting and recording group if 
-  [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used 
+* To specify `tenant` parameter per each alerting and recording group if
+  [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used
   with `-clusterMode` command-line flag. For example:
 
 ```yaml
@@ -224,18 +230,19 @@ groups:
     # Rules for accountID=456, projectID=789
 ```
 
-If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must 
-contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`. 
-`vmselect` automatically adds the specified tenant to urls per each recording rule in this case.
+If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must
+contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`.
+`vmalert` automatically adds the specified tenant to urls per each recording rule in this case.
 
-The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files 
-at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise` 
+The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files
+at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise`
 tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags).
 
 
 ### WEB
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>` - UI;
 * `http://<vmalert-addr>/api/v1/groups` - list of all loaded groups and rules;
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
 * `http://<vmalert-addr>/api/v1/<groupID>/<alertID>/status" ` - get alert status by ID.
@@ -304,6 +311,8 @@ to [/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#ran
 of the configured `-datasource.url`. Returned data then processed according to the rule type and
 backfilled to `-remoteWrite.url` via [Remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
 Vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
+Vmalert automatically disables caching on VictoriaMetrics side by sending `nocache=1` param. It allows
+to prevent cache pollution and unwanted time range boundaries adjustment during backfilling.
 
 #### Recording rules
 
@@ -340,6 +349,17 @@ See full description for these flags in `./vmalert --help`.
 * `query` template function is disabled for performance reasons (might be changed in future);
 
 
+## Monitoring
+
+`vmalert` exports various metrics in Prometheus exposition format at `http://vmalert-host:8880/metrics` page.
+We recommend setting up regular scraping of this page either through `vmagent` or by Prometheus so that the exported
+metrics may be analyzed later.
+
+Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview.
+If you have suggestions for improvements or have found a bug - please open an issue on github or add
+a review to the dashboard.
+
+
 ## Configuration
 
 Pass `-help` to `vmalert` in order to see the full list of supported
@@ -348,193 +368,219 @@ command-line flags with their descriptions.
 The shortlist of configuration flags is the following:
 ```
   -datasource.appendTypePrefix
-        Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
+    	Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.
   -datasource.basicAuth.password string
-        Optional basic auth password for -datasource.url
+    	Optional basic auth password for -datasource.url
+  -datasource.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -datasource.url
   -datasource.basicAuth.username string
-        Optional basic auth username for -datasource.url
+    	Optional basic auth username for -datasource.url
+  -datasource.bearerToken string
+    	Optional bearer auth token to use for -datasource.url.
+  -datasource.bearerTokenFile string
+    	Optional path to bearer token file to use for -datasource.url.
   -datasource.lookback duration
-        Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+    	Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
   -datasource.maxIdleConnections int
-        Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
+    	Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
   -datasource.queryStep duration
-        queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
+    	queryStep defines how far a value can fallback to when evaluating queries. For example, if datasource.queryStep=15s then param "step" with value "15s" will be added to every query.If queryStep isn't specified, rule's evaluationInterval will be used instead.
   -datasource.roundDigits int
-        Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
+    	Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
   -datasource.tlsCAFile string
-        Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
+    	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
   -datasource.tlsCertFile string
-        Optional path to client-side TLS certificate file to use when connecting to -datasource.url
+    	Optional path to client-side TLS certificate file to use when connecting to -datasource.url
   -datasource.tlsInsecureSkipVerify
-        Whether to skip tls verification when connecting to -datasource.url
+    	Whether to skip tls verification when connecting to -datasource.url
   -datasource.tlsKeyFile string
-        Optional path to client-side TLS certificate key to use when connecting to -datasource.url
+    	Optional path to client-side TLS certificate key to use when connecting to -datasource.url
   -datasource.tlsServerName string
-        Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
+    	Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
   -datasource.url string
-        VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428
+    	VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428
+  -disableAlertgroupLabel
+    	Whether to disable adding group's name as label to generated alerts and time series.
   -dryRun -rule
-        Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
+    	Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
   -enableTCP6
-        Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
+    	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-        Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
-        Prefix for environment variables if -envflag.enable is set
+    	Prefix for environment variables if -envflag.enable is set
   -evaluationInterval duration
-        How often to evaluate the rules (default 1m0s)
+    	How often to evaluate the rules (default 1m0s)
   -external.alert.source string
-        External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
-        eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
+    	External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
+    	eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used
   -external.label array
-        Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -external.url string
-        External URL is used as alert's source for sent alerts to the notifier
+    	External URL is used as alert's source for sent alerts to the notifier
   -fs.disableMmap
-        Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+    	Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -http.connTimeout duration
-        Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
+    	Incoming http connections are closed after the configured timeout. This may help to spread the incoming load among a cluster of services behind a load balancer. Please note that the real timeout may be bigger by up to 10% as a protection against the thundering herd problem (default 2m0s)
   -http.disableResponseCompression
-        Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
+    	Disable compression of HTTP responses to save CPU resources. By default compression is enabled to save network bandwidth
   -http.idleConnTimeout duration
-        Timeout for incoming idle http connections (default 1m0s)
+    	Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-        The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+    	The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
-        An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
+    	An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
-        Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
+    	Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-        Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+    	Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-        Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+    	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-        Address to listen for http connections (default ":8880")
+    	Address to listen for http connections (default ":8880")
   -loggerDisableTimestamps
-        Whether to disable writing timestamps in logs
+    	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
-        Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
+    	Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
-        Format for logs. Possible values: default, json (default "default")
+    	Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
-        Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
-        Output for the logs. Supported values: stderr, stdout (default "stderr")
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
   -loggerTimezone string
-        Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
+    	Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
-        Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+    	Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -memory.allowedBytes size
-        Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
-        Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
+    	Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+    	Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)
   -memory.allowedPercent float
-        Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
-        Auth key for /metrics. It overrides httpAuth settings
+    	Auth key for /metrics. It overrides httpAuth settings
   -notifier.basicAuth.password array
-        Optional basic auth password for -notifier.url
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Optional basic auth password for -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.basicAuth.username array
-        Optional basic auth username for -notifier.url
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Optional basic auth username for -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsCAFile array
-        Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsCertFile array
-        Optional path to client-side TLS certificate file to use when connecting to -notifier.url
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Optional path to client-side TLS certificate file to use when connecting to -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsInsecureSkipVerify array
-        Whether to skip tls verification when connecting to -notifier.url
-        Supports array of values separated by comma or specified via multiple flags.
+    	Whether to skip tls verification when connecting to -notifier.url
+    	Supports array of values separated by comma or specified via multiple flags.
   -notifier.tlsKeyFile array
-        Optional path to client-side TLS certificate key to use when connecting to -notifier.url
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Optional path to client-side TLS certificate key to use when connecting to -notifier.url
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.tlsServerName array
-        Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
+    	Supports an array of values separated by comma or specified via multiple flags.
   -notifier.url array
-        Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+    	Supports an array of values separated by comma or specified via multiple flags.
   -pprofAuthKey string
-        Auth key for /debug/pprof. It overrides httpAuth settings
+    	Auth key for /debug/pprof. It overrides httpAuth settings
   -remoteRead.basicAuth.password string
-        Optional basic auth password for -remoteRead.url
+    	Optional basic auth password for -remoteRead.url
+  -remoteRead.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -remoteRead.url
   -remoteRead.basicAuth.username string
-        Optional basic auth username for -remoteRead.url
+    	Optional basic auth username for -remoteRead.url
+  -remoteRead.bearerToken string
+    	Optional bearer auth token to use for -remoteRead.url.
+  -remoteRead.bearerTokenFile string
+    	Optional path to bearer token file to use for -remoteRead.url.
+  -remoteRead.disablePathAppend
+    	Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.
   -remoteRead.ignoreRestoreErrors
-        Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
+    	Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
   -remoteRead.lookback duration
-        Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+    	Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
   -remoteRead.tlsCAFile string
-        Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
+    	Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
   -remoteRead.tlsCertFile string
-        Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url
+    	Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url
   -remoteRead.tlsInsecureSkipVerify
-        Whether to skip tls verification when connecting to -remoteRead.url
+    	Whether to skip tls verification when connecting to -remoteRead.url
   -remoteRead.tlsKeyFile string
-        Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url
+    	Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url
   -remoteRead.tlsServerName string
-        Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
+    	Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
   -remoteRead.url vmalert
-        Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
+    	Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
   -remoteWrite.basicAuth.password string
-        Optional basic auth password for -remoteWrite.url
+    	Optional basic auth password for -remoteWrite.url
+  -remoteWrite.basicAuth.passwordFile string
+    	Optional path to basic auth password to use for -remoteWrite.url
   -remoteWrite.basicAuth.username string
-        Optional basic auth username for -remoteWrite.url
+    	Optional basic auth username for -remoteWrite.url
+  -remoteWrite.bearerToken string
+    	Optional bearer auth token to use for -remoteWrite.url.
+  -remoteWrite.bearerTokenFile string
+    	Optional path to bearer token file to use for -remoteWrite.url.
   -remoteWrite.concurrency int
-        Defines number of writers for concurrent writing into remote querier (default 1)
+    	Defines number of writers for concurrent writing into remote querier (default 1)
+  -remoteWrite.disablePathAppend
+    	Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.
   -remoteWrite.flushInterval duration
-        Defines interval of flushes to remote write endpoint (default 5s)
+    	Defines interval of flushes to remote write endpoint (default 5s)
   -remoteWrite.maxBatchSize int
-        Defines defines max number of timeseries to be flushed at once (default 1000)
+    	Defines defines max number of timeseries to be flushed at once (default 1000)
   -remoteWrite.maxQueueSize int
-        Defines the max number of pending datapoints to remote write endpoint (default 100000)
+    	Defines the max number of pending datapoints to remote write endpoint (default 100000)
   -remoteWrite.tlsCAFile string
-        Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
+    	Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
   -remoteWrite.tlsCertFile string
-        Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url
+    	Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url
   -remoteWrite.tlsInsecureSkipVerify
-        Whether to skip tls verification when connecting to -remoteWrite.url
+    	Whether to skip tls verification when connecting to -remoteWrite.url
   -remoteWrite.tlsKeyFile string
-        Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url
+    	Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url
   -remoteWrite.tlsServerName string
-        Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
+    	Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
   -remoteWrite.url string
-        Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428
+    	Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
   -replay.maxDatapointsPerQuery int
-        Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
+    	Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int
-        Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
+    	Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
   -replay.rulesDelay duration
-        Delay between rules evaluation within the group. Could be important if there are chained rules inside of the groupand processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule. Keep it equal or bigger than -remoteWrite.flushInterval. (default 1s)
+    	Delay between rules evaluation within the group. Could be important if there are chained rules inside of the groupand processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule.Keep it equal or bigger than -remoteWrite.flushInterval. (default 1s)
   -replay.timeFrom string
-        The time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+    	The time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
   -replay.timeTo string
-        The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+    	The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
   -rule array
-        Path to the file with alert rules.
-        Supports patterns. Flag can be specified multiple times.
-        Examples:
-         -rule="/path/to/file". Path to a single file with alerting rules
-         -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
-        absolute path to all .yaml files in root.
-        Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
-        Supports an array of values separated by comma or specified via multiple flags.
+    	Path to the file with alert rules.
+    	Supports patterns. Flag can be specified multiple times.
+    	Examples:
+    	 -rule="/path/to/file". Path to a single file with alerting rules
+    	 -rule="dir/*.yaml" -rule="/*.yaml". Relative path to all .yaml files in "dir" folder,
+    	absolute path to all .yaml files in root.
+    	Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+    	Supports an array of values separated by comma or specified via multiple flags.
   -rule.configCheckInterval duration
-        Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
+    	Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes
+  -rule.maxResolveDuration duration
+    	Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group.
   -rule.validateExpressions
-        Whether to validate rules expressions via MetricsQL engine (default true)
+    	Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
-        Whether to validate annotation and label templates (default true)
+    	Whether to validate annotation and label templates (default true)
   -tls
-        Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
+    	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
-        Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
+    	Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower
   -tlsKeyFile string
-        Path to file with TLS key. Used only if -tls is set
+    	Path to file with TLS key. Used only if -tls is set
   -version
-        Show VictoriaMetrics version
+    	Show VictoriaMetrics version
 ```
 
 `vmalert` supports "hot" config reload via the following methods:

app/vmalert/alerting.go

@@ -42,6 +42,9 @@ type AlertingRule struct {
 	// resets on every successful Exec
 	// may be used as Health state
 	lastExecError error
+	// stores the number of samples returned during
+	// the last evaluation
+	lastExecSamples int
 
 	metrics *alertingRuleMetrics
 }
@@ -50,6 +53,7 @@ type alertingRuleMetrics struct {
 	errors  *gauge
 	pending *gauge
 	active  *gauge
+	samples *gauge
 }
 
 func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
@@ -76,8 +80,8 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
 	ar.metrics.pending = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
-			ar.mu.Lock()
-			defer ar.mu.Unlock()
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StatePending {
@@ -88,8 +92,8 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		})
 	ar.metrics.active = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
 		func() float64 {
-			ar.mu.Lock()
-			defer ar.mu.Unlock()
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
 			var num int
 			for _, a := range ar.alerts {
 				if a.State == notifier.StateFiring {
@@ -98,15 +102,21 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerts_error{%s}`, labels),
+	ar.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
 		func() float64 {
-			ar.mu.Lock()
-			defer ar.mu.Unlock()
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
 			if ar.lastExecError == nil {
 				return 0
 			}
 			return 1
 		})
+	ar.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
+			return float64(ar.lastExecSamples)
+		})
 	return ar
 }
 
@@ -115,6 +125,7 @@ func (ar *AlertingRule) Close() {
 	metrics.UnregisterMetric(ar.metrics.active.name)
 	metrics.UnregisterMetric(ar.metrics.pending.name)
 	metrics.UnregisterMetric(ar.metrics.errors.name)
+	metrics.UnregisterMetric(ar.metrics.samples.name)
 }
 
 // String implements Stringer interface
@@ -152,7 +163,13 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 			// so the hash key will be consistent on restore
 			s.SetLabel(k, v)
 		}
-
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			s.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			s.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		a, err := ar.newAlert(s, time.Time{}, qFn) // initial alert
 		if err != nil {
 			return nil, fmt.Errorf("failed to create alert: %s", err)
@@ -167,13 +184,11 @@ func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]
 
 		// if alert with For > 0
 		prevT := time.Time{}
-		//activeAt := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
 			if at.Sub(prevT) > ar.EvalInterval {
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
-				//activeAt = at
 				a.Start = at
 			} else if at.Sub(a.Start) >= ar.For {
 				a.State = notifier.StateFiring
@@ -194,6 +209,7 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 
 	ar.lastExecError = err
 	ar.lastExecTime = time.Now()
+	ar.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
@@ -219,6 +235,14 @@ func (ar *AlertingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries, e
 			// so the hash key will be consistent on restore
 			m.SetLabel(k, v)
 		}
+		// set additional labels to identify group and rule name
+		// set additional labels to identify group and rule name
+		if ar.Name != "" {
+			m.SetLabel(alertNameLabel, ar.Name)
+		}
+		if !*disableAlertGroupLabel && ar.GroupName != "" {
+			m.SetLabel(alertGroupNameLabel, ar.GroupName)
+		}
 		h := hash(m)
 		if _, ok := updated[h]; ok {
 			// duplicate may be caused by extra labels
@@ -340,11 +364,6 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, qFn notif
 		Start:   start,
 		Expr:    ar.Expr,
 	}
-	// label defined here to make override possible by
-	// time series labels.
-	if ar.GroupName != "" {
-		a.Labels[alertGroupNameLabel] = ar.GroupName
-	}
 	for _, l := range m.Labels {
 		// drop __name__ to be consistent with Prometheus alerting
 		if l.Name == "__name__" {
@@ -384,6 +403,7 @@ func (ar *AlertingRule) RuleAPI() APIAlertingRule {
 		Expression:  ar.Expr,
 		For:         ar.For.String(),
 		LastError:   lastErr,
+		LastSamples: ar.lastExecSamples,
 		LastExec:    ar.lastExecTime,
 		Labels:      ar.Labels,
 		Annotations: ar.Annotations,
@@ -402,10 +422,11 @@ func (ar *AlertingRule) AlertsAPI() []*APIAlert {
 }
 
 func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
-	return &APIAlert{
+	aa := &APIAlert{
 		// encode as strings to avoid rounding
 		ID:      fmt.Sprintf("%d", a.ID),
 		GroupID: fmt.Sprintf("%d", a.GroupID),
+		RuleID:  fmt.Sprintf("%d", ar.RuleID),
 
 		Name:        a.Name,
 		Expression:  ar.Expr,
@@ -413,8 +434,13 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
 		Annotations: a.Annotations,
 		State:       a.State.String(),
 		ActiveAt:    a.Start,
-		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+		Restored:    a.Restored,
+		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
 	}
+	if alertURLGeneratorFn != nil {
+		aa.SourceLink = alertURLGeneratorFn(a)
+	}
+	return aa
 }
 
 const (
@@ -429,43 +455,42 @@ const (
 	alertStateLabel = "alertstate"
 
 	// alertGroupNameLabel defines the label name attached for generated time series.
+	// attaching this label may be disabled via `-disableAlertgroupLabel` flag.
 	alertGroupNameLabel = "alertgroup"
 )
 
 // alertToTimeSeries converts the given alert with the given timestamp to timeseries
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, alertToTimeSeries(ar.Name, a, timestamp))
+	tss = append(tss, alertToTimeSeries(a, timestamp))
 	if ar.For > 0 {
-		tss = append(tss, alertForToTimeSeries(ar.Name, a, timestamp))
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
 	}
 	return tss
 }
 
-func alertToTimeSeries(name string, a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertMetricName
-	labels[alertNameLabel] = name
 	labels[alertStateLabel] = a.State.String()
 	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
 // alertForToTimeSeries returns a timeseries that represents
 // state of active alerts, where value is time when alert become active
-func alertForToTimeSeries(name string, a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
 	labels := make(map[string]string)
 	for k, v := range a.Labels {
 		labels[k] = v
 	}
 	labels["__name__"] = alertForStateMetricName
-	labels[alertNameLabel] = name
 	return newTimeSeries([]float64{float64(a.Start.Unix())}, []int64{timestamp}, labels)
 }
 
-// Restore restores the state of active alerts basing on previously written timeseries.
+// Restore restores the state of active alerts basing on previously written time series.
 // Restore restores only Start field. Field State will be always Pending and supposed
 // to be updated on next Exec, as well as Value field.
 // Only rules with For > 0 will be restored.
@@ -493,23 +518,13 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 	}
 
 	for _, m := range qMetrics {
-		labels := m.Labels
-		m.Labels = make([]datasource.Label, 0)
-		// drop all extra labels, so hash key will
-		// be identical to time series received in Exec
-		for _, l := range labels {
-			if l.Name == alertNameLabel || l.Name == alertGroupNameLabel {
-				continue
-			}
-			m.Labels = append(m.Labels, l)
-		}
-
 		a, err := ar.newAlert(m, time.Unix(int64(m.Values[0]), 0), qFn)
 		if err != nil {
 			return fmt.Errorf("failed to create alert: %w", err)
 		}
 		a.ID = hash(m)
 		a.State = notifier.StatePending
+		a.Restored = true
 		ar.alerts[a.ID] = a
 		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
 	}

app/vmalert/alerting_test.go

@@ -27,7 +27,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant",
 				}),
 			},
 		},
@@ -41,7 +40,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant extra labels",
 					"job":           "foo",
 					"instance":      "bar",
 				}),
@@ -57,7 +55,6 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "instant labels override",
 				}),
 			},
 		},
@@ -68,13 +65,11 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
-					alertNameLabel:  "for",
 				}),
 				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 					[]int64{timestamp.UnixNano()},
 					map[string]string{
-						"__name__":     alertForStateMetricName,
-						alertNameLabel: "for",
+						"__name__": alertForStateMetricName,
 					}),
 			},
 		},
@@ -85,13 +80,11 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StatePending.String(),
-					alertNameLabel:  "for pending",
 				}),
 				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 					[]int64{timestamp.UnixNano()},
 					map[string]string{
-						"__name__":     alertForStateMetricName,
-						alertNameLabel: "for pending",
+						"__name__": alertForStateMetricName,
 					}),
 			},
 		},
@@ -109,23 +102,27 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 
 func TestAlertingRule_Exec(t *testing.T) {
 	const defaultStep = 5 * time.Millisecond
+	type testAlert struct {
+		labels []string
+		alert  *notifier.Alert
+	}
 	testCases := []struct {
 		rule      *AlertingRule
 		steps     [][]datasource.Metric
-		expAlerts map[uint64]*notifier.Alert
+		expAlerts []testAlert
 	}{
 		{
 			newTestAlertingRule("empty", 0),
 			[][]datasource.Metric{},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("empty labels", 0),
 			[][]datasource.Metric{
 				{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 			},
-			map[uint64]*notifier.Alert{
-				hash(datasource.Metric{}): {State: notifier.StateFiring},
+			[]testAlert{
+				{alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -133,8 +130,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -143,8 +140,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -154,8 +151,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -166,8 +163,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -179,7 +176,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
@@ -191,8 +188,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -204,10 +201,10 @@ func TestAlertingRule_Exec(t *testing.T) {
 					metricWithLabels(t, "name", "foo2"),
 				},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")):  {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateFiring},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -220,9 +217,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 			// 1: fire first alert
 			// 2: fire second alert, set first inactive
 			// 3: fire third alert, set second inactive, delete first one
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateInactive},
-				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -230,8 +227,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -240,8 +237,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 		{
@@ -252,7 +249,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset and delete pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{},
+			nil,
 		},
 		{
 			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
@@ -262,8 +259,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				// empty step to reset pending alerts
 				{},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			},
 		},
 		{
@@ -275,8 +272,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
 			},
 		},
 		{
@@ -289,8 +286,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			},
 		},
 	}
@@ -312,7 +309,15 @@ func TestAlertingRule_Exec(t *testing.T) {
 			if len(tc.rule.alerts) != len(tc.expAlerts) {
 				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
 			}
-			for key, exp := range tc.expAlerts {
+			expAlerts := make(map[uint64]*notifier.Alert)
+			for _, ta := range tc.expAlerts {
+				labels := ta.labels
+				labels = append(labels, alertNameLabel)
+				labels = append(labels, tc.rule.Name)
+				h := hash(metricWithLabels(t, labels...))
+				expAlerts[h] = ta.alert
+			}
+			for key, exp := range expAlerts {
 				got, ok := tc.rule.alerts[key]
 				if !ok {
 					t.Fatalf("expected to have key %d", key)
@@ -468,6 +473,11 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			var j int
 			for _, series := range tc.data {
 				for _, timestamp := range series.Timestamps {
+					a := tc.expAlerts[j]
+					if a.Labels == nil {
+						a.Labels = make(map[string]string)
+					}
+					a.Labels[alertNameLabel] = tc.rule.Name
 					expTS = append(expTS, tc.rule.alertToTimeSeries(tc.expAlerts[j], timestamp)...)
 					j++
 				}
@@ -496,7 +506,6 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 				),
 			},
 			map[uint64]*notifier.Alert{
@@ -509,7 +518,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
+					alertNameLabel, "metric labels",
 					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
@@ -517,6 +526,8 @@ func TestAlertingRule_Restore(t *testing.T) {
 			},
 			map[uint64]*notifier.Alert{
 				hash(metricWithLabels(t,
+					alertNameLabel, "metric labels",
+					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
 				)): {State: notifier.StatePending,
@@ -528,7 +539,6 @@ func TestAlertingRule_Restore(t *testing.T) {
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
-					alertNameLabel, "",
 					"foo", "bar",
 					"namespace", "baz",
 					// extra labels set by rule
@@ -645,18 +655,20 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 1, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "foo")): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						"region":   "east",
-						"instance": "foo",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "foo",
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(metricWithLabels(t, alertNameLabel, "common", "region", "east", "instance", "bar")): {
 					Annotations: map[string]string{},
 					Labels: map[string]string{
-						"region":   "east",
-						"instance": "bar",
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "bar",
 					},
 				},
 			},
@@ -679,20 +691,22 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 10, "instance", "bar"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "region", "east", "instance", "foo")): {
+				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "foo")): {
 					Labels: map[string]string{
-						"instance": "foo",
-						"region":   "east",
+						alertNameLabel: "override label",
+						"instance":     "foo",
+						"region":       "east",
 					},
 					Annotations: map[string]string{
 						"summary":     `Too high connection number for "foo" for region east`,
 						"description": `It is 2 connections for "foo"`,
 					},
 				},
-				hash(metricWithLabels(t, "region", "east", "instance", "bar")): {
+				hash(metricWithLabels(t, alertNameLabel, "override label", "region", "east", "instance", "bar")): {
 					Labels: map[string]string{
-						"instance": "bar",
-						"region":   "east",
+						alertNameLabel: "override label",
+						"instance":     "bar",
+						"region":       "east",
 					},
 					Annotations: map[string]string{
 						"summary":     `Too high connection number for "bar" for region east`,

app/vmalert/config/config.go

@@ -24,13 +24,16 @@ type Group struct {
 	Type        datasource.Type `yaml:"type,omitempty"`
 	File        string
 	Name        string             `yaml:"name"`
-	Interval    utils.PromDuration `yaml:"interval,omitempty"`
+	Interval    utils.PromDuration `yaml:"interval"`
 	Rules       []Rule             `yaml:"rules"`
 	Concurrency int                `yaml:"concurrency"`
 	// ExtraFilterLabels is a list label filters applied to every rule
 	// request withing a group. Is compatible only with VM datasources.
 	// See https://docs.victoriametrics.com#prometheus-querying-api-enhancements
 	ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	// It has priority over the external labels.
+	Labels map[string]string `yaml:"labels"`
 	// Checksum stores the hash of yaml definition for this group.
 	// May be used to detect any changes like rules re-ordering etc.
 	Checksum string

app/vmalert/config/config_test.go

@@ -436,7 +436,7 @@ rules:
 `)
 	})
 
-	t.Run("Ok, `for` must change cs", func(t *testing.T) {
+	t.Run("`for` change", func(t *testing.T) {
 		f(t, `
 name: TestGroup
 rules:
@@ -450,5 +450,34 @@ rules:
     expr: sum by(job) (up == 1)
 `)
 	})
-
+	t.Run("`interval` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+interval: 2s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+interval: 4s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`concurrency` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+concurrency: 2
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+concurrency: 16
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
 }

app/vmalert/config/testdata/dir/rules-update0-good.rules

@@ -3,6 +3,8 @@ groups:
     interval: 2s
     concurrency: 2
     type: prometheus
+    labels:
+      cluster: main
     rules:
       - alert: up
         expr: up == 0

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -1,5 +1,7 @@
 groups:
   - name: duplicatedGroupDiffFiles
+    labels:
+       dc: gcp
     rules:
       - alert: VMRows
         for: 5m

app/vmalert/datasource/init.go

@@ -12,9 +12,12 @@ import (
 var (
 	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
 		"E.g. http://127.0.0.1:8428")
-	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
-	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
-	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	appendTypePrefix      = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
+	bearerToken           = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
+	bearerTokenFile       = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
 
 	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
 	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
@@ -31,8 +34,15 @@ var (
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
+// Param represents an HTTP GET param
+type Param struct {
+	Key, Value string
+}
+
 // Init creates a Querier from provided flag values.
-func Init() (QuerierBuilder, error) {
+// Provided extraParams will be added as GET params to
+// each request.
+func Init(extraParams []Param) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
@@ -42,21 +52,30 @@ func Init() (QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
+	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+	}
 
-	var rd string
 	if *roundDigits > 0 {
-		rd = fmt.Sprintf("%d", *roundDigits)
+		extraParams = append(extraParams, Param{
+			Key:   "round_digits",
+			Value: fmt.Sprintf("%d", *roundDigits),
+		})
+	}
+
+	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
 
 	return &VMStorage{
 		c:                &http.Client{Transport: tr},
-		basicAuthUser:    *basicAuthUsername,
-		basicAuthPass:    *basicAuthPassword,
+		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(*addr, "/"),
 		appendTypePrefix: *appendTypePrefix,
 		lookBack:         *lookBack,
 		queryStep:        *queryStep,
-		roundDigits:      rd,
 		dataSourceType:   NewPrometheusType(),
+		extraParams:      extraParams,
 	}, nil
 }

app/vmalert/datasource/vm.go

@@ -7,35 +7,37 @@ import (
 	"net/http"
 	"strings"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c                *http.Client
+	authCfg          *promauth.Config
 	datasourceURL    string
-	basicAuthUser    string
-	basicAuthPass    string
 	appendTypePrefix bool
 	lookBack         time.Duration
 	queryStep        time.Duration
-	roundDigits      string
 
 	dataSourceType     Type
 	evaluationInterval time.Duration
 	extraLabels        []string
+	extraParams        []Param
+	disablePathAppend  bool
 }
 
 // Clone makes clone of VMStorage, shares http client.
 func (s *VMStorage) Clone() *VMStorage {
 	return &VMStorage{
-		c:                s.c,
-		datasourceURL:    s.datasourceURL,
-		basicAuthUser:    s.basicAuthUser,
-		basicAuthPass:    s.basicAuthPass,
-		lookBack:         s.lookBack,
-		queryStep:        s.queryStep,
-		appendTypePrefix: s.appendTypePrefix,
-		dataSourceType:   s.dataSourceType,
+		c:                 s.c,
+		authCfg:           s.authCfg,
+		datasourceURL:     s.datasourceURL,
+		lookBack:          s.lookBack,
+		queryStep:         s.queryStep,
+		appendTypePrefix:  s.appendTypePrefix,
+		dataSourceType:    s.dataSourceType,
+		disablePathAppend: s.disablePathAppend,
 	}
 }
 
@@ -57,16 +59,16 @@ func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
 }
 
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client, disablePathAppend bool) *VMStorage {
 	return &VMStorage{
-		c:                c,
-		basicAuthUser:    basicAuthUser,
-		basicAuthPass:    basicAuthPass,
-		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix: appendTypePrefix,
-		lookBack:         lookBack,
-		queryStep:        queryStep,
-		dataSourceType:   NewPrometheusType(),
+		c:                 c,
+		authCfg:           authCfg,
+		datasourceURL:     strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix:  appendTypePrefix,
+		lookBack:          lookBack,
+		queryStep:         queryStep,
+		dataSourceType:    NewPrometheusType(),
+		disablePathAppend: disablePathAppend,
 	}
 }
 
@@ -133,12 +135,12 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
 	resp, err := s.c.Do(req.WithContext(ctx))
 	if err != nil {
-		return nil, fmt.Errorf("error getting response from %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
+		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
 	}
 	return resp, nil
 }
@@ -149,8 +151,10 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 		return nil, err
 	}
 	req.Header.Set("Content-Type", "application/json; charset=utf-8")
-	if s.basicAuthPass != "" {
-		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
+	if s.authCfg != nil {
+		if auth := s.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
 	}
 	return req, nil
 }

app/vmalert/datasource/vm_graphite_api.go

@@ -38,7 +38,7 @@ func (r graphiteResponse) metrics() []Metric {
 func parseGraphiteResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
 	r := &graphiteResponse{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL.Redacted(), err)
 	}
 	return r.metrics(), nil
 }

app/vmalert/datasource/vm_prom_api.go

@@ -82,10 +82,10 @@ const (
 func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
 	r := &promResponse{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL, err)
+		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL.Redacted(), err)
 	}
 	if r.Status == statusError {
-		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error)
 	}
 	if r.Status != statusSuccess {
 		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
@@ -118,7 +118,9 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 	if s.appendTypePrefix {
 		r.URL.Path += prometheusPrefix
 	}
-	r.URL.Path += prometheusInstantPath
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusInstantPath
+	}
 	q := r.URL.Query()
 	if s.lookBack > 0 {
 		timestamp = timestamp.Add(-s.lookBack)
@@ -136,7 +138,9 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 	if s.appendTypePrefix {
 		r.URL.Path += prometheusPrefix
 	}
-	r.URL.Path += prometheusRangePath
+	if !s.disablePathAppend {
+		r.URL.Path += prometheusRangePath
+	}
 	q := r.URL.Query()
 	q.Add("start", fmt.Sprintf("%d", start.Unix()))
 	q.Add("end", fmt.Sprintf("%d", end.Unix()))
@@ -155,11 +159,11 @@ func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
 		// override step with user-specified value
 		q.Set("step", s.queryStep.String())
 	}
-	if s.roundDigits != "" {
-		q.Set("round_digits", s.roundDigits)
-	}
 	for _, l := range s.extraLabels {
 		q.Add("extra_label", l)
 	}
+	for _, p := range s.extraParams {
+		q.Add(p.Key, p.Value)
+	}
 	r.URL.RawQuery = q.Encode()
 }

app/vmalert/datasource/vm_test.go

@@ -10,14 +10,20 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
 	ctx           = context.Background()
 	basicAuthName = "foo"
 	basicAuthPass = "bar"
-	query         = "vm_rows"
-	queryRender   = "constantLine(10)"
+	baCfg         = &promauth.BasicAuthConfig{
+		Username: basicAuthName,
+		Password: basicAuthPass,
+	}
+	query       = "vm_rows"
+	queryRender = "constantLine(10)"
 )
 
 func TestVMInstantQuery(t *testing.T) {
@@ -73,7 +79,11 @@ func TestVMInstantQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, false, srv.Client())
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
@@ -179,12 +189,16 @@ func TestVMRangeQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	s := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, false, srv.Client())
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client(), false)
 
 	p := NewPrometheusType()
 	pq := s.BuildWithParams(QuerierParams{DataSourceType: &p, EvaluationInterval: 15 * time.Second})
 
-	_, err := pq.QueryRange(ctx, query, time.Now(), time.Time{})
+	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
 	expectError(t, err, "is missing")
 
 	_, err = pq.QueryRange(ctx, query, time.Time{}, time.Now())
@@ -216,6 +230,10 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
+	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
 	query := "up"
 	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
 	testCases := []struct {
@@ -234,6 +252,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusInstantPath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus path with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
 		{
 			"prometheus prefix",
 			false,
@@ -245,6 +274,18 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusPrefix+prometheusInstantPath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus prefix with disablePathAppend",
+			false,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
 		{
 			"prometheus range path",
 			true,
@@ -255,6 +296,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusRangePath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus range path with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "", r.URL.Path)
+			},
+		},
 		{
 			"prometheus range prefix",
 			true,
@@ -266,6 +318,18 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, prometheusPrefix+prometheusRangePath, r.URL.Path)
 			},
 		},
+		{
+			"prometheus range prefix with disablePathAppend",
+			true,
+			&VMStorage{
+				dataSourceType:    NewPrometheusType(),
+				appendTypePrefix:  true,
+				disablePathAppend: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, prometheusPrefix, r.URL.Path)
+			},
+		},
 		{
 			"graphite path",
 			false,
@@ -308,10 +372,7 @@ func TestRequestParams(t *testing.T) {
 		{
 			"basic auth",
 			false,
-			&VMStorage{
-				basicAuthUser: "foo",
-				basicAuthPass: "bar",
-			},
+			&VMStorage{authCfg: authCfg},
 			func(t *testing.T, r *http.Request) {
 				u, p, _ := r.BasicAuth()
 				checkEqualString(t, "foo", u)
@@ -321,10 +382,7 @@ func TestRequestParams(t *testing.T) {
 		{
 			"basic auth range",
 			true,
-			&VMStorage{
-				basicAuthUser: "foo",
-				basicAuthPass: "bar",
-			},
+			&VMStorage{authCfg: authCfg},
 			func(t *testing.T, r *http.Request) {
 				u, p, _ := r.BasicAuth()
 				checkEqualString(t, "foo", u)
@@ -385,7 +443,7 @@ func TestRequestParams(t *testing.T) {
 			"round digits",
 			false,
 			&VMStorage{
-				roundDigits: "10",
+				extraParams: []Param{{"round_digits", "10"}},
 			},
 			func(t *testing.T, r *http.Request) {
 				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
@@ -421,6 +479,20 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
+		{
+			"extra params",
+			false,
+			&VMStorage{
+				extraParams: []Param{
+					{Key: "nocache", Value: "1"},
+					{Key: "max_lookback", Value: "1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("max_lookback=1h&nocache=1&query=%s&time=%d", query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
 	}
 
 	for _, tc := range testCases {

app/vmalert/group.go

@@ -18,15 +18,17 @@ import (
 
 // Group is an entity for grouping rules
 type Group struct {
-	mu                sync.RWMutex
-	Name              string
-	File              string
-	Rules             []Rule
-	Type              datasource.Type
-	Interval          time.Duration
-	Concurrency       int
-	Checksum          string
+	mu          sync.RWMutex
+	Name        string
+	File        string
+	Rules       []Rule
+	Type        datasource.Type
+	Interval    time.Duration
+	Concurrency int
+	Checksum    string
+
 	ExtraFilterLabels map[string]string
+	Labels            map[string]string
 
 	doneCh     chan struct{}
 	finishedCh chan struct{}
@@ -50,6 +52,23 @@ func newGroupMetrics(name, file string) *groupMetrics {
 	return m
 }
 
+// merges group rule labels into result map
+// set2 has priority over set1.
+func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
+	r := map[string]string{}
+	for k, v := range set1 {
+		r[k] = v
+	}
+	for k, v := range set2 {
+		if prevV, ok := r[k]; ok {
+			logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+				k, prevV, groupName, ruleName, k, v)
+		}
+		r[k] = v
+	}
+	return r
+}
+
 func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
 	g := &Group{
 		Type:              cfg.Type,
@@ -59,6 +78,7 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Concurrency:       cfg.Concurrency,
 		Checksum:          cfg.Checksum,
 		ExtraFilterLabels: cfg.ExtraFilterLabels,
+		Labels:            cfg.Labels,
 
 		doneCh:     make(chan struct{}),
 		finishedCh: make(chan struct{}),
@@ -73,17 +93,20 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 	}
 	rules := make([]Rule, len(cfg.Rules))
 	for i, r := range cfg.Rules {
-		// override rule labels with external labels
-		for k, v := range labels {
-			if prevV, ok := r.Labels[k]; ok {
-				logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
-					k, prevV, g.Name, r.Name(), k, v)
-			}
-			if r.Labels == nil {
-				r.Labels = map[string]string{}
-			}
-			r.Labels[k] = v
+		var extraLabels map[string]string
+		// apply external labels
+		if len(labels) > 0 {
+			extraLabels = labels
 		}
+		// apply group labels, it has priority on external labels
+		if len(cfg.Labels) > 0 {
+			extraLabels = mergeLabels(g.Name, r.Name(), extraLabels, g.Labels)
+		}
+		// apply rules labels, it has priority on other labels
+		if len(extraLabels) > 0 {
+			r.Labels = mergeLabels(g.Name, r.Name(), extraLabels, r.Labels)
+		}
+
 		rules[i] = g.newRule(qb, r)
 	}
 	g.Rules = rules
@@ -100,6 +123,9 @@ func (g *Group) newRule(qb datasource.QuerierBuilder, rule config.Rule) Rule {
 // ID return unique group ID that consists of
 // rules file and group name
 func (g *Group) ID() uint64 {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
 	hash := fnv.New64a()
 	hash.Write([]byte(g.File))
 	hash.Write([]byte("\xff"))
@@ -110,6 +136,7 @@ func (g *Group) ID() uint64 {
 
 // Restore restores alerts state for group rules
 func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookback time.Duration, labels map[string]string) error {
+	labels = mergeLabels(g.Name, "", labels, g.Labels)
 	for _, rule := range g.Rules {
 		rr, ok := rule.(*AlertingRule)
 		if !ok {
@@ -166,20 +193,18 @@ func (g *Group) updateWith(newGroup *Group) error {
 	for _, nr := range rulesRegistry {
 		newRules = append(newRules, nr)
 	}
+	// note that g.Interval is not updated here
+	// so the value can be compared later in
+	// group.Start function
 	g.Type = newGroup.Type
 	g.Concurrency = newGroup.Concurrency
 	g.ExtraFilterLabels = newGroup.ExtraFilterLabels
+	g.Labels = newGroup.Labels
 	g.Checksum = newGroup.Checksum
 	g.Rules = newRules
 	return nil
 }
 
-var (
-	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
-	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
-	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
-)
-
 func (g *Group) close() {
 	if g.doneCh == nil {
 		return
@@ -220,7 +245,16 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 	}
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
-	e := &executor{nts, rw}
+	e := &executor{rw: rw}
+	for _, nt := range nts {
+		ent := eNotifier{
+			Notifier:         nt,
+			alertsSent:       getOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", nt.Addr())),
+			alertsSendErrors: getOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", nt.Addr())),
+		}
+		e.notifiers = append(e.notifiers, ent)
+	}
+
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 	for {
@@ -249,8 +283,8 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 		case <-t.C:
 			g.metrics.iterationTotal.Inc()
 			iterationStart := time.Now()
-
-			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, g.Interval)
+			resolveDuration := getResolveDuration(g.Interval)
+			errs := e.execConcurrently(ctx, g.Rules, g.Concurrency, resolveDuration)
 			for err := range errs {
 				if err != nil {
 					logger.Errorf("group %q: %s", g.Name, err)
@@ -262,17 +296,34 @@ func (g *Group) start(ctx context.Context, nts []notifier.Notifier, rw *remotewr
 	}
 }
 
+// resolveDuration for alerts is equal to 3 interval evaluations
+// so in case if vmalert stops sending updates for some reason,
+// notifier could automatically resolve the alert.
+func getResolveDuration(groupInterval time.Duration) time.Duration {
+	resolveInterval := groupInterval * 3
+	if *maxResolveDuration > 0 && (resolveInterval > *maxResolveDuration) {
+		return *maxResolveDuration
+	}
+	return resolveInterval
+}
+
 type executor struct {
-	notifiers []notifier.Notifier
+	notifiers []eNotifier
 	rw        *remotewrite.Client
 }
 
-func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, interval time.Duration) chan error {
+type eNotifier struct {
+	notifier.Notifier
+	alertsSent       *counter
+	alertsSendErrors *counter
+}
+
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurrency int, resolveDuration time.Duration) chan error {
 	res := make(chan error, len(rules))
 	if concurrency == 1 {
 		// fast path
 		for _, rule := range rules {
-			res <- e.exec(ctx, rule, interval)
+			res <- e.exec(ctx, rule, resolveDuration)
 		}
 		close(res)
 		return res
@@ -285,7 +336,7 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 			sem <- struct{}{}
 			wg.Add(1)
 			go func(r Rule) {
-				res <- e.exec(ctx, r, interval)
+				res <- e.exec(ctx, r, resolveDuration)
 				<-sem
 				wg.Done()
 			}(rule)
@@ -297,19 +348,16 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, concurren
 }
 
 var (
-	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
-	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
-	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+	alertsFired = metrics.NewCounter(`vmalert_alerts_fired_total`)
+
+	execTotal  = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)
 
 	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
 )
 
-func (e *executor) exec(ctx context.Context, rule Rule, interval time.Duration) error {
+func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
 	execTotal.Inc()
-	execStart := time.Now()
-	defer func() {
-		execDuration.UpdateDuration(execStart)
-	}()
 
 	tss, err := rule.Exec(ctx)
 	if err != nil {
@@ -334,10 +382,7 @@ func (e *executor) exec(ctx context.Context, rule Rule, interval time.Duration)
 	for _, a := range ar.alerts {
 		switch a.State {
 		case notifier.StateFiring:
-			// set End to execStart + 3 intervals
-			// so notifier can resolve it automatically if `vmalert`
-			// won't be able to send resolve for some reason
-			a.End = time.Now().Add(3 * interval)
+			a.End = time.Now().Add(resolveDuration)
 			alerts = append(alerts, *a)
 		case notifier.StateInactive:
 			// set End to execStart to notify
@@ -350,11 +395,11 @@ func (e *executor) exec(ctx context.Context, rule Rule, interval time.Duration)
 		return nil
 	}
 
-	alertsSent.Add(len(alerts))
 	errGr := new(utils.ErrGroup)
 	for _, nt := range e.notifiers {
+		nt.alertsSent.Add(len(alerts))
 		if err := nt.Send(ctx, alerts); err != nil {
-			alertsSendErrors.Inc()
+			nt.alertsSendErrors.Inc()
 			errGr.Add(fmt.Errorf("rule %q: failed to send alerts: %w", rule, err))
 		}
 	}

app/vmalert/group_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"testing"
 	"time"
@@ -191,7 +192,14 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert1.Labels["label"] = "bar"
 	alert1.Labels["host"] = inst1
-	alert1.ID = hash(m1)
+	// add service labels
+	alert1.Labels[alertNameLabel] = alert1.Name
+	alert1.Labels[alertGroupNameLabel] = g.Name
+	var labels1 []string
+	for k, v := range alert1.Labels {
+		labels1 = append(labels1, k, v)
+	}
+	alert1.ID = hash(metricWithLabels(t, labels1...))
 
 	alert2, err := r.newAlert(m2, time.Now(), nil)
 	if err != nil {
@@ -203,7 +211,14 @@ func TestGroupStart(t *testing.T) {
 	// add rule labels - see config/testdata/rules1-good.rules
 	alert2.Labels["label"] = "bar"
 	alert2.Labels["host"] = inst2
-	alert2.ID = hash(m2)
+	// add service labels
+	alert2.Labels[alertNameLabel] = alert2.Name
+	alert2.Labels[alertGroupNameLabel] = g.Name
+	var labels2 []string
+	for k, v := range alert2.Labels {
+		labels2 = append(labels2, k, v)
+	}
+	alert2.ID = hash(metricWithLabels(t, labels2...))
 
 	finished := make(chan struct{})
 	fs.add(m1)
@@ -235,3 +250,27 @@ func TestGroupStart(t *testing.T) {
 	g.close()
 	<-finished
 }
+
+func TestResolveDuration(t *testing.T) {
+	testCases := []struct {
+		groupInterval time.Duration
+		maxDuration   time.Duration
+		expected      time.Duration
+	}{
+		{time.Minute, 0, 3 * time.Minute},
+		{3 * time.Minute, 0, 9 * time.Minute},
+		{time.Minute, 2 * time.Minute, 2 * time.Minute},
+		{0, 0, 0},
+	}
+	defaultResolveDuration := *maxResolveDuration
+	defer func() { *maxResolveDuration = defaultResolveDuration }()
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
+			*maxResolveDuration = tc.maxDuration
+			got := getResolveDuration(tc.groupInterval)
+			if got != tc.expected {
+				t.Errorf("expected to have %v; got %v", tc.expected, got)
+			}
+		})
+	}
+}

app/vmalert/helpers_test.go

@@ -63,6 +63,7 @@ type fakeNotifier struct {
 	alerts []notifier.Alert
 }
 
+func (*fakeNotifier) Addr() string { return "" }
 func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
 	fn.Lock()
 	defer fn.Unlock()
@@ -204,7 +205,8 @@ func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 			}*/
 		}
 		if len(expTS.Labels) != len(gotTS.Labels) {
-			return fmt.Errorf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+			return fmt.Errorf("expected number of labels %d (%v); got %d (%v)",
+				len(expTS.Labels), expTS.Labels, len(gotTS.Labels), gotTS.Labels)
 		}
 		for i, exp := range expTS.Labels {
 			got := gotTS.Labels[i]

app/vmalert/main.go

@@ -42,6 +42,8 @@ Rule files may contain %{ENV_VAR} placeholders, which are substituted by the cor
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
+	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
+		"which is by default equal to 3 evaluation intervals of the parent group.")
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
@@ -52,9 +54,13 @@ eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.")
 
+	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's name as label to generated alerts and time series.")
+
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
 )
 
+var alertURLGeneratorFn notifier.AlertURLGenerator
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -75,21 +81,31 @@ func main() {
 		}
 		return
 	}
+
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		logger.Fatalf("failed to init `external.url`: %s", err)
+	}
+	notifier.InitTemplateFunc(eu)
+	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
+	if err != nil {
+		logger.Fatalf("failed to init `external.alert.source`: %s", err)
+	}
+
 	if *replayFrom != "" || *replayTo != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
 			logger.Fatalf("failed to init remoteWrite: %s", err)
 		}
-		eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-		if err != nil {
-			logger.Fatalf("failed to init `external.url`: %s", err)
-		}
 		notifier.InitTemplateFunc(eu)
 		groupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
 			logger.Fatalf("cannot parse configuration file: %s", err)
 		}
-		q, err := datasource.Init()
+		// prevent queries from caching and boundaries aligning
+		// when querying VictoriaMetrics datasource.
+		noCache := datasource.Param{Key: "nocache", Value: "1"}
+		q, err := datasource.Init([]datasource.Param{noCache})
 		if err != nil {
 			logger.Fatalf("failed to init datasource: %s", err)
 		}
@@ -111,11 +127,16 @@ func main() {
 		logger.Fatalf("cannot parse configuration file: %s", err)
 	}
 
+	// Register SIGHUP handler for config re-read just before manager.start call.
+	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
 	if err := manager.start(ctx, groupsCfg); err != nil {
 		logger.Fatalf("failed to start: %s", err)
 	}
 
-	go configReload(ctx, manager, groupsCfg)
+	go configReload(ctx, manager, groupsCfg, sighupCh)
 
 	rh := &requestHandler{m: manager}
 	go httpserver.Serve(*httpListenAddr, rh.handler)
@@ -137,24 +158,14 @@ var (
 )
 
 func newManager(ctx context.Context) (*manager, error) {
-	q, err := datasource.Init()
+	q, err := datasource.Init(nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.url`: %w", err)
-	}
-	notifier.InitTemplateFunc(eu)
-	aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
-	if err != nil {
-		return nil, fmt.Errorf("failed to init `external.alert.source`: %w", err)
-	}
-	nts, err := notifier.Init(aug)
+	nts, err := notifier.Init(alertURLGeneratorFn)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
-
 	manager := &manager{
 		groups:         make(map[uint64]*Group),
 		querierBuilder: q,
@@ -239,12 +250,7 @@ See the docs at https://docs.victoriametrics.com/vmalert.html .
 	flagutil.Usage(s)
 }
 
-func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
-	// Register SIGHUP handler for config re-read just before manager.start call.
-	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
-	sighupCh := procutil.NewSighupChan()
-
+func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
 	var configCheckCh <-chan time.Time
 	if *rulesCheckInterval > 0 {
 		ticker := time.NewTicker(*rulesCheckInterval)
@@ -266,10 +272,15 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group) {
 		}
 		newGroupsCfg, err := config.Parse(*rulePath, *validateTemplates, *validateExpressions)
 		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
 			logger.Errorf("cannot parse configuration file: %s", err)
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
+			// set success to 1 since previous reload
+			// could have been unsuccessful
+			configSuccess.Set(1)
 			// config didn't change - skip it
 			continue
 		}

app/vmalert/main_test.go

@@ -94,14 +94,19 @@ groups:
 	*rulesCheckInterval = 200 * time.Millisecond
 	*rulePath = []string{f.Name()}
 	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 
 	m := &manager{
 		querierBuilder: &fakeQuerier{},
 		groups:         make(map[uint64]*Group),
 		labels:         map[string]string{},
 	}
-	go configReload(ctx, m, nil)
+
+	syncCh := make(chan struct{})
+	sighupCh := procutil.NewSighupChan()
+	go func() {
+		configReload(ctx, m, nil, sighupCh)
+		close(syncCh)
+	}()
 
 	lenLocked := func(m *manager) int {
 		m.groupsMu.RLock()
@@ -138,6 +143,9 @@ groups:
 	if groupsLen != 1 { // should remain unchanged
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
 	}
+
+	cancel()
+	<-syncCh
 }
 
 func writeToFile(t *testing.T, file, b string) {

app/vmalert/manager.go

@@ -148,6 +148,7 @@ func (g *Group) toAPI() APIGroup {
 		Interval:          g.Interval.String(),
 		Concurrency:       g.Concurrency,
 		ExtraFilterLabels: g.ExtraFilterLabels,
+		Labels:            g.Labels,
 	}
 	for _, r := range g.Rules {
 		switch v := r.(type) {

app/vmalert/manager_test.go

@@ -148,7 +148,7 @@ func TestManagerUpdate(t *testing.T) {
 							Name:   "VMRows",
 							Expr:   "vm_rows > 0",
 							For:    5 * time.Minute,
-							Labels: map[string]string{"label": "bar"},
+							Labels: map[string]string{"dc": "gcp", "label": "bar"},
 							Annotations: map[string]string{
 								"summary":     "{{ $value }}",
 								"description": "{{$labels}}",

app/vmalert/notifier/alert.go

@@ -14,17 +14,28 @@ import (
 // Alert the triggered alert
 // TODO: Looks like alert name isn't unique
 type Alert struct {
-	GroupID     uint64
-	Name        string
-	Labels      map[string]string
+	// GroupID contains the ID of the parent rules group
+	GroupID uint64
+	// Name represents Alert name
+	Name string
+	// Labels is the list of label-value pairs attached to the Alert
+	Labels map[string]string
+	// Annotations is the list of annotations generated on Alert evaluation
 	Annotations map[string]string
-	State       AlertState
-
-	Expr  string
+	// State represents the current state of the Alert
+	State AlertState
+	// Expr contains expression that was executed to generate the Alert
+	Expr string
+	// Start defines the moment of time when Alert has triggered
 	Start time.Time
-	End   time.Time
+	// End defines the moment of time when Alert supposed to expire
+	End time.Time
+	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
-	ID    uint64
+	// ID is the unique identifer for the Alert
+	ID uint64
+	// Restored is true if Alert was restored after restart
+	Restored bool
 }
 
 // AlertState type indicates the Alert state
@@ -90,13 +101,13 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, funcs
 	eg := new(utils.ErrGroup)
 	r := make(map[string]string, len(annotations))
 	for key, text := range annotations {
-		r[key] = text
 		buf.Reset()
 		builder.Reset()
 		builder.Grow(len(tplHeader) + len(text))
 		builder.WriteString(tplHeader)
 		builder.WriteString(text)
 		if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
+			r[key] = text
 			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
 		}

app/vmalert/notifier/alertmanager.go

@@ -12,6 +12,7 @@ import (
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
+	addr          string
 	alertURL      string
 	basicAuthUser string
 	basicAuthPass string
@@ -19,6 +20,9 @@ type AlertManager struct {
 	client        *http.Client
 }
 
+// Addr returns address where alerts are sent.
+func (am AlertManager) Addr() string { return am.addr }
+
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	b := &bytes.Buffer{}
@@ -57,9 +61,10 @@ const alertManagerPath = "/api/v2/alerts"
 
 // NewAlertManager is a constructor for AlertManager
 func NewAlertManager(alertManagerURL, user, pass string, fn AlertURLGenerator, c *http.Client) *AlertManager {
-	addr := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
+	url := strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath
 	return &AlertManager{
-		alertURL:      addr,
+		addr:          alertManagerURL,
+		alertURL:      url,
 		argFunc:       fn,
 		client:        c,
 		basicAuthUser: user,

app/vmalert/notifier/alertmanager_test.go

@@ -10,6 +10,14 @@ import (
 	"time"
 )
 
+func TestAlertManager_Addr(t *testing.T) {
+	const addr = "http://localhost"
+	am := NewAlertManager(addr, "", "", nil, nil)
+	if am.Addr() != addr {
+		t.Errorf("expected to have %q; got %q", addr, am.Addr())
+	}
+}
+
 func TestAlertManager_Send(t *testing.T) {
 	const baUser, baPass = "foo", "bar"
 	mux := http.NewServeMux()

app/vmalert/notifier/notifier.go

@@ -2,7 +2,12 @@ package notifier
 
 import "context"
 
-// Notifier is common interface for alert manager provider
+// Notifier is a common interface for alert manager provider
 type Notifier interface {
+	// Send sends the given list of alerts.
+	// Returns an error if fails to send the alerts.
+	// Must unblock if the given ctx is cancelled.
 	Send(ctx context.Context, alerts []Alert) error
+	// Addr returns address where alerts are sent.
+	Addr() string
 }

app/vmalert/recording.go

@@ -35,12 +35,16 @@ type RecordingRule struct {
 	// resets on every successful Exec
 	// may be used as Health state
 	lastExecError error
+	// stores the number of samples returned during
+	// the last evaluation
+	lastExecSamples int
 
 	metrics *recordingRuleMetrics
 }
 
 type recordingRuleMetrics struct {
-	errors *gauge
+	errors  *gauge
+	samples *gauge
 }
 
 // String implements Stringer interface
@@ -73,19 +77,26 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
 	rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
-			rr.mu.Lock()
-			defer rr.mu.Unlock()
+			rr.mu.RLock()
+			defer rr.mu.RUnlock()
 			if rr.lastExecError == nil {
 				return 0
 			}
 			return 1
 		})
+	rr.metrics.samples = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			rr.mu.RLock()
+			defer rr.mu.RUnlock()
+			return float64(rr.lastExecSamples)
+		})
 	return rr
 }
 
 // Close unregisters rule metrics
 func (rr *RecordingRule) Close() {
 	metrics.UnregisterMetric(rr.metrics.errors.name)
+	metrics.UnregisterMetric(rr.metrics.samples.name)
 }
 
 // ExecRange executes recording rule on the given time range similarly to Exec.
@@ -118,6 +129,7 @@ func (rr *RecordingRule) Exec(ctx context.Context) ([]prompbmarshal.TimeSeries,
 
 	rr.lastExecTime = time.Now()
 	rr.lastExecError = err
+	rr.lastExecSamples = len(qMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", rr.Expr, err)
 	}
@@ -190,13 +202,14 @@ func (rr *RecordingRule) RuleAPI() APIRecordingRule {
 	}
 	return APIRecordingRule{
 		// encode as strings to avoid rounding
-		ID:         fmt.Sprintf("%d", rr.ID()),
-		GroupID:    fmt.Sprintf("%d", rr.GroupID),
-		Name:       rr.Name,
-		Type:       rr.Type.String(),
-		Expression: rr.Expr,
-		LastError:  lastErr,
-		LastExec:   rr.lastExecTime,
-		Labels:     rr.Labels,
+		ID:          fmt.Sprintf("%d", rr.ID()),
+		GroupID:     fmt.Sprintf("%d", rr.GroupID),
+		Name:        rr.Name,
+		Type:        rr.Type.String(),
+		Expression:  rr.Expr,
+		LastError:   lastErr,
+		LastSamples: rr.lastExecSamples,
+		LastExec:    rr.lastExecTime,
+		Labels:      rr.Labels,
 	}
 }

app/vmalert/remoteread/init.go

@@ -12,9 +12,13 @@ import (
 var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
 		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"E.g. http://127.0.0.1:8428")
+		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
+	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
+	bearerToken           = flag.String("remoteRead.bearerToken", "", "Optional bearer auth token to use for -remoteRead.url.")
+	bearerTokenFile       = flag.String("remoteRead.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteRead.url.")
+
 	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
 	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
 	tlsKeyFile            = flag.String("remoteRead.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url")
@@ -22,6 +26,7 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
 		"By default the server name from -remoteRead.url is used")
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path to the configured -remoteRead.url.")
 )
 
 // Init creates a Querier from provided flag values.
@@ -34,6 +39,10 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, 0, 0, false, c), nil
+	return datasource.NewVMStorage(*addr, authCfg, 0, 0, false, c, *disablePathAppend), nil
 }

app/vmalert/remotewrite/init.go

@@ -11,9 +11,13 @@ import (
 
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
-		"and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428")
-	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
-	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
+		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
+	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
+	bearerToken           = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url.")
+	bearerTokenFile       = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
@@ -27,6 +31,7 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
 		"By default the server name from -remoteWrite.url is used")
+	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
 )
 
 // Init creates Client object from given flags.
@@ -41,14 +46,19 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
+	authCfg, err := utils.AuthConfig(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile, *bearerToken, *bearerTokenFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
 	return NewClient(ctx, Config{
-		Addr:          *addr,
-		Concurrency:   *concurrency,
-		MaxQueueSize:  *maxQueueSize,
-		MaxBatchSize:  *maxBatchSize,
-		FlushInterval: *flushInterval,
-		BasicAuthUser: *basicAuthUsername,
-		BasicAuthPass: *basicAuthPassword,
-		Transport:     t,
+		Addr:              *addr,
+		AuthCfg:           authCfg,
+		Concurrency:       *concurrency,
+		MaxQueueSize:      *maxQueueSize,
+		MaxBatchSize:      *maxBatchSize,
+		FlushInterval:     *flushInterval,
+		DisablePathAppend: *disablePathAppend,
+		Transport:         t,
 	})
 }

app/vmalert/remotewrite/remotewrite.go

@@ -6,26 +6,30 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/golang/snappy"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 // Client is an asynchronous HTTP client for writing
 // timeseries via remote write protocol.
 type Client struct {
-	addr           string
-	c              *http.Client
-	input          chan prompbmarshal.TimeSeries
-	baUser, baPass string
-	flushInterval  time.Duration
-	maxBatchSize   int
-	maxQueueSize   int
+	addr              string
+	c                 *http.Client
+	authCfg           *promauth.Config
+	input             chan prompbmarshal.TimeSeries
+	flushInterval     time.Duration
+	maxBatchSize      int
+	maxQueueSize      int
+	disablePathAppend bool
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
@@ -34,10 +38,8 @@ type Client struct {
 // Config is config for remote write.
 type Config struct {
 	// Addr of remote storage
-	Addr string
-
-	BasicAuthUser string
-	BasicAuthPass string
+	Addr    string
+	AuthCfg *promauth.Config
 
 	// Concurrency defines number of readers that
 	// concurrently read from the queue and flush data
@@ -56,6 +58,8 @@ type Config struct {
 	WriteTimeout time.Duration
 	// Transport will be used by the underlying http.Client
 	Transport *http.Transport
+	// DisablePathAppend can be used to not automatically append '/api/v1/write' to the remote write url
+	DisablePathAppend bool
 }
 
 const (
@@ -89,24 +93,25 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Transport == nil {
 		cfg.Transport = http.DefaultTransport.(*http.Transport).Clone()
 	}
+	cc := defaultConcurrency
+	if cfg.Concurrency > 0 {
+		cc = cfg.Concurrency
+	}
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.WriteTimeout,
 			Transport: cfg.Transport,
 		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
-		baUser:        cfg.BasicAuthUser,
-		baPass:        cfg.BasicAuthPass,
-		flushInterval: cfg.FlushInterval,
-		maxBatchSize:  cfg.MaxBatchSize,
-		maxQueueSize:  cfg.MaxQueueSize,
-		doneCh:        make(chan struct{}),
-		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
-	}
-	cc := defaultConcurrency
-	if cfg.Concurrency > 0 {
-		cc = cfg.Concurrency
+		addr:              strings.TrimSuffix(cfg.Addr, "/"),
+		authCfg:           cfg.AuthCfg,
+		flushInterval:     cfg.FlushInterval,
+		maxBatchSize:      cfg.MaxBatchSize,
+		maxQueueSize:      cfg.MaxQueueSize,
+		doneCh:            make(chan struct{}),
+		input:             make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
+		disablePathAppend: cfg.DisablePathAppend,
 	}
+
 	for i := 0; i < cc; i++ {
 		c.run(ctx)
 	}
@@ -179,10 +184,11 @@ func (c *Client) run(ctx context.Context) {
 }
 
 var (
-	sentRows     = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
-	sentBytes    = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
-	droppedRows  = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	droppedBytes = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	droppedBytes        = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
+	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
 )
 
 // flush is a blocking function that marshals WriteRequest and sends
@@ -193,6 +199,7 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 		return
 	}
 	defer prompbmarshal.ResetWriteRequest(wr)
+	defer bufferFlushDuration.UpdateDuration(time.Now())
 
 	data, err := wr.Marshal()
 	if err != nil {
@@ -228,20 +235,24 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
-	if c.baPass != "" {
-		req.SetBasicAuth(c.baUser, c.baPass)
+	if c.authCfg != nil {
+		if auth := c.authCfg.GetAuthHeader(); auth != "" {
+			req.Header.Set("Authorization", auth)
+		}
+	}
+	if !c.disablePathAppend {
+		req.URL.Path = path.Join(req.URL.Path, writePath)
 	}
-	req.URL.Path += writePath
 	resp, err := c.c.Do(req.WithContext(ctx))
 	if err != nil {
 		return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)",
-			req.URL, err, len(data), r.Size())
+			req.URL.Redacted(), err, len(data), r.Size())
 	}
 	defer func() { _ = resp.Body.Close() }()
-	if resp.StatusCode != http.StatusNoContent {
+	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
 		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
-			resp.StatusCode, req.URL, body)
+			resp.StatusCode, req.URL.Redacted(), body)
 	}
 	return nil
 }

app/vmalert/tpl/footer.qtpl

@@ -0,0 +1,36 @@
+{% func Footer() %}
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+{% endfunc %}

app/vmalert/tpl/footer.qtpl.go

@@ -0,0 +1,86 @@
+// Code generated by qtc from "footer.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/footer.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/footer.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/footer.qtpl:1
+func StreamFooter(qw422016 *qt422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:1
+	qw422016.N().S(`
+        </main>
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+        <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
+        <script type="text/javascript">
+            function expandAll() {
+               $('.collapse').addClass('show');
+            }
+            function collapseAll() {
+               $('.collapse').removeClass('show');
+            }
+
+            $(document).ready(function() {
+              // prevent collapse logic on link click
+              $(".group-heading a").click(function(e) {
+                e.stopPropagation();
+              });
+
+              $(".group-heading").click(function(e) {
+                 let target = $(this).attr('data-bs-target');
+                 let el = $('#'+target);
+                  new bootstrap.Collapse(el, {
+                    toggle: true
+                  });
+              });
+
+              var hash = window.location.hash.substr(1);
+              let group = $('#'+hash);
+              if (group.length > 0) {
+                group.click();
+              }
+            });
+        </script>
+    </body>
+</html>
+`)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func WriteFooter(qq422016 qtio422016.Writer) {
+//line app/vmalert/tpl/footer.qtpl:36
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	StreamFooter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/footer.qtpl:36
+}
+
+//line app/vmalert/tpl/footer.qtpl:36
+func Footer() string {
+//line app/vmalert/tpl/footer.qtpl:36
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/footer.qtpl:36
+	WriteFooter(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/footer.qtpl:36
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/footer.qtpl:36
+	return qs422016
+//line app/vmalert/tpl/footer.qtpl:36
+}

app/vmalert/tpl/header.qtpl

@@ -0,0 +1,43 @@
+{% func Header(title string, pages []NavItem) %}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    {%= PrintNavItems(title, pages) %}
+    <main class="px-2">
+{% endfunc %}

app/vmalert/tpl/header.qtpl.go

@@ -0,0 +1,107 @@
+// Code generated by qtc from "header.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/header.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/header.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/header.qtpl:1
+func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:1
+	qw422016.N().S(`
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>vmalert`)
+//line app/vmalert/tpl/header.qtpl:5
+	if title != "" {
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.N().S(` - `)
+//line app/vmalert/tpl/header.qtpl:5
+		qw422016.E().S(title)
+//line app/vmalert/tpl/header.qtpl:5
+	}
+//line app/vmalert/tpl/header.qtpl:5
+	qw422016.N().S(`</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+    <style>
+        body{
+          min-height: 75rem;
+          padding-top: 4.5rem;
+        }
+        pre {
+            overflow: scroll;
+            max-width: 600px;
+            min-height: 30px;
+        }
+        .group-heading {
+            cursor: pointer;
+            padding: 5px;
+            margin-top: 5px;
+            position: relative;
+        }
+        .group-heading .anchor {
+            position:absolute;
+            top:-60px;
+        }
+        .group-heading span {
+            float: right;
+            margin-left: 5px;
+            margin-right: 5px;
+        }
+         .group-heading:hover {
+            background-color: #f8f9fa!important;
+        }
+        .table .error-cell{
+            word-break: break-word;
+        }
+    </style>
+</head>
+<body>
+    `)
+//line app/vmalert/tpl/header.qtpl:41
+	StreamPrintNavItems(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:41
+	qw422016.N().S(`
+    <main class="px-2">
+`)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
+//line app/vmalert/tpl/header.qtpl:43
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:43
+	StreamHeader(qw422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:43
+}
+
+//line app/vmalert/tpl/header.qtpl:43
+func Header(title string, pages []NavItem) string {
+//line app/vmalert/tpl/header.qtpl:43
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:43
+	WriteHeader(qb422016, title, pages)
+//line app/vmalert/tpl/header.qtpl:43
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:43
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:43
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:43
+}

app/vmalert/tpl/nav.qtpl

@@ -0,0 +1,25 @@
+{% code
+type NavItem struct {
+	Name string
+	Url  string
+}
+%}
+
+{% func PrintNavItems(current string, items []NavItem) %}
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            {% for _, item := range items  %}
+                <li class="nav-item">
+                    <a class="nav-link{% if current == item.Name %} active{% endif %}" href="{%s item.Url %}">
+                        {%s item.Name %}
+                    </a>
+                </li>
+            {% endfor %}
+        </ul>
+  </div>
+</nav>
+{% endfunc %}
+
+

app/vmalert/tpl/nav.qtpl.go

@@ -0,0 +1,96 @@
+// Code generated by qtc from "nav.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/tpl/nav.qtpl:1
+package tpl
+
+//line app/vmalert/tpl/nav.qtpl:1
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/tpl/nav.qtpl:1
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/tpl/nav.qtpl:2
+type NavItem struct {
+	Name string
+	Url  string
+}
+
+//line app/vmalert/tpl/nav.qtpl:8
+func StreamPrintNavItems(qw422016 *qt422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:8
+	qw422016.N().S(`
+<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+  <div class="container-fluid">
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+            `)
+//line app/vmalert/tpl/nav.qtpl:13
+	for _, item := range items {
+//line app/vmalert/tpl/nav.qtpl:13
+		qw422016.N().S(`
+                <li class="nav-item">
+                    <a class="nav-link`)
+//line app/vmalert/tpl/nav.qtpl:15
+		if current == item.Name {
+//line app/vmalert/tpl/nav.qtpl:15
+			qw422016.N().S(` active`)
+//line app/vmalert/tpl/nav.qtpl:15
+		}
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`" href="`)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.E().S(item.Url)
+//line app/vmalert/tpl/nav.qtpl:15
+		qw422016.N().S(`">
+                        `)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.E().S(item.Name)
+//line app/vmalert/tpl/nav.qtpl:16
+		qw422016.N().S(`
+                    </a>
+                </li>
+            `)
+//line app/vmalert/tpl/nav.qtpl:19
+	}
+//line app/vmalert/tpl/nav.qtpl:19
+	qw422016.N().S(`
+        </ul>
+  </div>
+</nav>
+`)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func WritePrintNavItems(qq422016 qtio422016.Writer, current string, items []NavItem) {
+//line app/vmalert/tpl/nav.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	StreamPrintNavItems(qw422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/nav.qtpl:23
+}
+
+//line app/vmalert/tpl/nav.qtpl:23
+func PrintNavItems(current string, items []NavItem) string {
+//line app/vmalert/tpl/nav.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/nav.qtpl:23
+	WritePrintNavItems(qb422016, current, items)
+//line app/vmalert/tpl/nav.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/nav.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/nav.qtpl:23
+	return qs422016
+//line app/vmalert/tpl/nav.qtpl:23
+}

app/vmalert/utils/auth.go

@@ -0,0 +1,18 @@
+package utils
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
+
+// AuthConfig returns promauth.Config based on the given params
+func AuthConfig(baUser, baPass, baFile, bearerToken, bearerTokenFile string) (*promauth.Config, error) {
+	var baCfg *promauth.BasicAuthConfig
+	if baUser != "" || baPass != "" || baFile != "" {
+		baCfg = &promauth.BasicAuthConfig{
+			Username:     baUser,
+			Password:     baPass,
+			PasswordFile: baFile,
+		}
+	}
+	return promauth.NewConfig(".", nil, baCfg, bearerToken, bearerTokenFile, nil, nil)
+}

app/vmalert/web.go

@@ -4,32 +4,63 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
+var (
+	once     = sync.Once{}
+	apiLinks [][2]string
+	navItems []tpl.NavItem
+)
+
+func initLinks() {
+	pathPrefix := httpserver.GetPathPrefix()
+	apiLinks = [][2]string{
+		{path.Join(pathPrefix, "api/v1/groups"), "list all loaded groups and rules"},
+		{path.Join(pathPrefix, "api/v1/alerts"), "list all active alerts"},
+		{path.Join(pathPrefix, "api/v1/groupID/alertID/status"), "get alert status by ID"},
+		{path.Join(pathPrefix, "flags"), "command-line flags"},
+		{path.Join(pathPrefix, "metrics"), "list of application metrics"},
+		{path.Join(pathPrefix, "-/reload"), "reload configuration"},
+	}
+	navItems = []tpl.NavItem{
+		{Name: "vmalert", Url: pathPrefix},
+		{Name: "Groups", Url: path.Join(pathPrefix, "groups")},
+		{Name: "Alerts", Url: path.Join(pathPrefix, "alerts")},
+		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert.html"},
+	}
+}
+
 type requestHandler struct {
 	m *manager
 }
 
 func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+	once.Do(func() {
+		initLinks()
+	})
+
 	switch r.URL.Path {
 	case "/":
 		if r.Method != "GET" {
 			return false
 		}
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"/api/v1/groups", "list all loaded groups and rules"},
-			{"/api/v1/alerts", "list all active alerts"},
-			{"/api/v1/groupID/alertID/status", "get alert status by ID"},
-			{"/metrics", "list of application metrics"},
-			{"/-/reload", "reload configuration"},
-		})
+		WriteWelcome(w)
+		return true
+	case "/alerts":
+		WriteListAlerts(w, rh.groupAlerts())
+		return true
+	case "/groups":
+		WriteListGroups(w, rh.groups())
 		return true
 	case "/api/v1/groups":
 		data, err := rh.listGroups()
@@ -58,14 +89,26 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		if !strings.HasSuffix(r.URL.Path, "/status") {
 			return false
 		}
-		// /api/v1/<groupName>/<alertID>/status
-		data, err := rh.alert(r.URL.Path)
+		alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/"))
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
-		w.Write(data)
+
+		// /api/v1/<groupID>/<alertID>/status
+		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
+			data, err := json.Marshal(alert)
+			if err != nil {
+				httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
+				return true
+			}
+			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			w.Write(data)
+			return true
+		}
+
+		// <groupID>/<alertID>/status
+		WriteAlert(w, alert)
 		return true
 	}
 }
@@ -77,20 +120,25 @@ type listGroupsResponse struct {
 	Status string `json:"status"`
 }
 
-func (rh *requestHandler) listGroups() ([]byte, error) {
+func (rh *requestHandler) groups() []APIGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	lr := listGroupsResponse{Status: "success"}
+	var groups []APIGroup
 	for _, g := range rh.m.groups {
-		lr.Data.Groups = append(lr.Data.Groups, g.toAPI())
+		groups = append(groups, g.toAPI())
 	}
 
 	// sort list of alerts for deterministic output
-	sort.Slice(lr.Data.Groups, func(i, j int) bool {
-		return lr.Data.Groups[i].Name < lr.Data.Groups[j].Name
+	sort.Slice(groups, func(i, j int) bool {
+		return groups[i].Name < groups[j].Name
 	})
 
+	return groups
+}
+func (rh *requestHandler) listGroups() ([]byte, error) {
+	lr := listGroupsResponse{Status: "success"}
+	lr.Data.Groups = rh.groups()
 	b, err := json.Marshal(lr)
 	if err != nil {
 		return nil, &httpserver.ErrorWithStatusCode{
@@ -108,6 +156,30 @@ type listAlertsResponse struct {
 	Status string `json:"status"`
 }
 
+func (rh *requestHandler) groupAlerts() []GroupAlerts {
+	rh.m.groupsMu.RLock()
+	defer rh.m.groupsMu.RUnlock()
+
+	var groupAlerts []GroupAlerts
+	for _, g := range rh.m.groups {
+		var alerts []*APIAlert
+		for _, r := range g.Rules {
+			a, ok := r.(*AlertingRule)
+			if !ok {
+				continue
+			}
+			alerts = append(alerts, a.AlertsAPI()...)
+		}
+		if len(alerts) > 0 {
+			groupAlerts = append(groupAlerts, GroupAlerts{
+				Group:  g.toAPI(),
+				Alerts: alerts,
+			})
+		}
+	}
+	return groupAlerts
+}
+
 func (rh *requestHandler) listAlerts() ([]byte, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
@@ -138,18 +210,17 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	return b, nil
 }
 
-func (rh *requestHandler) alert(path string) ([]byte, error) {
+func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+	parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", 3)
 	if len(parts) != 3 {
 		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path),
 			StatusCode: http.StatusBadRequest,
 		}
 	}
-
 	groupID, err := uint64FromPath(parts[0])
 	if err != nil {
 		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err))
@@ -162,7 +233,7 @@ func (rh *requestHandler) alert(path string) ([]byte, error) {
 	if err != nil {
 		return nil, errResponse(err, http.StatusNotFound)
 	}
-	return json.Marshal(resp)
+	return resp, nil
 }
 
 func uint64FromPath(path string) (uint64, error) {

app/vmalert/web.qtpl

@@ -0,0 +1,305 @@
+{% package main %}
+
+{% import (
+    "time"
+    "sort"
+
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
+) %}
+
+
+{% func Welcome() %}
+    {%= tpl.Header("vmalert", navItems) %}
+    <p>
+        API:<br>
+        {% for _, p := range apiLinks  %}
+            {%code
+                p, doc := p[0], p[1]
+            %}
+        	<a href="{%s p %}">{%s p %}</a> - {%s doc %}<br/>
+        {% endfor %}
+    </p>
+    {%= tpl.Footer() %}
+{% endfunc %}
+
+{% func ListGroups(groups []APIGroup) %}
+    {%= tpl.Header("Groups", navItems) %}
+    {%  if len(groups) > 0 %}
+        {%code
+            rOk := make(map[string]int)
+            rNotOk := make(map[string]int)
+            for _, g := range groups {
+                for _, r := range g.AlertingRules{
+                    if r.LastError != "" {
+                        rNotOk[g.Name]++
+                    } else {
+                        rOk[g.Name]++
+                    }
+                }
+                 for _, r := range g.RecordingRules{
+                    if r.LastError != "" {
+                        rNotOk[g.Name]++
+                    } else {
+                        rOk[g.Name]++
+                    }
+                }
+            }
+        %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+        {% for _, g := range groups  %}
+              <div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
+                <span class="anchor" id="group-{%s g.ID %}"></span>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%s g.Interval %})</a>
+                 {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
+                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
+                <p class="fs-6 fw-lighter">{%s g.File %}</p>
+                {% if len(g.ExtraFilterLabels) > 0 %}
+                    <div class="fs-6 fw-lighter">Extra filter labels
+                    {% for k, v := range g.ExtraFilterLabels %}
+                            <span class="float-left badge bg-primary">{%s k %}={%s v %}</span>
+                    {% endfor %}
+                    </div>
+                {% endif %}
+            </div>
+            <div class="collapse" id="rules-{%s g.ID %}">
+                <table class="table table-striped table-hover table-sm">
+                    <thead>
+                        <tr>
+                            <th scope="col">Rule</th>
+                            <th scope="col" title="Shows if rule's execution ended with error">Error</th>
+                            <th scope="col" title="How many samples were produced by the rule">Samples</th>
+                            <th scope="col" title="How many seconds ago rule was executed">Updated</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                    {% for _, ar := range g.AlertingRules %}
+                        <tr{% if ar.LastError != "" %} class="alert-danger"{% endif %}>
+                            <td>
+                                <b>alert:</b> {%s ar.Name %} (for: {%v ar.For %})<br>
+                                <code><pre>{%s ar.Expression %}</pre></code><br>
+                                {% if len(ar.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                {% for k, v := range ar.Labels %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                {% endfor %}
+                            </td>
+                            <td><div class="error-cell">{%s ar.LastError %}</div></td>
+                            <td>{%d ar.LastSamples %}</td>
+                            <td>{%f.3 time.Since(ar.LastExec).Seconds() %}s ago</td>
+                        </tr>
+                    {% endfor %}
+                    {% for _, rr := range g.RecordingRules  %}
+                        <tr>
+                            <td>
+                                <b>record:</b> {%s rr.Name %}<br>
+                                <code><pre>{%s rr.Expression %}</pre></code>
+                                {% if len(rr.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                {% for k, v := range rr.Labels %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
+                                {% endfor %}
+                            </td>
+                            <td><div class="error-cell">{%s rr.LastError %}</div></td>
+                            <td>{%d rr.LastSamples %}</td>
+                            <td>{%f.3 time.Since(rr.LastExec).Seconds() %}s ago</td>
+                        </tr>
+                    {% endfor %}
+                 </tbody>
+                </table>
+            </div>
+        {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+
+{% func ListAlerts(groupAlerts []GroupAlerts) %}
+    {%= tpl.Header("Alerts", navItems) %}
+    {% if len(groupAlerts) > 0 %}
+         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
+         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
+         {% for _, ga := range groupAlerts %}
+            {%code g := ga.Group %}
+            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}">
+                <span class="anchor" id="group-{%s g.ID %}"></span>
+                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
+                <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
+                <br>
+                <p class="fs-6 fw-lighter">{%s g.File %}</p>
+            </div>
+            {%code
+                var keys []string
+                alertsByRule := make(map[string][]*APIAlert)
+                for _, alert := range ga.Alerts {
+                    if len(alertsByRule[alert.RuleID]) < 1 {
+                        keys = append(keys, alert.RuleID)
+                    }
+                    alertsByRule[alert.RuleID] = append(alertsByRule[alert.RuleID], alert)
+                }
+                sort.Strings(keys)
+            %}
+            <div class="collapse" id="rules-{%s g.ID %}">
+                {% for _, ruleID := range keys %}
+                    {%code
+                        defaultAR := alertsByRule[ruleID][0]
+                        var labelKeys []string
+                        for k := range defaultAR.Labels {
+                            labelKeys = append(labelKeys, k)
+                        }
+                        sort.Strings(labelKeys)
+                    %}
+                    <br>
+                    <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                     | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                    <br>
+                    <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                    <table class="table table-striped table-hover table-sm">
+                        <thead>
+                            <tr>
+                                <th scope="col">Labels</th>
+                                <th scope="col">State</th>
+                                <th scope="col">Active at</th>
+                                <th scope="col">Value</th>
+                                <th scope="col">Link</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                        {% for _, ar := range alertsByRule[ruleID] %}
+                            <tr>
+                                <td>
+                                    {% for _, k := range labelKeys %}
+                                        <span class="ms-1 badge bg-primary">{%s k %}={%s ar.Labels[k] %}</span>
+                                    {% endfor %}
+                                </td>
+                                <td>{%= badgeState(ar.State) %}</td>
+                                <td>
+                                    {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                    {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                </td>
+                                <td>{%s ar.Value %}</td>
+                                <td>
+                                    <a href="/{%s g.ID %}/{%s ar.ID %}/status">Details</a>
+                                </td>
+                            </tr>
+                        {% endfor %}
+                     </tbody>
+                    </table>
+                {% endfor %}
+            </div>
+            <br>
+        {% endfor %}
+
+    {% else %}
+        <div>
+            <p>No items...</p>
+        </div>
+    {% endif %}
+
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func Alert(alert *APIAlert) %}
+    {%= tpl.Header("", navItems) %}
+    {%code
+        var labelKeys []string
+        for k := range alert.Labels {
+            labelKeys = append(labelKeys, k)
+        }
+        sort.Strings(labelKeys)
+
+        var annotationKeys []string
+        for k := range alert.Annotations {
+            annotationKeys = append(annotationKeys, k)
+        }
+        sort.Strings(annotationKeys)
+    %}
+    <div class="display-6 pb-3 mb-3">{%s alert.Name %}<span class="ms-2 badge {% if alert.State=="firing" %}bg-danger{% else %} bg-warning text-dark{% endif %}">{%s alert.State %}</span></div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Active at
+        </div>
+        <div class="col">
+          {%s alert.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+        </div>
+      </div>
+      </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Expr
+        </div>
+        <div class="col">
+          <code><pre>{%s alert.Expression %}</pre></code>
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Labels
+        </div>
+        <div class="col">
+           {% for _, k := range labelKeys %}
+                <span class="m-1 badge bg-primary">{%s k %}={%s alert.Labels[k] %}</span>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Annotations
+        </div>
+        <div class="col">
+           {% for _, k := range annotationKeys %}
+                <b>{%s k %}:</b><br>
+                <p>{%s alert.Annotations[k] %}</p>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Group
+        </div>
+        <div class="col">
+           <a target="_blank" href="/groups#group-{%s alert.GroupID %}">{%s alert.GroupID %}</a>
+        </div>
+      </div>
+    </div>
+     <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Source link
+        </div>
+        <div class="col">
+           <a target="_blank" href="{%s alert.SourceLink %}">Link</a>
+        </div>
+      </div>
+    </div>
+    {%= tpl.Footer() %}
+
+{% endfunc %}
+
+{% func badgeState(state string) %}
+{%code
+    badgeClass := "bg-warning text-dark"
+    if state == "firing" {
+        badgeClass = "bg-danger"
+    }
+%}
+<span class="badge {%s badgeClass %}">{%s state %}</span>
+{% endfunc %}
+
+{% func badgeRestored() %}
+<span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
+{% endfunc %}

app/vmalert/web_types.go

@@ -9,6 +9,7 @@ import (
 type APIAlert struct {
 	ID          string            `json:"id"`
 	Name        string            `json:"name"`
+	RuleID      string            `json:"rule_id"`
 	GroupID     string            `json:"group_id"`
 	Expression  string            `json:"expression"`
 	State       string            `json:"state"`
@@ -16,6 +17,8 @@ type APIAlert struct {
 	Labels      map[string]string `json:"labels"`
 	Annotations map[string]string `json:"annotations"`
 	ActiveAt    time.Time         `json:"activeAt"`
+	SourceLink  string            `json:"source"`
+	Restored    bool              `json:"restored"`
 }
 
 // APIGroup represents Group for WEB view
@@ -27,6 +30,7 @@ type APIGroup struct {
 	Interval          string             `json:"interval"`
 	Concurrency       int                `json:"concurrency"`
 	ExtraFilterLabels map[string]string  `json:"extra_filter_labels"`
+	Labels            map[string]string  `json:"labels,omitempty"`
 	AlertingRules     []APIAlertingRule  `json:"alerting_rules"`
 	RecordingRules    []APIRecordingRule `json:"recording_rules"`
 }
@@ -40,6 +44,7 @@ type APIAlertingRule struct {
 	Expression  string            `json:"expression"`
 	For         string            `json:"for"`
 	LastError   string            `json:"last_error"`
+	LastSamples int               `json:"last_samples"`
 	LastExec    time.Time         `json:"last_exec"`
 	Labels      map[string]string `json:"labels"`
 	Annotations map[string]string `json:"annotations"`
@@ -47,12 +52,19 @@ type APIAlertingRule struct {
 
 // APIRecordingRule represents RecordingRule for WEB view
 type APIRecordingRule struct {
-	ID         string            `json:"id"`
-	Name       string            `json:"name"`
-	Type       string            `json:"type"`
-	GroupID    string            `json:"group_id"`
-	Expression string            `json:"expression"`
-	LastError  string            `json:"last_error"`
-	LastExec   time.Time         `json:"last_exec"`
-	Labels     map[string]string `json:"labels"`
+	ID          string            `json:"id"`
+	Name        string            `json:"name"`
+	Type        string            `json:"type"`
+	GroupID     string            `json:"group_id"`
+	Expression  string            `json:"expression"`
+	LastError   string            `json:"last_error"`
+	LastSamples int               `json:"last_samples"`
+	LastExec    time.Time         `json:"last_exec"`
+	Labels      map[string]string `json:"labels"`
+}
+
+// GroupAlerts represents a group of alerts for WEB view
+type GroupAlerts struct {
+	Group  APIGroup
+	Alerts []*APIAlert
 }

app/vmauth/README.md

@@ -37,9 +37,8 @@ Each `url_prefix` in the [-auth.config](#auth-config) may contain either a singl
 `-auth.config` is represented in the following simple `yml` format:
 
 ```yml
-
 # Arbitrary number of usernames may be put here.
-# Usernames must be unique.
+# Username and bearer_token values must be unique.
 
 users:
   # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
@@ -47,6 +46,14 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
+  # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+- bearer_token: "YYY"
+  url_prefix: "http://localhost:8428"
+  headers:
+  - "X-Scope-OrgID: foobar"
+
   # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # will be proxied to http://localhost:8428 .
@@ -89,7 +96,6 @@ users:
   - "http://vminsert1:8480/insert/42/prometheus"
   - "http://vminsert2:8480/insert/42/prometheus"
 
-
   # A single user for querying and inserting data:
   # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
   #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
@@ -97,7 +103,8 @@ users:
   #     - http://vmselect2:8481/select/42/prometheus
   #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
   #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
-  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
 - username: "foobar"
   url_map:
   - src_paths:
@@ -109,7 +116,8 @@ users:
     - "http://vmselect2:8481/select/42/prometheus"
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
-```
+    headers:
+    - "X-Scope-OrgID: abc"```
 
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
@@ -207,7 +215,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -enableTCP6
     	Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
@@ -230,6 +238,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
     	Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
     	TCP address to listen for http connections (default ":8427")
+  -logInvalidAuthTokens
+    	Whether to log requests with invalid auth tokens. Such requests are always counted at vmagent_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
     	Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int

app/vmauth/auth_config.go

@@ -27,24 +27,53 @@ var (
 
 // AuthConfig represents auth config.
 type AuthConfig struct {
-	Users []UserInfo `yaml:"users"`
+	Users []UserInfo `yaml:"users,omitempty"`
 }
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	BearerToken string     `yaml:"bearer_token"`
-	Username    string     `yaml:"username"`
-	Password    string     `yaml:"password"`
-	URLPrefix   *URLPrefix `yaml:"url_prefix"`
-	URLMap      []URLMap   `yaml:"url_map"`
+	BearerToken string     `yaml:"bearer_token,omitempty"`
+	Username    string     `yaml:"username,omitempty"`
+	Password    string     `yaml:"password,omitempty"`
+	URLPrefix   *URLPrefix `yaml:"url_prefix,omitempty"`
+	URLMap      []URLMap   `yaml:"url_map,omitempty"`
+	Headers     []Header   `yaml:"headers,omitempty"`
 
 	requests *metrics.Counter
 }
 
+// Header is `Name: Value` http header, which must be added to the proxied request.
+type Header struct {
+	Name  string
+	Value string
+}
+
+// UnmarshalYAML unmarshals h from f.
+func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
+	var s string
+	if err := f(&s); err != nil {
+		return err
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
+	}
+	h.Name = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
+}
+
+// MarshalYAML marshals h to yaml.
+func (h *Header) MarshalYAML() (interface{}, error) {
+	s := fmt.Sprintf("%s: %s", h.Name, h.Value)
+	return s, nil
+}
+
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths  []*SrcPath `yaml:"src_paths"`
-	URLPrefix *URLPrefix `yaml:"url_prefix"`
+	SrcPaths  []*SrcPath `yaml:"src_paths,omitempty"`
+	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
+	Headers   []Header   `yaml:"headers,omitempty"`
 }
 
 // SrcPath represents an src path

app/vmauth/auth_config_test.go

@@ -69,6 +69,14 @@ users:
   - [foo]
 `)
 
+	// Invalid headers
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  headers: foobar
+`)
+
 	// empty url_prefix
 	f(`
 users:
@@ -156,6 +164,27 @@ users:
   - src_paths: ['fo[obar']
     url_prefix: http://foobar
 `)
+
+	// Invalid headers in url_map (missing ':')
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+    - foobar
+`)
+	// Invalid headers in url_map (dictionary instead of array)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_paths: ['/foobar']
+    url_prefix: http://foobar
+    headers:
+      aaa: bbb
+`)
 }
 
 func TestParseAuthConfigSuccess(t *testing.T) {
@@ -231,6 +260,9 @@ users:
     url_prefix: http://vmselect/select/0/prometheus
   - src_paths: ["/api/v1/write"]
     url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
+    headers:
+    - "foo: bar"
+    - "xxx: y"
 `, map[string]*UserInfo{
 		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
@@ -245,6 +277,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
 				},
 			},
 		},

app/vmauth/example_config.yml

@@ -1,5 +1,5 @@
 # Arbitrary number of usernames may be put here.
-# Usernames must be unique.
+# Username and bearer_token values must be unique.
 
 users:
   # Requests with the 'Authorization: Bearer XXXX' header are proxied to http://localhost:8428 .
@@ -7,6 +7,14 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
+  # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
+- bearer_token: "YYY"
+  url_prefix: "http://localhost:8428"
+  headers:
+  - "X-Scope-OrgID: foobar"
+
   # The user for querying local single-node VictoriaMetrics.
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # will be proxied to http://localhost:8428 .
@@ -49,7 +57,6 @@ users:
   - "http://vminsert1:8480/insert/42/prometheus"
   - "http://vminsert2:8480/insert/42/prometheus"
 
-
   # A single user for querying and inserting data:
   # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
   #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
@@ -57,7 +64,8 @@ users:
   #     - http://vmselect2:8481/select/42/prometheus
   #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
   #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
-  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
 - username: "foobar"
   url_map:
   - src_paths:
@@ -69,3 +77,5 @@ users:
     - "http://vmselect2:8481/select/42/prometheus"
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
+    headers:
+    - "X-Scope-OrgID: abc"

app/vmauth/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -21,6 +22,8 @@ var (
 	httpListenAddr         = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
 	reloadAuthKey          = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	logInvalidAuthTokens   = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
+		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 )
 
 func main() {
@@ -71,16 +74,25 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	ac := authConfig.Load().(map[string]*UserInfo)
 	ui := ac[authToken]
 	if ui == nil {
-		httpserver.Errorf(w, r, "cannot find the provided auth token %q in config", authToken)
+		invalidAuthTokenRequests.Inc()
+		if *logInvalidAuthTokens {
+			httpserver.Errorf(w, r, "cannot find the provided auth token %q in config", authToken)
+		} else {
+			errStr := fmt.Sprintf("cannot find the provided auth token %q in config", authToken)
+			http.Error(w, errStr, http.StatusBadRequest)
+		}
 		return true
 	}
 	ui.requests.Inc()
-	targetURL, err := createTargetURL(ui, r.URL)
+	targetURL, headers, err := createTargetURL(ui, r.URL)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
 		return true
 	}
 	r.Header.Set("vm-target-url", targetURL.String())
+	for _, h := range headers {
+		r.Header.Set(h.Name, h.Value)
+	}
 	proxyRequest(w, r)
 	return true
 }
@@ -99,7 +111,11 @@ func proxyRequest(w http.ResponseWriter, r *http.Request) {
 	reverseProxy.ServeHTTP(w, r)
 }
 
-var configReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
+var (
+	configReloadRequests     = metrics.NewCounter(`vmauth_http_requests_total{path="/-/reload"}`)
+	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
+	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
+)
 
 var reverseProxy = &httputil.ReverseProxy{
 	Director: func(r *http.Request) {
@@ -117,6 +133,9 @@ var reverseProxy = &httputil.ReverseProxy{
 		// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
 		tr.ForceAttemptHTTP2 = false
 		tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
+		if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+			tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+		}
 		return tr
 	}(),
 	FlushInterval: time.Second,

app/vmauth/target_url.go

@@ -35,22 +35,24 @@ func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	return &targetURL
 }
 
-func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, error) {
+func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, []Header, error) {
 	u := *uOrig
 	// Prevent from attacks with using `..` in r.URL.Path
 	u.Path = path.Clean(u.Path)
 	if !strings.HasPrefix(u.Path, "/") {
 		u.Path = "/" + u.Path
 	}
+	u.Path = strings.TrimSuffix(u.Path, "/")
 	for _, e := range ui.URLMap {
 		for _, sp := range e.SrcPaths {
 			if sp.match(u.Path) {
-				return e.URLPrefix.mergeURLs(&u), nil
+				return e.URLPrefix.mergeURLs(&u), e.Headers, nil
 			}
 		}
 	}
 	if ui.URLPrefix != nil {
-		return ui.URLPrefix.mergeURLs(&u), nil
+		return ui.URLPrefix.mergeURLs(&u), ui.Headers, nil
 	}
-	return nil, fmt.Errorf("missing route for %q", u.String())
+	missingRouteRequests.Inc()
+	return nil, nil, fmt.Errorf("missing route for %q", u.String())
 }

app/vmauth/target_url_test.go

@@ -1,44 +1,56 @@
 package main
 
 import (
+	"fmt"
 	"net/url"
 	"testing"
 )
 
 func TestCreateTargetURLSuccess(t *testing.T) {
-	f := func(ui *UserInfo, requestURI, expectedTarget string) {
+	f := func(ui *UserInfo, requestURI, expectedTarget, expectedHeaders string) {
 		t.Helper()
 		u, err := url.Parse(requestURI)
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, err := createTargetURL(ui, u)
+		target, headers, err := createTargetURL(ui, u)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if target.String() != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
+		headersStr := fmt.Sprintf("%q", headers)
+		if headersStr != expectedHeaders {
+			t.Fatalf("unexpected headers; got %s; want %s", headersStr, expectedHeaders)
+		}
 	}
 	// Simple routing with `url_prefix`
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "", "http://foo.bar/.")
+	}, "", "http://foo.bar/.", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "/", "http://foo.bar/")
+		Headers: []Header{{
+			Name:  "bb",
+			Value: "aaa",
+		}},
+	}, "/", "http://foo.bar", `[{"bb" "aaa"}]`)
+	f(&UserInfo{
+		URLPrefix: mustParseURL("http://foo.bar/federate"),
+	}, "/", "http://foo.bar/federate", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "a/b?c=d", "http://foo.bar/a/b?c=d")
+	}, "a/b?c=d", "http://foo.bar/a/b?c=d", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/z", "https://sss:3894/x/y/z")
+	}, "/z", "https://sss:3894/x/y/z", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/../../aaa", "https://sss:3894/x/y/aaa")
+	}, "/../../aaa", "https://sss:3894/x/y/aaa", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd")
+	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "[]")
 
 	// Complex routing with `url_map`
 	ui := &UserInfo{
@@ -46,6 +58,16 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
+				Headers: []Header{
+					{
+						Name:  "xx",
+						Value: "aa",
+					},
+					{
+						Name:  "yy",
+						Value: "asdf",
+					},
+				},
 			},
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
@@ -53,10 +75,14 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
+		Headers: []Header{{
+			Name:  "bb",
+			Value: "aaa",
+		}},
 	}
-	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up")
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write")
-	f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range")
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`)
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]")
+	f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`)
 
 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{
@@ -72,17 +98,17 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		},
 		URLPrefix: mustParseURL("http://default-server"),
 	}
-	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up")
-	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up")
-	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values")
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write")
-	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar")
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "[]")
+	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "[]")
+	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]")
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]")
+	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"),
-	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev")
+	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]")
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=mobile"),
-	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile")
+	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "[]")
 
 }
 
@@ -93,13 +119,16 @@ func TestCreateTargetURLFailure(t *testing.T) {
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
-		target, err := createTargetURL(ui, u)
+		target, headers, err := createTargetURL(ui, u)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
 		if target != nil {
 			t.Fatalf("unexpected target=%q; want empty string", target)
 		}
+		if headers != nil {
+			t.Fatalf("unexpected headers=%q; want empty string", headers)
+		}
 	}
 	f(&UserInfo{}, "/foo/bar")
 	f(&UserInfo{

app/vmbackup/README.md

@@ -4,7 +4,7 @@
 
 Supported storage systems for backups:
 
-* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
+* [GCS](https://cloud.google.com/storage/). Example: `gs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
 * Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`
@@ -19,7 +19,7 @@ Backed up data can be restored with [vmrestore](https://docs.victoriametrics.com
 
 See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
 
-See also [vmbackupmanager](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) tool built on top of `vmbackup`. This tool simplifies
+See also [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html) tool built on top of `vmbackup`. This tool simplifies
 creation of hourly, daily, weekly and monthly backups.
 
 
@@ -30,7 +30,7 @@ creation of hourly, daily, weekly and monthly backups.
 Regular backup can be performed with the following command:
 
 ```
-vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup>
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup>
 ```
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
@@ -46,7 +46,7 @@ If the destination GCS bucket already contains the previous backup at `-origin`
 with the following command:
 
 ```
-vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
 ```
 
 It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
@@ -58,7 +58,7 @@ Incremental backups performed if `-dst` points to an already existing backup. In
 It saves time and network bandwidth costs when working with big backups:
 
 ```
-vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/existing/backup>
 ```
 
 
@@ -69,16 +69,16 @@ Smart backups mean storing full daily backups into `YYYYMMDD` folders and creati
 * Run the following command every hour:
 
 ```
-vmbackup -snapshotName=<latest-snapshot> -dst=gcs://<bucket>/latest
+vmbackup -snapshotName=<latest-snapshot> -dst=gs://<bucket>/latest
 ```
 
 Where `<latest-snapshot>` is the latest [snapshot](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
-The command will upload only changed data to `gcs://<bucket>/latest`.
+The command will upload only changed data to `gs://<bucket>/latest`.
 
 * Run the following command once a day:
 
 ```
-vmbackup -snapshotName=<daily-snapshot> -dst=gcs://<bucket>/<YYYYMMDD> -origin=gcs://<bucket>/latest
+vmbackup -snapshotName=<daily-snapshot> -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
 ```
 
 Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
@@ -89,7 +89,7 @@ or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when
 
 Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
 
-See also [vmbackupmanager tool](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/466) for automating smart backups.
+See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager.html) for automating smart backups.
 
 
 ## How does it work?
@@ -183,10 +183,10 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
   -customS3Endpoint string
     	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
   -dst string
-    	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
     	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap

app/vmbackup/main.go

@@ -25,7 +25,7 @@ var (
 	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+
 		"All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete")
 	dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+
-		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
+		"Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
 		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
 	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
 	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")

app/vmbackupmanager/README.md

@@ -1,6 +1,6 @@
 ## vmbackupmanager
 
-***vmbackupmanager is a part of [enterprise package](https://victoriametrics.com/enterprise.html)***
+***vmbackupmanager is a part of [enterprise package](https://victoriametrics.com/enterprise.html). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
 
 The VictoriaMetrics backup manager automates regular backup procedures. It supports the following backup intervals: **hourly**, **daily**, **weekly** and **monthly**. Multiple backup intervals may be configured simultaneously. I.e. the backup manager creates hourly backups every hour, while it creates daily backups every day, etc. Backup manager must have read access to the storage data, so best practice is to install it on the same machine (or as a sidecar) where the storage node is installed.
 The backup service makes a backup every hour and puts it to the latest folder and then copies data to the folders which represent the backup intervals (hourly, daily, weekly and monthly)
@@ -76,7 +76,7 @@ Backup manager launched with the following configuration:
 ```console
 export NODE_IP=192.168.0.10
 export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
-./vmbackupmanager -dst=gcs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula 
+./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula 
 ```
 
 Expected logs in vmbackupmanager:
@@ -125,7 +125,7 @@ We enable backup retention policy for backup manager by using following configur
 ```console
 export NODE_IP=192.168.0.10
 export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
-./vmbackupmanager -dst=gcs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create
+./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create
 -keepLastDaily=3 -eula
 ```
 

app/vmctl/README.md

@@ -10,56 +10,63 @@ Features:
 - [x] OpenTSDB: migrate data from OpenTSDB to VictoriaMetrics
 - [ ] Storage Management: data re-balancing between nodes 
 
+vmctl acts as a proxy between data source ([Prometheus](#migrating-data-from-prometheus), 
+[InfluxDB](#migrating-data-from-influxdb-1x), [VictoriaMetrics](##migrating-data-from-victoriametrics), etc.)
+and destination - VictoriaMetrics single or cluster version. To see the full list of supported modes 
+run the following command:
+```
+./vmctl --help                                        
+NAME:
+   vmctl - VictoriaMetrics command-line tool
+
+USAGE:
+   vmctl [global options] command [command options] [arguments...]
+
+COMMANDS:
+   opentsdb    Migrate timeseries from OpenTSDB
+   influx      Migrate timeseries from InfluxDB
+   prometheus  Migrate timeseries from Prometheus
+   vm-native   Migrate time series between VictoriaMetrics installations via native binary format
+```
+
+Each mode has its own unique set of flags specific (e.g. prefixed with `influx` for influx mode)
+to the data source and common list of flags for destination (prefixed with `vm` for VictoriaMetrics):
+```
+./vmctl influx --help
+OPTIONS:
+   --influx-addr value              InfluxDB server addr (default: "http://localhost:8086")
+   --influx-user value              InfluxDB user [$INFLUX_USERNAME]
+...
+   --vm-addr vmctl                             VictoriaMetrics address to perform import requests. 
+Should be the same as --httpListenAddr value for single-node version or vminsert component. 
+When importing into the clustered version do not forget to set additionally --vm-account-id flag. 
+Please note, that vmctl performs initial readiness check for the given address by checking `/health` endpoint. (default: "http://localhost:8428")
+   --vm-user value        VictoriaMetrics username for basic auth [$VM_USERNAME]
+   --vm-password value    VictoriaMetrics password for basic auth [$VM_PASSWORD]
+```
+
+When doing a migration user needs to specify flags for source (where and how to fetch data) and for
+destination (where to migrate data). Every mode has additional details and nuances, please see 
+them below in corresponding sections.
+
+For the destination flags see the full description by running the following command:
+```
+./vmctl influx --help | grep vm-
+```
+
+Some flags like [--vm-extra-label](#adding-extra-labels) or [--vm-significant-figures](#significant-figures)
+has additional sections with description below. Details about tweaking and adjusting settings
+are explained in [Tuning](#tuning) section.
+
+Please note, that if you're going to import data into VictoriaMetrics cluster do not
+forget to specify the `--vm-account-id` flag. See more details for cluster version 
+[here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+
 ## Articles
 
 * [How to migrate data from Prometheus](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
 * [How to migrate data from Prometheus. Filtering and modifying time series](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
 
-## How to build
-
-It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmctl` is located in `vmutils-*` archives there.
-
-
-### Development build
-
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
-2. Run `make vmctl` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmctl` binary and puts it into the `bin` folder.
-
-### Production build
-
-1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmctl-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmctl-prod` binary and puts it into the `bin` folder.
-
-### Building docker images
-
-Run `make package-vmctl`. It builds `victoriametrics/vmctl:<PKG_TAG>` docker image locally.
-`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
-The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmctl`.
-
-The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
-by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
-
-```bash
-ROOT_IMAGE=scratch make package-vmctl
-```
-
-### ARM build
-
-ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
-
-#### Development ARM build
-
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
-2. Run `make vmctl-arm` or `make vmctl-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmctl-arm` or `vmctl-arm64` binary respectively and puts it into the `bin` folder.
-
-#### Production ARM build
-
-1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmctl-arm-prod` or `make vmctl-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
-   It builds `vmctl-arm-prod` or `vmctl-arm64-prod` binary respectively and puts it into the `bin` folder.
 
 ## Migrating data from OpenTSDB
 
@@ -209,16 +216,16 @@ Found 40000 timeseries to import. Continue? [Y/n] y
 
 ### Data mapping
 
-Vmctl maps Influx data the same way as VictoriaMetrics does by using the following rules:
+Vmctl maps InfluxDB data the same way as VictoriaMetrics does by using the following rules:
 
-* `influx-database` arg is mapped into `db` label value unless `db` tag exists in the Influx line.
+* `influx-database` arg is mapped into `db` label value unless `db` tag exists in the InfluxDB line.
 * Field names are mapped to time series names prefixed with {measurement}{separator} value, 
 where {separator} equals to _ by default. 
 It can be changed with `--influx-measurement-field-separator` command-line flag.
 * Field values are mapped to time series values.
 * Tags are mapped to Prometheus labels format as-is.
 
-For example, the following Influx line:
+For example, the following InfluxDB line:
 ```
 foo,tag1=value1,tag2=value2 field1=12,field2=40
 ```
@@ -287,7 +294,7 @@ if flags `--prom-filter-time-start` or `--prom-filter-time-end` were set. The ex
 Please note that stats are not taking into account timeseries or samples filtering. This will be done during importing process.
  
 The importing process takes the snapshot blocks revealed from Explore procedure and processes them one by one
-accumulating timeseries and samples. Please note, that `vmctl` relies on responses from Influx on this stage,
+accumulating timeseries and samples. Please note, that `vmctl` relies on responses from InfluxDB on this stage,
 so ensure that Explore queries are executed without errors or limits. Please see this 
 [issue](https://github.com/VictoriaMetrics/vmctl/issues/30) for details.
 The data processed in chunks and then sent to VM.
@@ -472,11 +479,12 @@ To avoid such situation try to filter out VM process metrics via `--vm-native-fi
 [Backfilling tips](https://github.com/VictoriaMetrics/VictoriaMetrics#backfilling) section.
 3. `vmctl` doesn't provide relabeling or other types of labels management in this mode.
 Instead, use [relabeling in VictoriaMetrics](https://github.com/VictoriaMetrics/vmctl/issues/4#issuecomment-683424375).
-
+4. When importing in or from cluster version remember to use correct [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format)
+and specify `accountID` param.
 
 ## Tuning
 
-### Influx mode
+### InfluxDB mode
 
 The flag `--influx-concurrency` controls how many concurrent requests may be sent to InfluxDB while fetching
 timeseries. Please set it wisely to avoid InfluxDB overwhelming.
@@ -551,4 +559,49 @@ results such as `average`, `rate`, etc.
  `vmctl` allows to add extra labels to all imported series. It can be achived with flag `--vm-extra-label label=value`. 
  If multiple labels needs to be added, set flag for each label, for example, `--vm-extra-label label1=value1 --vm-extra-label label2=value2`.
  If timeseries already have label, that must be added with `--vm-extra-label` flag, flag has priority and will override label value from timeseries.
- 
+
+## How to build
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmctl` is located in `vmutils-*` archives there.
+
+
+### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+2. Run `make vmctl` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl` binary and puts it into the `bin` folder.
+
+### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmctl-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl-prod` binary and puts it into the `bin` folder.
+
+### Building docker images
+
+Run `make package-vmctl`. It builds `victoriametrics/vmctl:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmctl`.
+
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
+
+```bash
+ROOT_IMAGE=scratch make package-vmctl
+```
+
+### ARM build
+
+ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
+
+#### Development ARM build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.16.
+2. Run `make vmctl-arm` or `make vmctl-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl-arm` or `vmctl-arm64` binary respectively and puts it into the `bin` folder.
+
+#### Production ARM build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmctl-arm-prod` or `make vmctl-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl-arm-prod` or `vmctl-arm64-prod` binary respectively and puts it into the `bin` folder.

app/vmctl/flags.go

@@ -40,6 +40,7 @@ var (
 			Value: "http://localhost:8428",
 			Usage: "VictoriaMetrics address to perform import requests. \n" +
 				"Should be the same as --httpListenAddr value for single-node version or vminsert component. \n" +
+				"When importing into the clustered version do not forget to set additionally --vm-account-id flag. \n" +
 				"Please note, that `vmctl` performs initial readiness check for the given address by checking `/health` endpoint.",
 		},
 		&cli.StringFlag{
@@ -55,6 +56,7 @@ var (
 		&cli.StringFlag{
 			Name: vmAccountID,
 			Usage: "AccountID is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). \n" +
+				"AccountID is required when importing into the clustered version of VictoriaMetrics. \n" +
 				"It is possible to set it as accountID:projectID, where projectID is also arbitrary 32-bit integer. \n" +
 				"If projectID isn't set, then it equals to 0",
 		},
@@ -187,26 +189,26 @@ var (
 		&cli.StringFlag{
 			Name:  influxAddr,
 			Value: "http://localhost:8086",
-			Usage: "Influx server addr",
+			Usage: "InfluxDB server addr",
 		},
 		&cli.StringFlag{
 			Name:    influxUser,
-			Usage:   "Influx user",
+			Usage:   "InfluxDB user",
 			EnvVars: []string{"INFLUX_USERNAME"},
 		},
 		&cli.StringFlag{
 			Name:    influxPassword,
-			Usage:   "Influx user password",
+			Usage:   "InfluxDB user password",
 			EnvVars: []string{"INFLUX_PASSWORD"},
 		},
 		&cli.StringFlag{
 			Name:     influxDB,
-			Usage:    "Influx database",
+			Usage:    "InfluxDB database",
 			Required: true,
 		},
 		&cli.StringFlag{
 			Name:  influxRetention,
-			Usage: "Influx retention policy",
+			Usage: "InfluxDB retention policy",
 			Value: "autogen",
 		},
 		&cli.IntFlag{
@@ -221,7 +223,7 @@ var (
 		},
 		&cli.StringFlag{
 			Name: influxFilterSeries,
-			Usage: "Influx filter expression to select series. E.g. \"from cpu where arch='x86' AND hostname='host_2753'\".\n" +
+			Usage: "InfluxDB filter expression to select series. E.g. \"from cpu where arch='x86' AND hostname='host_2753'\".\n" +
 				"See for details https://docs.influxdata.com/influxdb/v1.7/query_language/schema_exploration#show-series",
 		},
 		&cli.StringFlag{

app/vmctl/influx/parser.go

@@ -58,6 +58,11 @@ func toFloat64(v interface{}) (float64, error) {
 		return float64(i), nil
 	case string:
 		return strconv.ParseFloat(i, 64)
+	case bool:
+		if i {
+			return 1, nil
+		}
+		return 0, nil
 	default:
 		return 0, fmt.Errorf("unexpected value type %v", i)
 	}

app/vmctl/influx/parser_test.go

@@ -1,6 +1,7 @@
 package influx
 
 import (
+	"encoding/json"
 	"reflect"
 	"testing"
 )
@@ -58,3 +59,28 @@ func TestSeries_Unmarshal(t *testing.T) {
 		}
 	}
 }
+
+func TestToFloat64(t *testing.T) {
+	f := func(in interface{}, want float64) {
+		t.Helper()
+		got, err := toFloat64(in)
+		if err != nil {
+			t.Fatalf("unexpected err: %s", err)
+		}
+		if got != want {
+			t.Errorf("got %v; want %v", got, want)
+		}
+	}
+	f("123.4", 123.4)
+	f(float64(123.4), 123.4)
+	f(float32(12), 12)
+	f(123, 123)
+	f(true, 1)
+	f(false, 0)
+	f(json.Number("123456.789"), 123456.789)
+
+	_, err := toFloat64("text")
+	if err == nil {
+		t.Fatalf("expected to get err; got nil instead")
+	}
+}

app/vmgateway/README.md

@@ -1,6 +1,6 @@
 # vmgateway
 
-***vmgateway is a part of [enterprise package](https://victoriametrics.com/enterprise.html)***
+***vmgateway is a part of [enterprise package](https://victoriametrics.com/enterprise.html). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)***
 
 
 <img alt="vmgateway" src="vmgateway-overview.jpeg">

app/vminsert/datadog/request_handler.go

@@ -0,0 +1,97 @@
+package datadog
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="datadog"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="datadog"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		ce := req.Header.Get("Content-Encoding")
+		err := parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+			return insertRows(series, extraLabels)
+		})
+		if err != nil {
+			return fmt.Errorf("headers: %q; err: %w", req.Header, err)
+		}
+		return nil
+	})
+}
+
+func insertRows(series []parser.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsLen := 0
+	for i := range series {
+		rowsLen += len(series[i].Points)
+	}
+	ctx.Reset(rowsLen)
+	rowsTotal := 0
+	hasRelabeling := relabel.HasRelabeling()
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", ss.Metric)
+		ctx.AddLabel("host", ss.Host)
+		for _, tag := range ss.Tags {
+			n := strings.IndexByte(tag, ':')
+			if n < 0 {
+				return fmt.Errorf("cannot find ':' in tag %q", tag)
+			}
+			name := tag[:n]
+			value := tag[n+1:]
+			if name == "host" {
+				name = "exported_host"
+			}
+			ctx.AddLabel(name, value)
+		}
+		for j := range extraLabels {
+			label := &extraLabels[j]
+			ctx.AddLabel(label.Name, label.Value)
+		}
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		ctx.SortLabelsIfNeeded()
+		var metricNameRaw []byte
+		var err error
+		for _, pt := range ss.Points {
+			timestamp := pt.Timestamp()
+			value := pt.Value()
+			metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, timestamp, value)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vminsert/influx/request_handler.go

@@ -20,8 +20,8 @@ import (
 )
 
 var (
-	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 )
 

app/vminsert/main.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/native"
@@ -35,7 +36,7 @@ import (
 
 var (
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
-	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8189 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
@@ -155,6 +156,36 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "/datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/intake/":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, `{}`)
+		return true
 	case "/prometheus/targets", "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
@@ -165,6 +196,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		state := r.FormValue("state")
 		promscrape.WriteAPIV1Targets(w, state)
 		return true
+	case "/prometheus/config", "/config":
+		promscrapeConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		promscrape.WriteConfigData(w)
+		return true
 	case "/prometheus/-/reload", "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
@@ -204,14 +240,23 @@ var (
 	nativeimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 	nativeimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
 
-	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
-	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
+	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/influx/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/query", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/influx/query", protocol="influx"}`)
+
+	datadogWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogValidateRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
+	datadogCheckRunRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vm_http_requests_total{path="/datadog/intake/", protocol="datadog"}`)
 
 	promscrapeTargetsRequests      = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
 	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
 
+	promscrapeConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/config"}`)
+
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
 
 	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {

app/vminsert/promremotewrite/request_handler.go

@@ -25,7 +25,7 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(tss []prompb.TimeSeries) error {
+		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
 			return insertRows(tss, extraLabels)
 		})
 	})

app/vminsert/relabel/relabel.go

@@ -16,7 +16,7 @@ import (
 
 var (
 	relabelConfig = flag.String("relabelConfig", "", "Optional path to a file with relabeling rules, which are applied to all the ingested metrics. "+
-		"See https://docs.victoriametrics.com/#relabeling for details")
+		"See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal")
 	relabelDebug = flag.Bool("relabelDebug", false, "Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, "+
 		"then the metrics aren't sent to storage. This is useful for debugging the relabeling configs")
 )

app/vminsert/vmimport/request_handler.go

@@ -30,7 +30,8 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
 			return insertRows(rows, extraLabels)
 		})
 	})

app/vmrestore/README.md

@@ -12,7 +12,7 @@ when restarting `vmrestore` with the same args.
 VictoriaMetrics must be stopped during the restore process.
 
 ```
-vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
+vmrestore -src=gs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
 
 ```
 
@@ -86,7 +86,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
   -customS3Endpoint string
     	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
   -envflag.enable
-    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
     	Prefix for environment variables if -envflag.enable is set
   -fs.disableMmap
@@ -116,7 +116,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
   -skipBackupCompleteCheck
     	Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file
   -src string
-    	Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	Source path with backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
   -storageDataPath string
     	Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir is synchronized with -src contents, i.e. it works like 'rsync --delete' (default "victoria-metrics-data")
   -version

app/vmrestore/main.go

@@ -16,7 +16,7 @@ import (
 
 var (
 	src = flag.String("src", "", "Source path with backup on the remote storage. "+
-		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
+		"Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
 	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
 		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir "+
 		"is synchronized with -src contents, i.e. it works like 'rsync --delete'")

app/vmselect/main.go

@@ -85,12 +85,6 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
 
 // RequestHandler handles remote read API requests
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	// vmui access.
-	if strings.HasPrefix(r.URL.Path, "/vmui") {
-		vmuiFileServer.ServeHTTP(w, r)
-		return true
-	}
-
 	startTime := time.Now()
 	defer requestDuration.UpdateDuration(startTime)
 
@@ -153,11 +147,32 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	//
 	// See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
 	switch {
-	case strings.HasPrefix(path, "/prometheus"):
+	case strings.HasPrefix(path, "/prometheus/"):
 		path = path[len("/prometheus"):]
-	case strings.HasPrefix(path, "/graphite"):
+	case strings.HasPrefix(path, "/graphite/"):
 		path = path[len("/graphite"):]
 	}
+	// vmui access.
+	if strings.HasPrefix(path, "/vmui") {
+		r.URL.Path = path
+		vmuiFileServer.ServeHTTP(w, r)
+		return true
+	}
+	if path == "/graph" {
+		// Redirect to /graph/, otherwise vmui redirects to /vmui/, which can be inaccessible in user env.
+		// Use relative redirect, since, since the hostname and path prefix may be incorrect if VictoriaMetrics
+		// is hidden behind vmauth or similar proxy.
+		_ = r.ParseForm()
+		newURL := "graph/?" + r.Form.Encode()
+		http.Redirect(w, r, newURL, http.StatusFound)
+		return true
+	}
+	if strings.HasPrefix(path, "/graph/") {
+		// This is needed for serving /graph URLs from Prometheus datasource in Grafana.
+		r.URL.Path = strings.Replace(path, "/graph/", "/vmui/", 1)
+		vmuiFileServer.ServeHTTP(w, r)
+		return true
+	}
 
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := path[len("/api/v1/label/"):]
@@ -183,7 +198,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	}
-
+	if strings.HasPrefix(path, "/functions") {
+		graphiteFunctionsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, "%s", `{}`)
+		return true
+	}
 	switch path {
 	case "/api/v1/query":
 		queryRequests.Inc()
@@ -381,14 +401,14 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
-	case "/api/v1/rules":
+	case "/api/v1/rules", "/rules":
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
 		rulesRequests.Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
 		return true
-	case "/api/v1/alerts":
-		// Return dumb placehloder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
+	case "/api/v1/alerts", "/alerts":
+		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
 		alertsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
@@ -529,6 +549,8 @@ var (
 	graphiteTagsDelSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/delSeries"}`)
 	graphiteTagsDelSeriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/delSeries"}`)
 
+	graphiteFunctionsRequests = metrics.NewCounter(`vm_http_request_total{path="/functions"}`)
+
 	rulesRequests          = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
 	alertsRequests         = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
 	metadataRequests       = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)

app/vmselect/netstorage/netstorage.go

@@ -19,13 +19,16 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastrand"
 )
 
 var (
 	maxTagKeysPerSearch          = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned from /api/v1/labels")
 	maxTagValuesPerSearch        = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned from /api/v1/label/<label_name>/values")
 	maxTagValueSuffixesPerSearch = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
-	maxMetricsPerSearch          = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series each search can scan")
+	maxMetricsPerSearch          = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series each search can scan. This option allows limiting memory usage")
+	maxSamplesPerSeries          = flag.Int("search.maxSamplesPerSeries", 30e6, "The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage")
+	maxSamplesPerQuery           = flag.Int("search.maxSamplesPerQuery", 1e9, "The maximum number of raw samples a single query can process across all time series. This protects from heavy queries, which select unexpectedly high number of raw samples. See also -search.maxSamplesPerSeries")
 )
 
 // Result is a single timeseries result.
@@ -38,17 +41,12 @@ type Result struct {
 	// Values are sorted by Timestamps.
 	Values     []float64
 	Timestamps []int64
-
-	// Marshaled MetricName. Used only for results sorting
-	// in app/vmselect/promql
-	MetricNameMarshaled []byte
 }
 
 func (r *Result) reset() {
 	r.MetricName.Reset()
 	r.Values = r.Values[:0]
 	r.Timestamps = r.Timestamps[:0]
-	r.MetricNameMarshaled = r.MetricNameMarshaled[:0]
 }
 
 // Results holds results returned from ProcessSearchQuery.
@@ -79,8 +77,6 @@ func (rss *Results) mustClose() {
 	rss.tbf = nil
 }
 
-var timeseriesWorkCh = make(chan *timeseriesWork, gomaxprocs*16)
-
 type timeseriesWork struct {
 	mustStop *uint32
 	rss      *Results
@@ -119,49 +115,77 @@ func putTimeseriesWork(tsw *timeseriesWork) {
 
 var tswPool sync.Pool
 
-func init() {
-	for i := 0; i < gomaxprocs; i++ {
-		go timeseriesWorker(uint(i))
+func scheduleTimeseriesWork(workChs []chan *timeseriesWork, tsw *timeseriesWork) {
+	if len(workChs) == 1 {
+		// Fast path for a single worker
+		workChs[0] <- tsw
+		return
+	}
+	attempts := 0
+	for {
+		idx := fastrand.Uint32n(uint32(len(workChs)))
+		select {
+		case workChs[idx] <- tsw:
+			return
+		default:
+			attempts++
+			if attempts >= len(workChs) {
+				workChs[idx] <- tsw
+				return
+			}
+		}
 	}
 }
 
-func timeseriesWorker(workerID uint) {
-	var rs Result
-	var rsLastResetTime uint64
-	for tsw := range timeseriesWorkCh {
-		if atomic.LoadUint32(tsw.mustStop) != 0 {
-			tsw.doneCh <- nil
-			continue
-		}
-		rss := tsw.rss
-		if rss.deadline.Exceeded() {
+func (tsw *timeseriesWork) do(r *Result, workerID uint) error {
+	if atomic.LoadUint32(tsw.mustStop) != 0 {
+		return nil
+	}
+	rss := tsw.rss
+	if rss.deadline.Exceeded() {
+		atomic.StoreUint32(tsw.mustStop, 1)
+		return fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
+	}
+	if err := tsw.pts.Unpack(r, rss.tbf, rss.tr, rss.fetchData); err != nil {
+		atomic.StoreUint32(tsw.mustStop, 1)
+		return fmt.Errorf("error during time series unpacking: %w", err)
+	}
+	if len(r.Timestamps) > 0 || !rss.fetchData {
+		if err := tsw.f(r, workerID); err != nil {
 			atomic.StoreUint32(tsw.mustStop, 1)
-			tsw.doneCh <- fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
-			continue
-		}
-		if err := tsw.pts.Unpack(&rs, rss.tbf, rss.tr, rss.fetchData); err != nil {
-			atomic.StoreUint32(tsw.mustStop, 1)
-			tsw.doneCh <- fmt.Errorf("error during time series unpacking: %w", err)
-			continue
-		}
-		if len(rs.Timestamps) > 0 || !rss.fetchData {
-			if err := tsw.f(&rs, workerID); err != nil {
-				atomic.StoreUint32(tsw.mustStop, 1)
-				tsw.doneCh <- err
-				continue
-			}
-		}
-		tsw.rowsProcessed = len(rs.Values)
-		tsw.doneCh <- nil
-		currentTime := fasttime.UnixTimestamp()
-		if cap(rs.Values) > 1024*1024 && 4*len(rs.Values) < cap(rs.Values) && currentTime-rsLastResetTime > 10 {
-			// Reset rs in order to preseve memory usage after processing big time series with millions of rows.
-			rs = Result{}
-			rsLastResetTime = currentTime
+			return err
 		}
 	}
+	tsw.rowsProcessed = len(r.Values)
+	return nil
 }
 
+func timeseriesWorker(ch <-chan *timeseriesWork, workerID uint) {
+	v := resultPool.Get()
+	if v == nil {
+		v = &result{}
+	}
+	r := v.(*result)
+	for tsw := range ch {
+		err := tsw.do(&r.rs, workerID)
+		tsw.doneCh <- err
+	}
+	currentTime := fasttime.UnixTimestamp()
+	if cap(r.rs.Values) > 1024*1024 && 4*len(r.rs.Values) < cap(r.rs.Values) && currentTime-r.lastResetTime > 10 {
+		// Reset r.rs in order to preseve memory usage after processing big time series with millions of rows.
+		r.rs = Result{}
+		r.lastResetTime = currentTime
+	}
+	resultPool.Put(r)
+}
+
+type result struct {
+	rs            Result
+	lastResetTime uint64
+}
+
+var resultPool sync.Pool
+
 // RunParallel runs f in parallel for all the results from rss.
 //
 // f shouldn't hold references to rs after returning.
@@ -172,6 +196,30 @@ func timeseriesWorker(workerID uint) {
 func (rss *Results) RunParallel(f func(rs *Result, workerID uint) error) error {
 	defer rss.mustClose()
 
+	// Spin up local workers.
+	//
+	// Do not use a global workChs with a global pool of workers, since it may lead to a deadlock in the following case:
+	// - RunParallel is called with f, which blocks without forward progress.
+	// - All the workers in the global pool became blocked in f.
+	// - workChs is filled up, so it cannot accept new work items from other RunParallel calls.
+	workers := len(rss.packedTimeseries)
+	if workers > gomaxprocs {
+		workers = gomaxprocs
+	}
+	if workers < 1 {
+		workers = 1
+	}
+	workChs := make([]chan *timeseriesWork, workers)
+	var workChsWG sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		workChs[i] = make(chan *timeseriesWork, 16)
+		workChsWG.Add(1)
+		go func(workerID int) {
+			defer workChsWG.Done()
+			timeseriesWorker(workChs[workerID], uint(workerID))
+		}(i)
+	}
+
 	// Feed workers with work.
 	tsws := make([]*timeseriesWork, len(rss.packedTimeseries))
 	var mustStop uint32
@@ -181,7 +229,7 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint) error) error {
 		tsw.pts = &rss.packedTimeseries[i]
 		tsw.f = f
 		tsw.mustStop = &mustStop
-		timeseriesWorkCh <- tsw
+		scheduleTimeseriesWork(workChs, tsw)
 		tsws[i] = tsw
 	}
 	seriesProcessedTotal := len(rss.packedTimeseries)
@@ -201,6 +249,13 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint) error) error {
 
 	perQueryRowsProcessed.Update(float64(rowsProcessedTotal))
 	perQuerySeriesProcessed.Update(float64(seriesProcessedTotal))
+
+	// Shut down local workers
+	for _, workCh := range workChs {
+		close(workCh)
+	}
+	workChsWG.Wait()
+
 	return firstErr
 }
 
@@ -214,8 +269,6 @@ type packedTimeseries struct {
 	brs        []blockRef
 }
 
-var unpackWorkCh = make(chan *unpackWork, gomaxprocs*128)
-
 type unpackWorkItem struct {
 	br blockRef
 	tr storage.TimeRange
@@ -277,23 +330,49 @@ func putUnpackWork(upw *unpackWork) {
 
 var unpackWorkPool sync.Pool
 
-func init() {
-	for i := 0; i < gomaxprocs; i++ {
-		go unpackWorker()
+func scheduleUnpackWork(workChs []chan *unpackWork, uw *unpackWork) {
+	if len(workChs) == 1 {
+		// Fast path for a single worker
+		workChs[0] <- uw
+		return
+	}
+	attempts := 0
+	for {
+		idx := fastrand.Uint32n(uint32(len(workChs)))
+		select {
+		case workChs[idx] <- uw:
+			return
+		default:
+			attempts++
+			if attempts >= len(workChs) {
+				workChs[idx] <- uw
+				return
+			}
+		}
 	}
 }
 
-func unpackWorker() {
-	var tmpBlock storage.Block
-	for upw := range unpackWorkCh {
-		upw.unpack(&tmpBlock)
+func unpackWorker(ch <-chan *unpackWork) {
+	v := tmpBlockPool.Get()
+	if v == nil {
+		v = &storage.Block{}
 	}
+	tmpBlock := v.(*storage.Block)
+	for upw := range ch {
+		upw.unpack(tmpBlock)
+	}
+	tmpBlockPool.Put(v)
 }
 
+var tmpBlockPool sync.Pool
+
 // unpackBatchSize is the maximum number of blocks that may be unpacked at once by a single goroutine.
 //
-// This batch is needed in order to reduce contention for upackWorkCh in multi-CPU system.
-var unpackBatchSize = 32 * cgroup.AvailableCPUs()
+// It is better to load a single goroutine for up to one second on a system with many CPU cores
+// in order to reduce inter-CPU memory ping-pong.
+// A single goroutine can unpack up to 40 millions of rows per second, while a single block contains up to 8K rows.
+// So the batch size should be 40M / 8K = 5K.
+var unpackBatchSize = 5000
 
 // Unpack unpacks pts to dst.
 func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.TimeRange, fetchData bool) error {
@@ -306,14 +385,39 @@ func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.
 		return nil
 	}
 
-	// Feed workers with work
+	// Spin up local workers.
+	// Do not use global workers pool, since it increases inter-CPU memory ping-poing,
+	// which reduces the scalability on systems with many CPU cores.
 	brsLen := len(pts.brs)
+	workers := brsLen / unpackBatchSize
+	if workers > gomaxprocs {
+		workers = gomaxprocs
+	}
+	if workers < 1 {
+		workers = 1
+	}
+	workChs := make([]chan *unpackWork, workers)
+	var workChsWG sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		// Use unbuffered channel on purpose, since there are high chances
+		// that only a single unpackWork is needed to unpack.
+		// The unbuffered channel should reduce inter-CPU ping-pong in this case,
+		// which should improve the performance in a system with many CPU cores.
+		workChs[i] = make(chan *unpackWork)
+		workChsWG.Add(1)
+		go func(workerID int) {
+			defer workChsWG.Done()
+			unpackWorker(workChs[workerID])
+		}(i)
+	}
+
+	// Feed workers with work
 	upws := make([]*unpackWork, 0, 1+brsLen/unpackBatchSize)
 	upw := getUnpackWork()
 	upw.tbf = tbf
 	for _, br := range pts.brs {
 		if len(upw.ws) >= unpackBatchSize {
-			unpackWorkCh <- upw
+			scheduleUnpackWork(workChs, upw)
 			upws = append(upws, upw)
 			upw = getUnpackWork()
 			upw.tbf = tbf
@@ -323,11 +427,12 @@ func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.
 			tr: tr,
 		})
 	}
-	unpackWorkCh <- upw
+	scheduleUnpackWork(workChs, upw)
 	upws = append(upws, upw)
 	pts.brs = pts.brs[:0]
 
 	// Wait until work is complete
+	samples := 0
 	sbs := make([]*sortBlock, 0, brsLen)
 	var firstErr error
 	for _, upw := range upws {
@@ -336,14 +441,30 @@ func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.
 			firstErr = err
 		}
 		if firstErr == nil {
-			sbs = append(sbs, upw.sbs...)
-		} else {
+			for _, sb := range upw.sbs {
+				samples += len(sb.Timestamps)
+			}
+			if *maxSamplesPerSeries <= 0 || samples < *maxSamplesPerSeries {
+				sbs = append(sbs, upw.sbs...)
+			} else {
+				firstErr = fmt.Errorf("cannot process more than %d samples per series; either increase -search.maxSamplesPerSeries "+
+					"or reduce time range for the query", *maxSamplesPerSeries)
+			}
+		}
+		if firstErr != nil {
 			for _, sb := range upw.sbs {
 				putSortBlock(sb)
 			}
 		}
 		putUnpackWork(upw)
 	}
+
+	// Shut down local workers
+	for _, workCh := range workChs {
+		close(workCh)
+	}
+	workChsWG.Wait()
+
 	if firstErr != nil {
 		return firstErr
 	}
@@ -913,6 +1034,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 	m := make(map[string][]blockRef, maxSeriesCount)
 	orderedMetricNames := make([]string, 0, maxSeriesCount)
 	blocksRead := 0
+	samples := 0
 	tbf := getTmpBlocksFile()
 	var buf []byte
 	for sr.NextMetricBlock() {
@@ -922,7 +1044,14 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 			putStorageSearch(sr)
 			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
-		buf = sr.MetricBlockRef.BlockRef.Marshal(buf[:0])
+		br := sr.MetricBlockRef.BlockRef
+		samples += br.RowsCount()
+		if *maxSamplesPerQuery > 0 && samples > *maxSamplesPerQuery {
+			putTmpBlocksFile(tbf)
+			putStorageSearch(sr)
+			return nil, fmt.Errorf("cannot select more than -search.maxSamplesPerQuery=%d samples; possible solutions: to increase the -search.maxSamplesPerQuery; to reduce time range for the query; to use more specific label filters in order to select lower number of series", *maxSamplesPerQuery)
+		}
+		buf = br.Marshal(buf[:0])
 		addr, err := tbf.WriteBlockRefData(buf)
 		if err != nil {
 			putTmpBlocksFile(tbf)
@@ -930,15 +1059,13 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
 		metricName := sr.MetricBlockRef.MetricName
-		metricNameStrUnsafe := bytesutil.ToUnsafeString(metricName)
-		brs := m[metricNameStrUnsafe]
+		brs := m[string(metricName)]
 		brs = append(brs, blockRef{
-			partRef: sr.MetricBlockRef.BlockRef.PartRef(),
+			partRef: br.PartRef(),
 			addr:    addr,
 		})
 		if len(brs) > 1 {
-			// An optimization: do not allocate a string for already existing metricName key in m
-			m[metricNameStrUnsafe] = brs
+			m[string(metricName)] = brs
 		} else {
 			// An optimization for big number of time series with long metricName values:
 			// use only a single copy of metricName for both orderedMetricNames and m.
@@ -1008,7 +1135,6 @@ func setupTfss(tr storage.TimeRange, tagFilterss [][]storage.TagFilter, deadline
 			}
 		}
 		tfss = append(tfss, tfs)
-		tfss = append(tfss, tfs.Finalize()...)
 	}
 	return tfss, nil
 }

app/vmselect/prometheus/prometheus.go

@@ -49,6 +49,8 @@ const defaultStep = 5 * 60 * 1000
 
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
 func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer federateDuration.UpdateDuration(startTime)
+
 	ct := startTime.UnixNano() / 1e6
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %w", err)
@@ -96,12 +98,11 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 		return err
 	})
 	if err != nil {
-		return fmt.Errorf("error during data fetching: %w", err)
+		return fmt.Errorf("error during sending data to remote client: %w", err)
 	}
 	if err := bw.Flush(); err != nil {
 		return err
 	}
-	federateDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -109,6 +110,8 @@ var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/fe
 
 // ExportCSVHandler exports data in CSV format from /api/v1/export/csv
 func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer exportCSVDuration.UpdateDuration(startTime)
+
 	ct := startTime.UnixNano() / 1e6
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %w", err)
@@ -172,9 +175,8 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 	}
 	err = <-doneCh
 	if err != nil {
-		return fmt.Errorf("error during exporting data to csv: %w", err)
+		return fmt.Errorf("error during sending the exported csv data to remote client: %w", err)
 	}
-	exportCSVDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -182,6 +184,8 @@ var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/a
 
 // ExportNativeHandler exports data in native format from /api/v1/export/native.
 func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer exportNativeDuration.UpdateDuration(startTime)
+
 	ct := startTime.UnixNano() / 1e6
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %w", err)
@@ -240,12 +244,11 @@ func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
 		return err
 	})
 	if err != nil {
-		return err
+		return fmt.Errorf("error during sending native data to remote client: %w", err)
 	}
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("error during flushing native data to remote client: %w", err)
 	}
-	exportNativeDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -255,6 +258,8 @@ var bbPool bytesutil.ByteBufferPool
 
 // ExportHandler exports data in raw format from /api/v1/export.
 func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer exportDuration.UpdateDuration(startTime)
+
 	ct := startTime.UnixNano() / 1e6
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %w", err)
@@ -285,7 +290,6 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	if err := exportHandler(w, matches, etf, start, end, format, maxRowsPerLine, reduceMemUsage, deadline); err != nil {
 		return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %w", matches, start, end, err)
 	}
-	exportDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -410,7 +414,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etf []storage.TagFil
 	}
 	err = <-doneCh
 	if err != nil {
-		return fmt.Errorf("error during data fetching: %w", err)
+		return fmt.Errorf("error during sending the data to remote client: %w", err)
 	}
 	return nil
 }
@@ -437,6 +441,8 @@ var exportBlockPool = &sync.Pool{
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#delete-series
 func DeleteHandler(startTime time.Time, r *http.Request) error {
+	defer deleteDuration.UpdateDuration(startTime)
+
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %w", err)
@@ -457,7 +463,6 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
 	if deletedCount > 0 {
 		promql.ResetRollupResultCache()
 	}
-	deleteDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -467,6 +472,8 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
 func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWriter, r *http.Request) error {
+	defer labelValuesDuration.UpdateDuration(startTime)
+
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
@@ -531,9 +538,8 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 	defer bufferedwriter.Put(bw)
 	WriteLabelValuesResponse(bw, labelValues)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("canot flush label values to remote client: %w", err)
 	}
-	labelValuesDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -594,7 +600,7 @@ func labelValuesWithMatches(labelName string, matches []string, etf []storage.Ta
 			return nil
 		})
 		if err != nil {
-			return nil, fmt.Errorf("error when data fetching: %w", err)
+			return nil, fmt.Errorf("cannot fetch label values from storage: %w", err)
 		}
 	}
 	labelValues := make([]string, 0, len(m))
@@ -609,6 +615,8 @@ var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 
 // LabelsCountHandler processes /api/v1/labels/count request.
 func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer labelsCountDuration.UpdateDuration(startTime)
+
 	deadline := searchutils.GetDeadlineForStatusRequest(r, startTime)
 	labelEntries, err := netstorage.GetLabelEntries(deadline)
 	if err != nil {
@@ -619,9 +627,8 @@ func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	defer bufferedwriter.Put(bw)
 	WriteLabelsCountResponse(bw, labelEntries)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send labels count response to remote client: %w", err)
 	}
-	labelsCountDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -635,6 +642,8 @@ const secsPerDay = 3600 * 24
 //
 // It can accept `match[]` filters in order to narrow down the search.
 func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer tsdbStatusDuration.UpdateDuration(startTime)
+
 	deadline := searchutils.GetDeadlineForStatusRequest(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
@@ -686,9 +695,8 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	defer bufferedwriter.Put(bw)
 	WriteTSDBStatusResponse(bw, status)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send tsdb status response to remote client: %w", err)
 	}
-	tsdbStatusDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -717,6 +725,8 @@ var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
 func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer labelsDuration.UpdateDuration(startTime)
+
 	deadline := searchutils.GetDeadlineForQuery(r, startTime)
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
@@ -779,9 +789,8 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	defer bufferedwriter.Put(bw)
 	WriteLabelsResponse(bw, labels)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send labels response to remote client: %w", err)
 	}
-	labelsDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -829,7 +838,7 @@ func labelsWithMatches(matches []string, etf []storage.TagFilter, start, end int
 			return nil
 		})
 		if err != nil {
-			return nil, fmt.Errorf("error when data fetching: %w", err)
+			return nil, fmt.Errorf("cannot fetch labels from storage: %w", err)
 		}
 	}
 	labels := make([]string, 0, len(m))
@@ -844,6 +853,8 @@ var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 
 // SeriesCountHandler processes /api/v1/series/count request.
 func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer seriesCountDuration.UpdateDuration(startTime)
+
 	deadline := searchutils.GetDeadlineForStatusRequest(r, startTime)
 	n, err := netstorage.GetSeriesCount(deadline)
 	if err != nil {
@@ -854,9 +865,8 @@ func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	defer bufferedwriter.Put(bw)
 	WriteSeriesCountResponse(bw, n)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send series count response to remote client: %w", err)
 	}
-	seriesCountDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -866,6 +876,8 @@ var seriesCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers
 func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer seriesDuration.UpdateDuration(startTime)
+
 	ct := startTime.UnixNano() / 1e6
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
@@ -945,13 +957,12 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	// WriteSeriesResponse must consume all the data from resultsCh.
 	WriteSeriesResponse(bw, resultsCh)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot flush series response to remote client: %w", err)
 	}
 	err = <-doneCh
 	if err != nil {
-		return fmt.Errorf("error during data fetching: %w", err)
+		return fmt.Errorf("cannot send series response to remote client: %w", err)
 	}
-	seriesDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -961,6 +972,8 @@ var seriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer queryDuration.UpdateDuration(startTime)
+
 	ct := startTime.UnixNano() / 1e6
 	query := r.FormValue("query")
 	if len(query) == 0 {
@@ -1062,9 +1075,8 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	defer bufferedwriter.Put(bw)
 	WriteQueryResponse(bw, result)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot flush query response to remote client: %w", err)
 	}
-	queryDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -1074,6 +1086,8 @@ var queryDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 func QueryRangeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer queryRangeDuration.UpdateDuration(startTime)
+
 	ct := startTime.UnixNano() / 1e6
 	query := r.FormValue("query")
 	if len(query) == 0 {
@@ -1098,7 +1112,6 @@ func QueryRangeHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	if err := queryRangeHandler(startTime, w, query, start, end, step, r, ct, etf); err != nil {
 		return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", query, start, end, step, err)
 	}
-	queryRangeDuration.UpdateDuration(startTime)
 	return nil
 }
 
@@ -1155,7 +1168,7 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
 	defer bufferedwriter.Put(bw)
 	WriteQueryRangeResponse(bw, result)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send query range response to remote client: %w", err)
 	}
 	return nil
 }
@@ -1311,6 +1324,8 @@ func getLatencyOffsetMilliseconds() int64 {
 
 // QueryStatsHandler returns query stats at `/api/v1/status/top_queries`
 func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	defer queryStatsDuration.UpdateDuration(startTime)
+
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
@@ -1333,9 +1348,8 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	defer bufferedwriter.Put(bw)
 	querystats.WriteJSONQueryStats(bw, topN, maxLifetime)
 	if err := bw.Flush(); err != nil {
-		return err
+		return fmt.Errorf("cannot send query stats response to client: %w", err)
 	}
-	queryStatsDuration.UpdateDuration(startTime)
 	return nil
 }
 

app/vmselect/promql/aggr.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/VictoriaMetrics/metricsql"
-	"github.com/valyala/histogram"
+	xxhash "github.com/cespare/xxhash/v2"
 )
 
 var aggrFuncs = map[string]aggrFunc{
@@ -40,14 +40,19 @@ var aggrFuncs = map[string]aggrFunc{
 	"topk_max":       newAggrFuncRangeTopK(maxValue, false),
 	"topk_avg":       newAggrFuncRangeTopK(avgValue, false),
 	"topk_median":    newAggrFuncRangeTopK(medianValue, false),
+	"topk_last":      newAggrFuncRangeTopK(lastValue, false),
 	"bottomk_min":    newAggrFuncRangeTopK(minValue, true),
 	"bottomk_max":    newAggrFuncRangeTopK(maxValue, true),
 	"bottomk_avg":    newAggrFuncRangeTopK(avgValue, true),
 	"bottomk_median": newAggrFuncRangeTopK(medianValue, true),
+	"bottomk_last":   newAggrFuncRangeTopK(lastValue, true),
 	"any":            aggrFuncAny,
+	"mad":            newAggrFunc(aggrFuncMAD),
+	"outliers_mad":   aggrFuncOutliersMAD,
 	"outliersk":      aggrFuncOutliersK,
 	"mode":           newAggrFunc(aggrFuncMode),
 	"zscore":         aggrFuncZScore,
+	"quantiles":      aggrFuncQuantiles,
 }
 
 type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error)
@@ -484,7 +489,6 @@ func aggrFuncZScore(afa *aggrFuncArg) ([]*timeseries, error) {
 				ts.Values[i] = (v - avg) / stddev
 			}
 		}
-
 		// Remove MetricGroup from all the tss.
 		for _, ts := range tss {
 			ts.MetricName.ResetMetricGroup()
@@ -575,7 +579,7 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 			var dst timeseries
 			dst.CopyFromShallowTimestamps(tss[0])
 			dst.MetricName.RemoveTag(dstLabel)
-			dst.MetricName.AddTag(dstLabel, strconv.FormatFloat(v, 'g', -1, 64))
+			dst.MetricName.AddTag(dstLabel, strconv.FormatFloat(v, 'f', -1, 64))
 			for i := range dst.Values {
 				count := 0
 				for _, ts := range tss {
@@ -799,15 +803,127 @@ func avgValue(values []float64) float64 {
 }
 
 func medianValue(values []float64) float64 {
-	h := histogram.GetFast()
-	for _, v := range values {
-		if !math.IsNaN(v) {
-			h.Update(v)
-		}
+	return quantile(0.5, values)
+}
+
+func lastValue(values []float64) float64 {
+	values = skipTrailingNaNs(values)
+	if len(values) == 0 {
+		return nan
 	}
-	value := h.Quantile(0.5)
-	histogram.PutFast(h)
-	return value
+	return values[len(values)-1]
+}
+
+// quantiles calculates the given phis from originValues without modifying originValues, appends them to qs and returns the result.
+func quantiles(qs, phis []float64, originValues []float64) []float64 {
+	a := getFloat64s()
+	a.A = prepareForQuantileFloat64(a.A[:0], originValues)
+	qs = quantilesSorted(qs, phis, a.A)
+	putFloat64s(a)
+	return qs
+}
+
+// quantile calculates the given phi from originValues without modifying originValues
+func quantile(phi float64, originValues []float64) float64 {
+	a := getFloat64s()
+	a.A = prepareForQuantileFloat64(a.A[:0], originValues)
+	q := quantileSorted(phi, a.A)
+	putFloat64s(a)
+	return q
+}
+
+// prepareForQuantileFloat64 copies items from src to dst but removes NaNs and sorts the dst
+func prepareForQuantileFloat64(dst, src []float64) []float64 {
+	for _, v := range src {
+		if math.IsNaN(v) {
+			continue
+		}
+		dst = append(dst, v)
+	}
+	sort.Float64s(dst)
+	return dst
+}
+
+// quantilesSorted calculates the given phis over a sorted list of values, appends them to qs and returns the result.
+//
+// It is expected that values won't contain NaN items.
+// The implementation mimics Prometheus implementation for compatibility's sake.
+func quantilesSorted(qs, phis []float64, values []float64) []float64 {
+	for _, phi := range phis {
+		q := quantileSorted(phi, values)
+		qs = append(qs, q)
+	}
+	return qs
+}
+
+// quantileSorted calculates the given quantile over a sorted list of values.
+//
+// It is expected that values won't contain NaN items.
+// The implementation mimics Prometheus implementation for compatibility's sake.
+func quantileSorted(phi float64, values []float64) float64 {
+	if len(values) == 0 || math.IsNaN(phi) {
+		return nan
+	}
+	if phi < 0 {
+		return math.Inf(-1)
+	}
+	if phi > 1 {
+		return math.Inf(+1)
+	}
+	n := float64(len(values))
+	rank := phi * (n - 1)
+
+	lowerIndex := math.Max(0, math.Floor(rank))
+	upperIndex := math.Min(n-1, lowerIndex+1)
+
+	weight := rank - math.Floor(rank)
+	return values[int(lowerIndex)]*(1-weight) + values[int(upperIndex)]*weight
+}
+
+func aggrFuncMAD(tss []*timeseries) []*timeseries {
+	// Calculate medians for each point across tss.
+	medians := getPerPointMedians(tss)
+	// Calculate MAD values multipled by tolerance for each point across tss.
+	// See https://en.wikipedia.org/wiki/Median_absolute_deviation
+	mads := getPerPointMADs(tss, medians)
+	tss[0].Values = append(tss[0].Values[:0], mads...)
+	return tss[:1]
+}
+
+func aggrFuncOutliersMAD(afa *aggrFuncArg) ([]*timeseries, error) {
+	args := afa.args
+	if err := expectTransformArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	tolerances, err := getScalar(args[0], 0)
+	if err != nil {
+		return nil, err
+	}
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
+		// Calculate medians for each point across tss.
+		medians := getPerPointMedians(tss)
+		// Calculate MAD values multipled by tolerance for each point across tss.
+		// See https://en.wikipedia.org/wiki/Median_absolute_deviation
+		mads := getPerPointMADs(tss, medians)
+		for n := range mads {
+			mads[n] *= tolerances[n]
+		}
+		// Leave only time series with at least a single peak above the MAD multiplied by tolerance.
+		tssDst := tss[:0]
+		for _, ts := range tss {
+			values := ts.Values
+			for n, v := range values {
+				ad := math.Abs(v - medians[n])
+				mad := mads[n]
+				if ad > mad {
+					tssDst = append(tssDst, ts)
+					break
+				}
+			}
+		}
+		return tssDst
+	}
+	return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true)
 }
 
 func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
@@ -821,20 +937,7 @@ func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
 	}
 	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		// Calculate medians for each point across tss.
-		medians := make([]float64, len(ks))
-		h := histogram.GetFast()
-		for n := range ks {
-			h.Reset()
-			for j := range tss {
-				v := tss[j].Values[n]
-				if !math.IsNaN(v) {
-					h.Update(v)
-				}
-			}
-			medians[n] = h.Quantile(0.5)
-		}
-		histogram.PutFast(h)
-
+		medians := getPerPointMedians(tss)
 		// Return topK time series with the highest variance from median.
 		f := func(values []float64) float64 {
 			sum2 := float64(0)
@@ -849,6 +952,48 @@ func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
 	return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true)
 }
 
+func getPerPointMedians(tss []*timeseries) []float64 {
+	if len(tss) == 0 {
+		logger.Panicf("BUG: expecting non-empty tss")
+	}
+	medians := make([]float64, len(tss[0].Values))
+	a := getFloat64s()
+	values := a.A
+	for n := range medians {
+		values = values[:0]
+		for j := range tss {
+			v := tss[j].Values[n]
+			if !math.IsNaN(v) {
+				values = append(values, v)
+			}
+		}
+		medians[n] = quantile(0.5, values)
+	}
+	a.A = values
+	putFloat64s(a)
+	return medians
+}
+
+func getPerPointMADs(tss []*timeseries, medians []float64) []float64 {
+	mads := make([]float64, len(medians))
+	a := getFloat64s()
+	values := a.A
+	for n, median := range medians {
+		values = values[:0]
+		for j := range tss {
+			v := tss[j].Values[n]
+			if !math.IsNaN(v) {
+				ad := math.Abs(v - median)
+				values = append(values, ad)
+			}
+		}
+		mads[n] = quantile(0.5, values)
+	}
+	a.A = values
+	putFloat64s(a)
+	return mads
+}
+
 func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 	args := afa.args
 	if err := expectTransformArgsNum(args, 2); err != nil {
@@ -866,6 +1011,28 @@ func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 		}
 	}
 	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
+		// Sort series by metricName in order to get consistent set of output series
+		// across multiple calls to limitk() function.
+		// Sort series by hash in order to guarantee uniform selection across series.
+		type hashSeries struct {
+			h  uint64
+			ts *timeseries
+		}
+		hss := make([]hashSeries, len(tss))
+		d := xxhash.New()
+		for i, ts := range tss {
+			h := getHash(d, &ts.MetricName)
+			hss[i] = hashSeries{
+				h:  h,
+				ts: ts,
+			}
+		}
+		sort.Slice(hss, func(i, j int) bool {
+			return hss[i].h < hss[j].h
+		})
+		for i, hs := range hss {
+			tss[i] = hs.ts
+		}
 		if len(tss) > maxK {
 			tss = tss[:maxK]
 		}
@@ -883,6 +1050,72 @@ func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 	return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true)
 }
 
+func getHash(d *xxhash.Digest, mn *storage.MetricName) uint64 {
+	d.Reset()
+	_, _ = d.Write(mn.MetricGroup)
+	for _, tag := range mn.Tags {
+		_, _ = d.Write(tag.Key)
+		_, _ = d.Write(tag.Value)
+	}
+	return d.Sum64()
+
+}
+
+func aggrFuncQuantiles(afa *aggrFuncArg) ([]*timeseries, error) {
+	args := afa.args
+	if len(args) < 3 {
+		return nil, fmt.Errorf("unexpected number of args: %d; expecting at least 3 args", len(args))
+	}
+	dstLabel, err := getString(args[0], 0)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain dstLabel: %w", err)
+	}
+	phiArgs := args[1 : len(args)-1]
+	phis := make([]float64, len(phiArgs))
+	for i, phiArg := range phiArgs {
+		phisLocal, err := getScalar(phiArg, i+1)
+		if err != nil {
+			return nil, err
+		}
+		if len(phis) == 0 {
+			logger.Panicf("BUG: expecting at least a single sample")
+		}
+		phis[i] = phisLocal[0]
+	}
+	argOrig := args[len(args)-1]
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
+		tssDst := make([]*timeseries, len(phiArgs))
+		for j := range tssDst {
+			ts := &timeseries{}
+			ts.CopyFromShallowTimestamps(tss[0])
+			ts.MetricName.RemoveTag(dstLabel)
+			ts.MetricName.AddTag(dstLabel, fmt.Sprintf("%g", phis[j]))
+			tssDst[j] = ts
+		}
+
+		b := getFloat64s()
+		qs := b.A
+		a := getFloat64s()
+		values := a.A
+		for n := range tss[0].Values {
+			values = values[:0]
+			for j := range tss {
+				values = append(values, tss[j].Values[n])
+			}
+			qs = quantiles(qs[:0], phis, values)
+			for j := range tssDst {
+				tssDst[j].Values[n] = qs[j]
+			}
+		}
+		a.A = values
+		putFloat64s(a)
+		b.A = qs
+		putFloat64s(b)
+		return tssDst
+	}
+	return aggrFuncExt(afe, argOrig, &afa.ae.Modifier, afa.ae.Limit, false)
+}
+
 func aggrFuncQuantile(afa *aggrFuncArg) ([]*timeseries, error) {
 	args := afa.args
 	if err := expectTransformArgsNum(args, 2); err != nil {
@@ -909,19 +1142,17 @@ func aggrFuncMedian(afa *aggrFuncArg) ([]*timeseries, error) {
 func newAggrQuantileFunc(phis []float64) func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 	return func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		dst := tss[0]
-		h := histogram.GetFast()
-		defer histogram.PutFast(h)
+		a := getFloat64s()
+		values := a.A
 		for n := range dst.Values {
-			h.Reset()
+			values = values[:0]
 			for j := range tss {
-				v := tss[j].Values[n]
-				if !math.IsNaN(v) {
-					h.Update(v)
-				}
+				values = append(values, tss[j].Values[n])
 			}
-			phi := phis[n]
-			dst.Values[n] = h.Quantile(phi)
+			dst.Values[n] = quantile(phis[n], values)
 		}
+		a.A = values
+		putFloat64s(a)
 		tss[0] = dst
 		return tss[:1]
 	}

app/vmselect/promql/binary_op.go

@@ -19,6 +19,9 @@ var binaryOpFuncs = map[string]binaryOpFunc{
 	"%": newBinaryOpArithFunc(binaryop.Mod),
 	"^": newBinaryOpArithFunc(binaryop.Pow),
 
+	// See https://github.com/prometheus/prometheus/pull/9248
+	"atan2": newBinaryOpArithFunc(binaryop.Atan2),
+
 	// cmp ops
 	"==": newBinaryOpCmpFunc(binaryop.Eq),
 	"!=": newBinaryOpCmpFunc(binaryop.Neq),
@@ -59,12 +62,12 @@ func newBinaryOpCmpFunc(cf func(left, right float64) bool) binaryOpFunc {
 			}
 			return nan
 		}
-		if cf(left, right) {
-			return 1
-		}
 		if math.IsNaN(left) {
 			return nan
 		}
+		if cf(left, right) {
+			return 1
+		}
 		return 0
 	}
 	return newBinaryOpFunc(cfe)
@@ -100,9 +103,8 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 				dstValues[j] = bf(a, b, isBool)
 			}
 		}
-		// Optimization: remove time series containing only NaNs.
-		// This is quite common after applying filters like `q > 0`.
-		dst = removeNaNs(dst)
+		// Do not remove time series containing only NaNs, since then the `(foo op bar) default N`
+		// won't work as expected if `(foo op bar)` results to NaN series.
 		return dst, nil
 	}
 }
@@ -304,12 +306,18 @@ func binaryOpAnd(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 		if tssLeft == nil {
 			continue
 		}
-		// Add gaps to tssLeft if there are gaps at valuesRight.
-		valuesRight := tssRight[0].Values
+		// Add gaps to tssLeft if there are gaps at tssRight.
 		for _, tsLeft := range tssLeft {
 			valuesLeft := tsLeft.Values
-			for i, v := range valuesRight {
-				if math.IsNaN(v) {
+			for i := range valuesLeft {
+				hasValue := false
+				for _, tsRight := range tssRight {
+					if !math.IsNaN(tsRight.Values[i]) {
+						hasValue = true
+						break
+					}
+				}
+				if !hasValue {
 					valuesLeft[i] = nan
 				}
 			}
@@ -334,12 +342,18 @@ func binaryOpOr(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 		}
 		// Fill gaps in tssLeft with values from tssRight as Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/552
-		valuesRight := tssRight[0].Values
 		for _, tsLeft := range tssLeft {
 			valuesLeft := tsLeft.Values
 			for i, v := range valuesLeft {
-				if math.IsNaN(v) {
-					valuesLeft[i] = valuesRight[i]
+				if !math.IsNaN(v) {
+					continue
+				}
+				for _, tsRight := range tssRight {
+					vRight := tsRight.Values[i]
+					if !math.IsNaN(vRight) {
+						valuesLeft[i] = vRight
+						break
+					}
 				}
 			}
 		}
@@ -356,13 +370,15 @@ func binaryOpUnless(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 			rvs = append(rvs, tssLeft...)
 			continue
 		}
-		// Add gaps to tssLeft if the are no gaps at valuesRight.
-		valuesRight := tssRight[0].Values
+		// Add gaps to tssLeft if the are no gaps at tssRight.
 		for _, tsLeft := range tssLeft {
 			valuesLeft := tsLeft.Values
-			for i, v := range valuesRight {
-				if !math.IsNaN(v) {
-					valuesLeft[i] = nan
+			for i := range valuesLeft {
+				for _, tsRight := range tssRight {
+					if !math.IsNaN(tsRight.Values[i]) {
+						valuesLeft[i] = nan
+						break
+					}
 				}
 			}
 		}

app/vmselect/promql/eval.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"math"
+	"strings"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -22,6 +24,7 @@ var (
 	maxPointsPerTimeseries = flag.Int("search.maxPointsPerTimeseries", 30e3, "The maximum points per a single timeseries returned from /api/v1/query_range. "+
 		"This option doesn't limit the number of scanned raw samples in the database. The main purpose of this option is to limit the number of per-series points "+
 		"returned to graphing UI such as Grafana. There is no sense in setting this limit to values bigger than the horizontal resolution of the graph")
+	noStaleMarkers = flag.Bool("search.noStaleMarkers", false, "Set this flag to true if the database doesn't contain Prometheus stale markers, so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets")
 )
 
 // The minimum number of points per timeseries for enabling time rounding.
@@ -388,7 +391,7 @@ func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.F
 	if nrf == nil {
 		return nil, nil
 	}
-	rollupArgIdx := getRollupArgIdx(fe.Name)
+	rollupArgIdx := getRollupArgIdx(fe)
 	if rollupArgIdx >= len(fe.Args) {
 		// Incorrect number of args for rollup func.
 		return nil, nil
@@ -428,7 +431,7 @@ func evalExprs(ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
 
 func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
 	var re *metricsql.RollupExpr
-	rollupArgIdx := getRollupArgIdx(fe.Name)
+	rollupArgIdx := getRollupArgIdx(fe)
 	if len(fe.Args) <= rollupArgIdx {
 		return nil, nil, fmt.Errorf("expecting at least %d args to %q; got %d args; expr: %q", rollupArgIdx+1, fe.Name, len(fe.Args), fe.AppendString(nil))
 	}
@@ -476,7 +479,8 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 	return &reNew
 }
 
-func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+func evalRollupFunc(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
 	if re.Offset != nil {
@@ -489,7 +493,7 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.E
 		// so cache hit rate should be quite good.
 		// See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/976
 	}
-	if name == "rollup_candlestick" {
+	if funcName == "rollup_candlestick" {
 		// Automatically apply `offset -step` to `rollup_candlestick` function
 		// in order to obtain expected OHLC results.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/309#issuecomment-582113462
@@ -502,12 +506,12 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.E
 	var rvs []*timeseries
 	var err error
 	if me, ok := re.Expr.(*metricsql.MetricExpr); ok {
-		rvs, err = evalRollupFuncWithMetricExpr(ecNew, name, rf, expr, me, iafc, re.Window)
+		rvs, err = evalRollupFuncWithMetricExpr(ecNew, funcName, rf, expr, me, iafc, re.Window)
 	} else {
 		if iafc != nil {
-			logger.Panicf("BUG: iafc must be nil for rollup %q over subquery %q", name, re.AppendString(nil))
+			logger.Panicf("BUG: iafc must be nil for rollup %q over subquery %q", funcName, re.AppendString(nil))
 		}
-		rvs, err = evalRollupFuncWithSubquery(ecNew, name, rf, expr, re)
+		rvs, err = evalRollupFuncWithSubquery(ecNew, funcName, rf, expr, re)
 	}
 	if err != nil {
 		return nil, err
@@ -526,7 +530,7 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.E
 	return rvs, nil
 }
 
-func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr) ([]*timeseries, error) {
+func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr) ([]*timeseries, error) {
 	// TODO: determine whether to use rollupResultCacheV here.
 	step := re.Step.Duration(ec.Step)
 	if step == 0 {
@@ -548,25 +552,24 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, expr
 		return nil, err
 	}
 	if len(tssSQ) == 0 {
-		if name == "absent_over_time" {
+		if funcName == "absent_over_time" {
 			tss := evalNumber(ec, 1)
 			return tss, nil
 		}
 		return nil, nil
 	}
 	sharedTimestamps := getTimestamps(ec.Start, ec.End, ec.Step)
-	preFunc, rcs, err := getRollupConfigs(name, rf, expr, ec.Start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
+	preFunc, rcs, err := getRollupConfigs(funcName, rf, expr, ec.Start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
 	if err != nil {
 		return nil, err
 	}
 	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
 	var tssLock sync.Mutex
-	removeMetricGroup := !rollupFuncsKeepMetricGroup[name]
 	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) {
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
 		preFunc(values, timestamps)
 		for _, rc := range rcs {
-			if tsm := newTimeseriesMap(name, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
+			if tsm := newTimeseriesMap(funcName, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
 				rc.DoTimeseriesMap(tsm, values, timestamps)
 				tssLock.Lock()
 				tss = tsm.AppendTimeseriesTo(tss)
@@ -574,7 +577,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, expr
 				continue
 			}
 			var ts timeseries
-			doRollupForTimeseries(rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps, removeMetricGroup)
+			doRollupForTimeseries(funcName, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps)
 			tssLock.Lock()
 			tss = append(tss, &ts)
 			tssLock.Unlock()
@@ -640,7 +643,7 @@ var (
 	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
 )
 
-func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
+func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc,
 	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
 	if me.IsEmpty() {
 		return evalNumber(ec, nan), nil
@@ -663,7 +666,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
 	// Obtain rollup configs before fetching data from db,
 	// so type errors can be caught earlier.
 	sharedTimestamps := getTimestamps(start, ec.End, ec.Step)
-	preFunc, rcs, err := getRollupConfigs(name, rf, expr, start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
+	preFunc, rcs, err := getRollupConfigs(funcName, rf, expr, start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
 	if err != nil {
 		return nil, err
 	}
@@ -687,7 +690,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
 	if rssLen == 0 {
 		rss.Cancel()
 		var tss []*timeseries
-		if name == "absent_over_time" {
+		if funcName == "absent_over_time" {
 			tss = getAbsentTimeseries(ec, me)
 		}
 		// Add missing points until ec.End.
@@ -733,12 +736,11 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
 	defer rml.Put(uint64(rollupMemorySize))
 
 	// Evaluate rollup
-	removeMetricGroup := !rollupFuncsKeepMetricGroup[name]
 	var tss []*timeseries
 	if iafc != nil {
-		tss, err = evalRollupWithIncrementalAggregate(name, iafc, rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
+		tss, err = evalRollupWithIncrementalAggregate(funcName, iafc, rss, rcs, preFunc, sharedTimestamps)
 	} else {
-		tss, err = evalRollupNoIncrementalAggregate(name, rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
+		tss, err = evalRollupNoIncrementalAggregate(funcName, rss, rcs, preFunc, sharedTimestamps)
 	}
 	if err != nil {
 		return nil, err
@@ -760,14 +762,15 @@ func getRollupMemoryLimiter() *memoryLimiter {
 	return &rollupMemoryLimiter
 }
 
-func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
+func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
 		ts := getTimeseries()
 		defer putTimeseries(ts)
 		for _, rc := range rcs {
-			if tsm := newTimeseriesMap(name, sharedTimestamps, &rs.MetricName); tsm != nil {
+			if tsm := newTimeseriesMap(funcName, sharedTimestamps, &rs.MetricName); tsm != nil {
 				rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
 				for _, ts := range tsm.m {
 					iafc.updateTimeseries(ts, workerID)
@@ -775,7 +778,7 @@ func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncCo
 				continue
 			}
 			ts.Reset()
-			doRollupForTimeseries(rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps, removeMetricGroup)
+			doRollupForTimeseries(funcName, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
 			iafc.updateTimeseries(ts, workerID)
 
 			// ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used.
@@ -791,14 +794,15 @@ func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncCo
 	return tss, nil
 }
 
-func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
+func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results, rcs []*rollupConfig,
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
 	var tssLock sync.Mutex
 	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {
-			if tsm := newTimeseriesMap(name, sharedTimestamps, &rs.MetricName); tsm != nil {
+			if tsm := newTimeseriesMap(funcName, sharedTimestamps, &rs.MetricName); tsm != nil {
 				rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
 				tssLock.Lock()
 				tss = tsm.AppendTimeseriesTo(tss)
@@ -806,7 +810,7 @@ func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs
 				continue
 			}
 			var ts timeseries
-			doRollupForTimeseries(rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps, removeMetricGroup)
+			doRollupForTimeseries(funcName, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
 			tssLock.Lock()
 			tss = append(tss, &ts)
 			tssLock.Unlock()
@@ -819,13 +823,13 @@ func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs
 	return tss, nil
 }
 
-func doRollupForTimeseries(rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName, valuesSrc []float64, timestampsSrc []int64,
-	sharedTimestamps []int64, removeMetricGroup bool) {
+func doRollupForTimeseries(funcName string, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName, valuesSrc []float64, timestampsSrc []int64,
+	sharedTimestamps []int64) {
 	tsDst.MetricName.CopyFrom(mnSrc)
 	if len(rc.TagValue) > 0 {
 		tsDst.MetricName.AddTag("rollup", rc.TagValue)
 	}
-	if removeMetricGroup {
+	if !rollupFuncsKeepMetricGroup[funcName] {
 		tsDst.MetricName.ResetMetricGroup()
 	}
 	tsDst.Values = rc.Do(tsDst.Values[:0], valuesSrc, timestampsSrc)
@@ -891,3 +895,34 @@ func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) {
 	dst.IsRegexp = src.IsRegexp
 	dst.IsNegative = src.IsNegative
 }
+
+func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]float64, []int64) {
+	if *noStaleMarkers || funcName == "default_rollup" {
+		// Do not drop Prometheus staleness marks (aka stale NaNs) for default_rollup() function,
+		// since it uses them for Prometheus-style staleness detection.
+		return values, timestamps
+	}
+	// Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values.
+	hasStaleSamples := false
+	for _, v := range values {
+		if decimal.IsStaleNaN(v) {
+			hasStaleSamples = true
+			break
+		}
+	}
+	if !hasStaleSamples {
+		// Fast path: values have no Prometheus staleness marks.
+		return values, timestamps
+	}
+	// Slow path: drop Prometheus staleness marks from values.
+	dstValues := values[:0]
+	dstTimestamps := timestamps[:0]
+	for i, v := range values {
+		if decimal.IsStaleNaN(v) {
+			continue
+		}
+		dstValues = append(dstValues, v)
+		dstTimestamps = append(dstTimestamps, timestamps[i])
+	}
+	return dstValues, dstTimestamps
+}

app/vmselect/promql/exec.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/VictoriaMetrics/metricsql"
 )
@@ -80,8 +81,8 @@ func maySortResults(e metricsql.Expr, tss []*timeseries) bool {
 	case *metricsql.AggrFuncExpr:
 		switch strings.ToLower(v.Name) {
 		case "topk", "bottomk", "outliersk",
-			"topk_max", "topk_min", "topk_avg", "topk_median",
-			"bottomk_max", "bottomk_min", "bottomk_avg", "bottomk_median":
+			"topk_max", "topk_min", "topk_avg", "topk_median", "topk_last",
+			"bottomk_max", "bottomk_min", "bottomk_avg", "bottomk_median", "bottomk_last":
 			return false
 		}
 	}
@@ -101,7 +102,6 @@ func timeseriesToResult(tss []*timeseries, maySort bool) ([]netstorage.Result, e
 		m[string(bb.B)] = struct{}{}
 
 		rs := &result[i]
-		rs.MetricNameMarshaled = append(rs.MetricNameMarshaled[:0], bb.B...)
 		rs.MetricName.CopyFrom(&ts.MetricName)
 		rs.Values = append(rs.Values[:0], ts.Values...)
 		rs.Timestamps = append(rs.Timestamps[:0], ts.Timestamps...)
@@ -110,13 +110,39 @@ func timeseriesToResult(tss []*timeseries, maySort bool) ([]netstorage.Result, e
 
 	if maySort {
 		sort.Slice(result, func(i, j int) bool {
-			return string(result[i].MetricNameMarshaled) < string(result[j].MetricNameMarshaled)
+			return metricNameLess(&result[i].MetricName, &result[j].MetricName)
 		})
 	}
 
 	return result, nil
 }
 
+func metricNameLess(a, b *storage.MetricName) bool {
+	if string(a.MetricGroup) != string(b.MetricGroup) {
+		return string(a.MetricGroup) < string(b.MetricGroup)
+	}
+	// Metric names for a and b match. Compare tags.
+	// Tags must be already sorted by the caller, so just compare them.
+	ats := a.Tags
+	bts := b.Tags
+	for i := range ats {
+		if i >= len(bts) {
+			// a contains more tags than b and all the previous tags were identical,
+			// so a is considered bigger than b.
+			return false
+		}
+		at := &ats[i]
+		bt := &bts[i]
+		if string(at.Key) != string(bt.Key) {
+			return string(at.Key) < string(bt.Key)
+		}
+		if string(at.Value) != string(bt.Value) {
+			return string(at.Value) < string(bt.Value)
+		}
+	}
+	return len(ats) < len(bts)
+}
+
 func removeNaNs(tss []*timeseries) []*timeseries {
 	rvs := tss[:0]
 	for _, ts := range tss {

app/vmselect/promql/exec_test.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"math"
 	"testing"
 	"time"
 
@@ -132,6 +133,72 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("bitmap_and(0xB3, 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_and(0xB3, 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{17, 17, 17, 17, 17, 17},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_and(time(), 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_and(time(), 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 16, 16, 0, 0, 16},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_or(0xA2, 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_or(0xA2, 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{179, 179, 179, 179, 179, 179},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_or(time(), 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_or(time(), 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1017, 1201, 1401, 1617, 1817, 2001},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_xor(0xB3, 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_xor(0xB3, 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{162, 162, 162, 162, 162, 162},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_xor(time(), 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_xor(time(), 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1017, 1185, 1385, 1617, 1817, 1985},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("timezone_offset(UTC)", func(t *testing.T) {
 		t.Parallel()
 		q := `timezone_offset("UTC")`
@@ -698,6 +765,39 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run("present_over_time(time())", func(t *testing.T) {
+		t.Parallel()
+		q := `present_over_time(time())`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("present_over_time(time()[100:300])", func(t *testing.T) {
+		t.Parallel()
+		q := `present_over_time(time()[100:300])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1, nan, nan, 1, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("present_over_time(time()<10m)", func(t *testing.T) {
+		t.Parallel()
+		q := `present_over_time(time()<1600)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("absent(123)", func(t *testing.T) {
 		t.Parallel()
 		q := `absent(123)`
@@ -939,6 +1039,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("sinh()", func(t *testing.T) {
+		t.Parallel()
+		q := `sinh(pi()*(2000-time())/1000)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11.548739357257748, 6.132140673514712, 3.217113080357038, 1.6144880404748523, 0.6704839982471175, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("asin()", func(t *testing.T) {
 		t.Parallel()
 		q := `asin((2000-time())/1000)`
@@ -950,6 +1061,50 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("asinh(sinh)", func(t *testing.T) {
+		t.Parallel()
+		q := `asinh(sinh((2000-time())/1000))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 0.8000000000000002, 0.6, 0.4000000000000001, 0.2, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("atan2()", func(t *testing.T) {
+		t.Parallel()
+		q := `time() atan2 time()/10`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.07853981633974483, 0.07853981633974483, 0.07853981633974483, 0.07853981633974483, 0.07853981633974483, 0.07853981633974483},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("atan()", func(t *testing.T) {
+		t.Parallel()
+		q := `atan((2000-time())/1000)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.7853981633974483, 0.6747409422235526, 0.5404195002705842, 0.3805063771123649, 0.19739555984988078, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("atanh(tanh)", func(t *testing.T) {
+		t.Parallel()
+		q := `atanh(tanh((2000-time())/1000))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 0.8000000000000002, 0.6, 0.4000000000000001, 0.2, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("cos()", func(t *testing.T) {
 		t.Parallel()
 		q := `cos(pi()*(2000-time())/1000)`
@@ -972,6 +1127,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("acosh(cosh)", func(t *testing.T) {
+		t.Parallel()
+		q := `acosh(cosh((2000-time())/1000))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 0.8000000000000002, 0.5999999999999999, 0.40000000000000036, 0.20000000000000023, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("rad(deg)", func(t *testing.T) {
+		t.Parallel()
+		q := `rad(deg(time()/500))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2.3999999999999995, 2.8, 3.2, 3.6, 4},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("floor(time()/500)", func(t *testing.T) {
 		t.Parallel()
 		q := `floor(time()/500)`
@@ -1044,6 +1221,21 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("default_for_nan_series", func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(0, "foo", "bar")/0 default 7`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{7, 7, 7, 7, 7, 7},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`alias()`, func(t *testing.T) {
 		t.Parallel()
 		q := `alias(time(), "foobar")`
@@ -1691,10 +1883,12 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1123.456, 1323.456, 1523.456, 1723.456, 1923.456, 2123.456},
 			Timestamps: timestampsExpected,
 		}
-		r2.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("123.456"),
-		}}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("123.456"),
+			},
+		}
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
@@ -1838,9 +2032,9 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
-	t.Run(`sign(time()-1400)`, func(t *testing.T) {
+	t.Run(`sgn(time()-1400)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sign(time()-1400)`
+		q := `sgn(time()-1400)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{-1, -1, 0, 1, 1, 1},
@@ -2053,6 +2247,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`nan!=bool scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `(time() > 1234) !=bool 1400`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 0, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`scalar!=bool nan`, func(t *testing.T) {
+		t.Parallel()
+		q := `1400 !=bool (time() > 1234)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 0, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`scalar > time()`, func(t *testing.T) {
 		t.Parallel()
 		q := `123 > time()`
@@ -3312,6 +3528,46 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_quantiles()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label(histogram_quantiles("phi", 0.2, 0.3,
+			label_set(0, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+		), "phi")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("phi"),
+				Value: []byte("0.2"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{28, 28, 28, 28, 28, 28},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("phi"),
+				Value: []byte("0.3"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_share(normal-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_share(35,
@@ -4459,6 +4715,67 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`quantile_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `quantile_over_time(0.9, label_set(round(rand(0), 0.01), "__name__", "foo", "xx", "yy")[200s:5s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.893, 0.892, 0.9510000000000001, 0.8730000000000001, 0.9250000000000002, 0.891},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foo")
+		r.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`quantiles_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label(
+			quantiles_over_time("phi", 0.5, 0.9,
+				label_set(round(rand(0), 0.01), "__name__", "foo", "xx", "yy")[200s:5s]
+			),
+			"phi",
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.46499999999999997, 0.57, 0.485, 0.54, 0.555, 0.515},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("foo")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("phi"),
+				Value: []byte("0.5"),
+			},
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.893, 0.892, 0.9510000000000001, 0.8730000000000001, 0.9250000000000002, 0.891},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("foo")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("phi"),
+				Value: []byte("0.9"),
+			},
+			{
+				Key:   []byte("xx"),
+				Value: []byte("yy"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_by_label(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]), "vmrange")`
@@ -4753,7 +5070,7 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`limitk(1)`, func(t *testing.T) {
 		t.Parallel()
-		q := `limitk(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
+		q := `limitk(1, label_set(10, "foo", "bar") or label_set(time()/150, "xbaz", "sss"))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{10, 10, 10, 10, 10, 10},
@@ -4881,7 +5198,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(topk_min(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Values:     []float64{10, 10, 10, 10, 10, 10},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -4896,7 +5213,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(bottomk_min(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -4911,7 +5228,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `topk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -4926,7 +5243,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort_desc(topk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"), "remaining_sum=foo"))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -4952,7 +5269,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort_desc(topk_max(2, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"), "remaining_sum"))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -4978,7 +5295,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort_desc(topk_max(3, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"), "remaining_sum"))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -5004,7 +5321,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(bottomk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Values:     []float64{10, 10, 10, 10, 10, 10},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -5019,7 +5336,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(topk_avg(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -5049,7 +5366,22 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(topk_median(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`topk_last(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(topk_last(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -5061,10 +5393,25 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`bottomk_median(1)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort(bottomk_median(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		q := `sort(bottomk_median(1, label_set(10, "foo", "bar") or label_set(time()/15, "baz", "sss")))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`bottomk_last(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(bottomk_last(1, label_set(10, "foo", "bar") or label_set(time()/15, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -5242,7 +5589,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `distinct_over_time((time() < 1700)[500s])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{3, 3, 3, 3, nan, nan},
+			Values:     []float64{3, 3, 3, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -5253,7 +5600,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `distinct_over_time((time() < 1700)[2.5i])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{3, 3, 3, 3, nan, nan},
+			Values:     []float64{3, 3, 3, 3, 2, 1},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -5315,9 +5662,10 @@ func TestExecSuccess(t *testing.T) {
 	t.Run(`quantile(-2)`, func(t *testing.T) {
 		t.Parallel()
 		q := `quantile(-2, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
+		inf := math.Inf(-1)
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10, 10, 10},
+			Values:     []float64{inf, inf, inf, inf, inf, inf},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -5328,7 +5676,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `quantile(0.2, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10, 10, 10},
+			Values:     []float64{7.333333333333334, 8.4, 9.466666666666669, 10.133333333333333, 10.4, 10.666666666666668},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -5339,18 +5687,42 @@ func TestExecSuccess(t *testing.T) {
 		q := `quantile(0.5, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{8.333333333333334, 9, 9.666666666666668, 10.333333333333332, 11, 11.666666666666668},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`quantiles("phi", 0.2, 0.5)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(quantiles("phi", 0.2, 0.5, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{7.333333333333334, 8.4, 9.466666666666669, 10.133333333333333, 10.4, 10.666666666666668},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("phi"),
+			Value: []byte("0.2"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{8.333333333333334, 9, 9.666666666666668, 10.333333333333332, 11, 11.666666666666668},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("phi"),
+			Value: []byte("0.5"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`median()`, func(t *testing.T) {
 		t.Parallel()
 		q := `median(label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{8.333333333333334, 9, 9.666666666666668, 10.333333333333332, 11, 11.666666666666668},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -5370,9 +5742,10 @@ func TestExecSuccess(t *testing.T) {
 	t.Run(`quantile(3)`, func(t *testing.T) {
 		t.Parallel()
 		q := `quantile(3, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
+		inf := math.Inf(+1)
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, 10.666666666666666, 12, 13.333333333333334},
+			Values:     []float64{inf, inf, inf, inf, inf, inf},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -5384,6 +5757,47 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`mad()`, func(t *testing.T) {
+		t.Parallel()
+		q := `mad(
+			alias(time(), "metric1"),
+			alias(time()*1.5, "metric2"),
+			label_set(time()*0.9, "baz", "sss"),
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{100, 120, 140, 160, 180, 200},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`outliers_mad(1)`, func(t *testing.T) {
+		t.Parallel()
+		q := `outliers_mad(1, (
+			alias(time(), "metric1"),
+			alias(time()*1.5, "metric2"),
+			label_set(time()*0.9, "baz", "sss"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1500, 1800, 2100, 2400, 2700, 3000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("metric2")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`outliers_mad(5)`, func(t *testing.T) {
+		t.Parallel()
+		q := `outliers_mad(5, (
+			alias(time(), "metric1"),
+			alias(time()*1.5, "metric2"),
+			label_set(time()*0.9, "baz", "sss"),
+		))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`outliersk(0)`, func(t *testing.T) {
 		t.Parallel()
 		q := `outliersk(0, (
@@ -5443,7 +5857,8 @@ func TestExecSuccess(t *testing.T) {
 		q := `range_quantile(0.5, time())`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1600, 1600, 1600, 1600, 1600, 1600},
+			// time() results in [1000 1200 1400 1600 1800 2000]
+			Values:     []float64{1500, 1500, 1500, 1500, 1500, 1500},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -5454,7 +5869,8 @@ func TestExecSuccess(t *testing.T) {
 		q := `range_median(time())`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1600, 1600, 1600, 1600, 1600, 1600},
+			// time() results in [1000 1200 1400 1600 1800 2000]
+			Values:     []float64{1500, 1500, 1500, 1500, 1500, 1500},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -6102,12 +6518,13 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_candlestick()`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort(rollup_candlestick(round(rand(0),0.01)[:10s]))`
+		q := `sort(rollup_candlestick(alias(round(rand(0),0.01),"foobar")[:10s]))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{0.02, 0.02, 0.03, 0, 0.03, 0.02},
 			Timestamps: timestampsExpected,
 		}
+		r1.MetricName.MetricGroup = []byte("foobar")
 		r1.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
 			Value: []byte("low"),
@@ -6117,6 +6534,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{0.9, 0.32, 0.82, 0.13, 0.28, 0.86},
 			Timestamps: timestampsExpected,
 		}
+		r2.MetricName.MetricGroup = []byte("foobar")
 		r2.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
 			Value: []byte("open"),
@@ -6126,6 +6544,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{0.1, 0.04, 0.49, 0.46, 0.57, 0.92},
 			Timestamps: timestampsExpected,
 		}
+		r3.MetricName.MetricGroup = []byte("foobar")
 		r3.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
 			Value: []byte("close"),
@@ -6135,6 +6554,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{0.9, 0.94, 0.97, 0.93, 0.98, 0.92},
 			Timestamps: timestampsExpected,
 		}
+		r4.MetricName.MetricGroup = []byte("foobar")
 		r4.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
 			Value: []byte("high"),
@@ -6175,6 +6595,39 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
+	t.Run(`rollup_scrape_interval()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label(rollup_scrape_interval(1[5m:10s]), "rollup")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("avg"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("max"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("min"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`rollup()`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(rollup(time()[:50s]))`
@@ -6518,6 +6971,37 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4}
 		f(q, resultExpected)
 	})
+	t.Run(`count_values_big_numbers`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label(
+			count_values("xxx", (alias(772424014, "first"), alias(772424230, "second"))),
+			"xxx"
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("772424014"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("772424230"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`count_values`, func(t *testing.T) {
 		t.Parallel()
 		q := `count_values("xxx", label_set(10, "foo", "bar") or label_set(time()/100, "foo", "bar", "baz", "xx"))`
@@ -6637,7 +7121,8 @@ func TestExecSuccess(t *testing.T) {
 				Value: []byte("10"),
 			},
 		}
-		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		// expected sorted output for strings 1, 10, 2, 3
+		resultExpected := []netstorage.Result{r1, r4, r2, r3}
 		f(q, resultExpected)
 	})
 	t.Run(`count_values without (baz)`, func(t *testing.T) {
@@ -6691,6 +7176,44 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
+	t.Run(`result sorting`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(1, "instance", "localhost:1001", "type", "free")
+			or label_set(1, "instance", "localhost:1001", "type", "buffers")
+			or label_set(1, "instance", "localhost:1000", "type", "buffers")
+			or label_set(1, "instance", "localhost:1000", "type", "free")
+`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		testAddLabels(t, &r1.MetricName,
+			"instance", "localhost:1000", "type", "buffers")
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		testAddLabels(t, &r2.MetricName,
+			"instance", "localhost:1000", "type", "free")
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		testAddLabels(t, &r3.MetricName,
+			"instance", "localhost:1001", "type", "buffers")
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		testAddLabels(t, &r4.MetricName,
+			"instance", "localhost:1001", "type", "free")
+		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		f(q, resultExpected)
+	})
 }
 
 func TestExecError(t *testing.T) {
@@ -6755,7 +7278,7 @@ func TestExecError(t *testing.T) {
 	f(`label_mismatch()`)
 	f(`round()`)
 	f(`round(1,2,3)`)
-	f(`sign()`)
+	f(`sgn()`)
 	f(`scalar()`)
 	f(`sort(1,2)`)
 	f(`sort_desc()`)
@@ -6764,6 +7287,7 @@ func TestExecError(t *testing.T) {
 	f(`timestamp()`)
 	f(`vector()`)
 	f(`histogram_quantile()`)
+	f(`histogram_quantiles()`)
 	f(`sum()`)
 	f(`count_values()`)
 	f(`quantile()`)
@@ -6774,12 +7298,14 @@ func TestExecError(t *testing.T) {
 	f(`topk_max()`)
 	f(`topk_avg()`)
 	f(`topk_median()`)
+	f(`topk_last()`)
 	f(`limitk()`)
 	f(`bottomk()`)
 	f(`bottomk_min()`)
 	f(`bottomk_max()`)
 	f(`bottomk_avg()`)
 	f(`bottomk_median()`)
+	f(`bottomk_last()`)
 	f(`time(123)`)
 	f(`start(1)`)
 	f(`end(1)`)
@@ -6792,9 +7318,13 @@ func TestExecError(t *testing.T) {
 	f(`smooth_exponential(1)`)
 	f(`remove_resets()`)
 	f(`sin()`)
+	f(`sinh()`)
 	f(`cos()`)
+	f(`cosh()`)
 	f(`asin()`)
+	f(`asinh()`)
 	f(`acos()`)
+	f(`acosh()`)
 	f(`rand(123, 456)`)
 	f(`rand_normal(123, 456)`)
 	f(`rand_exponential(122, 456)`)
@@ -6825,6 +7355,9 @@ func TestExecError(t *testing.T) {
 	f(`hoeffding_bound_upper()`)
 	f(`hoeffding_bound_upper(1)`)
 	f(`hoeffding_bound_upper(0.99, foo, 1)`)
+	f(`mad()`)
+	f(`outliers_mad()`)
+	f(`outliers_mad(1)`)
 	f(`outliersk()`)
 	f(`outliersk(1)`)
 	f(`mode_over_time()`)
@@ -6841,6 +7374,11 @@ func TestExecError(t *testing.T) {
 	f(`count_gt_over_time()`)
 	f(`count_eq_over_time()`)
 	f(`count_ne_over_time()`)
+	f(`timezone_offset()`)
+	f(`bitmap_and()`)
+	f(`bitmap_or()`)
+	f(`bitmap_xor()`)
+	f(`quantiles()`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)
@@ -6941,19 +7479,34 @@ func testResultsEqual(t *testing.T, result, resultExpected []netstorage.Result)
 func testMetricNamesEqual(t *testing.T, mn, mnExpected *storage.MetricName, pos int) {
 	t.Helper()
 	if string(mn.MetricGroup) != string(mnExpected.MetricGroup) {
-		t.Fatalf(`unexpected MetricGroup at #%d; got %q; want %q`, pos, mn.MetricGroup, mnExpected.MetricGroup)
+		t.Fatalf(`unexpected MetricGroup at #%d; got %q; want %q; metricGot=%s, metricExpected=%s`,
+			pos, mn.MetricGroup, mnExpected.MetricGroup, mn.String(), mnExpected.String())
 	}
 	if len(mn.Tags) != len(mnExpected.Tags) {
-		t.Fatalf(`unexpected tags count at #%d; got %d; want %d`, pos, len(mn.Tags), len(mnExpected.Tags))
+		t.Fatalf(`unexpected tags count at #%d; got %d; want %d; metricGot=%s, metricExpected=%s`, pos, len(mn.Tags), len(mnExpected.Tags), mn.String(), mnExpected.String())
 	}
 	for i := range mn.Tags {
 		tag := &mn.Tags[i]
 		tagExpected := &mnExpected.Tags[i]
 		if string(tag.Key) != string(tagExpected.Key) {
-			t.Fatalf(`unexpected tag key at #%d,%d; got %q; want %q`, pos, i, tag.Key, tagExpected.Key)
+			t.Fatalf(`unexpected tag key at #%d,%d; got %q; want %q; metricGot=%s, metricExpected=%s`, pos, i, tag.Key, tagExpected.Key, mn.String(), mnExpected.String())
 		}
 		if string(tag.Value) != string(tagExpected.Value) {
-			t.Fatalf(`unexpected tag value for key %q at #%d,%d; got %q; want %q`, tag.Key, pos, i, tag.Value, tagExpected.Value)
+			t.Fatalf(`unexpected tag value for key %q at #%d,%d; got %q; want %q; metricGot=%s, metricExpected=%s`,
+				tag.Key, pos, i, tag.Value, tagExpected.Value, mn.String(), mnExpected.String())
 		}
 	}
 }
+
+func testAddLabels(t *testing.T, mn *storage.MetricName, labels ...string) {
+	t.Helper()
+	if len(labels)%2 > 0 {
+		t.Fatalf("uneven number of labels passed: %v", labels)
+	}
+	for i := 0; i < len(labels); i += 2 {
+		mn.Tags = append(mn.Tags, storage.Tag{
+			Key:   []byte(labels[i]),
+			Value: []byte(labels[i+1]),
+		})
+	}
+}

app/vmselect/promql/rollup.go

@@ -12,7 +12,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/VictoriaMetrics/metricsql"
-	"github.com/valyala/histogram"
 )
 
 var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The minimum interval for staleness calculations. "+
@@ -42,46 +41,49 @@ var rollupFuncs = map[string]newRollupFunc{
 	"stddev_over_time":   newRollupFuncOneArg(rollupStddev),
 	"stdvar_over_time":   newRollupFuncOneArg(rollupStdvar),
 	"absent_over_time":   newRollupFuncOneArg(rollupAbsent),
+	"present_over_time":  newRollupFuncOneArg(rollupPresent),
+	"last_over_time":     newRollupFuncOneArg(rollupLast),
 
 	// Additional rollup funcs.
-	"default_rollup":        newRollupFuncOneArg(rollupDefault), // default rollup func
-	"range_over_time":       newRollupFuncOneArg(rollupRange),
-	"sum2_over_time":        newRollupFuncOneArg(rollupSum2),
-	"geomean_over_time":     newRollupFuncOneArg(rollupGeomean),
-	"first_over_time":       newRollupFuncOneArg(rollupFirst),
-	"last_over_time":        newRollupFuncOneArg(rollupLast),
-	"distinct_over_time":    newRollupFuncOneArg(rollupDistinct),
-	"increases_over_time":   newRollupFuncOneArg(rollupIncreases),
-	"decreases_over_time":   newRollupFuncOneArg(rollupDecreases),
-	"increase_pure":         newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets
-	"integrate":             newRollupFuncOneArg(rollupIntegrate),
-	"ideriv":                newRollupFuncOneArg(rollupIderiv),
-	"lifetime":              newRollupFuncOneArg(rollupLifetime),
-	"lag":                   newRollupFuncOneArg(rollupLag),
-	"scrape_interval":       newRollupFuncOneArg(rollupScrapeInterval),
-	"tmin_over_time":        newRollupFuncOneArg(rollupTmin),
-	"tmax_over_time":        newRollupFuncOneArg(rollupTmax),
-	"tfirst_over_time":      newRollupFuncOneArg(rollupTfirst),
-	"tlast_over_time":       newRollupFuncOneArg(rollupTlast),
-	"share_le_over_time":    newRollupShareLE,
-	"share_gt_over_time":    newRollupShareGT,
-	"count_le_over_time":    newRollupCountLE,
-	"count_gt_over_time":    newRollupCountGT,
-	"count_eq_over_time":    newRollupCountEQ,
-	"count_ne_over_time":    newRollupCountNE,
-	"histogram_over_time":   newRollupFuncOneArg(rollupHistogram),
-	"rollup":                newRollupFuncOneArg(rollupFake),
-	"rollup_rate":           newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_deriv":          newRollupFuncOneArg(rollupFake),
-	"rollup_delta":          newRollupFuncOneArg(rollupFake),
-	"rollup_increase":       newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_candlestick":    newRollupFuncOneArg(rollupFake),
-	"aggr_over_time":        newRollupFuncTwoArgs(rollupFake),
-	"hoeffding_bound_upper": newRollupHoeffdingBoundUpper,
-	"hoeffding_bound_lower": newRollupHoeffdingBoundLower,
-	"ascent_over_time":      newRollupFuncOneArg(rollupAscentOverTime),
-	"descent_over_time":     newRollupFuncOneArg(rollupDescentOverTime),
-	"zscore_over_time":      newRollupFuncOneArg(rollupZScoreOverTime),
+	"default_rollup":         newRollupFuncOneArg(rollupDefault), // default rollup func
+	"range_over_time":        newRollupFuncOneArg(rollupRange),
+	"sum2_over_time":         newRollupFuncOneArg(rollupSum2),
+	"geomean_over_time":      newRollupFuncOneArg(rollupGeomean),
+	"first_over_time":        newRollupFuncOneArg(rollupFirst),
+	"distinct_over_time":     newRollupFuncOneArg(rollupDistinct),
+	"increases_over_time":    newRollupFuncOneArg(rollupIncreases),
+	"decreases_over_time":    newRollupFuncOneArg(rollupDecreases),
+	"increase_pure":          newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets
+	"integrate":              newRollupFuncOneArg(rollupIntegrate),
+	"ideriv":                 newRollupFuncOneArg(rollupIderiv),
+	"lifetime":               newRollupFuncOneArg(rollupLifetime),
+	"lag":                    newRollupFuncOneArg(rollupLag),
+	"scrape_interval":        newRollupFuncOneArg(rollupScrapeInterval),
+	"tmin_over_time":         newRollupFuncOneArg(rollupTmin),
+	"tmax_over_time":         newRollupFuncOneArg(rollupTmax),
+	"tfirst_over_time":       newRollupFuncOneArg(rollupTfirst),
+	"tlast_over_time":        newRollupFuncOneArg(rollupTlast),
+	"share_le_over_time":     newRollupShareLE,
+	"share_gt_over_time":     newRollupShareGT,
+	"count_le_over_time":     newRollupCountLE,
+	"count_gt_over_time":     newRollupCountGT,
+	"count_eq_over_time":     newRollupCountEQ,
+	"count_ne_over_time":     newRollupCountNE,
+	"histogram_over_time":    newRollupFuncOneArg(rollupHistogram),
+	"rollup":                 newRollupFuncOneArg(rollupFake),
+	"rollup_rate":            newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_deriv":           newRollupFuncOneArg(rollupFake),
+	"rollup_delta":           newRollupFuncOneArg(rollupFake),
+	"rollup_increase":        newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_candlestick":     newRollupFuncOneArg(rollupFake),
+	"rollup_scrape_interval": newRollupFuncOneArg(rollupFake),
+	"aggr_over_time":         newRollupFuncTwoArgs(rollupFake),
+	"hoeffding_bound_upper":  newRollupHoeffdingBoundUpper,
+	"hoeffding_bound_lower":  newRollupHoeffdingBoundLower,
+	"ascent_over_time":       newRollupFuncOneArg(rollupAscentOverTime),
+	"descent_over_time":      newRollupFuncOneArg(rollupDescentOverTime),
+	"zscore_over_time":       newRollupFuncOneArg(rollupZScoreOverTime),
+	"quantiles_over_time":    newRollupQuantiles,
 
 	// `timestamp` function must return timestamp for the last datapoint on the current window
 	// in order to properly handle offset and timestamps unaligned to the current step.
@@ -97,23 +99,24 @@ var rollupFuncs = map[string]newRollupFunc{
 // rollupAggrFuncs are functions that can be passed to `aggr_over_time()`
 var rollupAggrFuncs = map[string]rollupFunc{
 	// Standard rollup funcs from PromQL.
-	"changes":          rollupChanges,
-	"delta":            rollupDelta,
-	"deriv":            rollupDerivSlow,
-	"deriv_fast":       rollupDerivFast,
-	"idelta":           rollupIdelta,
-	"increase":         rollupDelta,     // + rollupFuncsRemoveCounterResets
-	"irate":            rollupIderiv,    // + rollupFuncsRemoveCounterResets
-	"rate":             rollupDerivFast, // + rollupFuncsRemoveCounterResets
-	"resets":           rollupResets,
-	"avg_over_time":    rollupAvg,
-	"min_over_time":    rollupMin,
-	"max_over_time":    rollupMax,
-	"sum_over_time":    rollupSum,
-	"count_over_time":  rollupCount,
-	"stddev_over_time": rollupStddev,
-	"stdvar_over_time": rollupStdvar,
-	"absent_over_time": rollupAbsent,
+	"changes":           rollupChanges,
+	"delta":             rollupDelta,
+	"deriv":             rollupDerivSlow,
+	"deriv_fast":        rollupDerivFast,
+	"idelta":            rollupIdelta,
+	"increase":          rollupDelta,     // + rollupFuncsRemoveCounterResets
+	"irate":             rollupIderiv,    // + rollupFuncsRemoveCounterResets
+	"rate":              rollupDerivFast, // + rollupFuncsRemoveCounterResets
+	"resets":            rollupResets,
+	"avg_over_time":     rollupAvg,
+	"min_over_time":     rollupMin,
+	"max_over_time":     rollupMax,
+	"sum_over_time":     rollupSum,
+	"count_over_time":   rollupCount,
+	"stddev_over_time":  rollupStddev,
+	"stdvar_over_time":  rollupStdvar,
+	"absent_over_time":  rollupAbsent,
+	"present_over_time": rollupPresent,
 
 	// Additional rollup funcs.
 	"range_over_time":     rollupRange,
@@ -142,31 +145,26 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"rate_over_sum":       rollupRateOverSum,
 }
 
-var rollupFuncsCannotAdjustWindow = map[string]bool{
-	"changes":             true,
-	"delta":               true,
-	"holt_winters":        true,
-	"idelta":              true,
-	"increase":            true,
-	"predict_linear":      true,
-	"resets":              true,
-	"avg_over_time":       true,
-	"sum_over_time":       true,
-	"count_over_time":     true,
-	"quantile_over_time":  true,
-	"stddev_over_time":    true,
-	"stdvar_over_time":    true,
-	"absent_over_time":    true,
-	"sum2_over_time":      true,
-	"geomean_over_time":   true,
-	"distinct_over_time":  true,
-	"increases_over_time": true,
-	"decreases_over_time": true,
-	"increase_pure":       true,
-	"integrate":           true,
-	"ascent_over_time":    true,
-	"descent_over_time":   true,
-	"zscore_over_time":    true,
+// VictoriaMetrics can increase lookbehind window in square brackets for these functions
+// if the given window doesn't contain enough samples for calculations.
+//
+// This is needed in order to return the expected non-empty graphs when zooming in the graph in Grafana,
+// which is built with `func_name(metric[$__interval])` query.
+var rollupFuncsCanAdjustWindow = map[string]bool{
+	"default_rollup":         true,
+	"deriv":                  true,
+	"deriv_fast":             true,
+	"ideriv":                 true,
+	"irate":                  true,
+	"rate":                   true,
+	"rate_over_sum":          true,
+	"rollup":                 true,
+	"rollup_candlestick":     true,
+	"rollup_deriv":           true,
+	"rollup_rate":            true,
+	"rollup_scrape_interval": true,
+	"scrape_interval":        true,
+	"timestamp":              true,
 }
 
 var rollupFuncsRemoveCounterResets = map[string]bool{
@@ -186,6 +184,7 @@ var rollupFuncsKeepMetricGroup = map[string]bool{
 	"min_over_time":         true,
 	"max_over_time":         true,
 	"quantile_over_time":    true,
+	"quantiles_over_time":   true,
 	"rollup":                true,
 	"geomean_over_time":     true,
 	"hoeffding_bound_lower": true,
@@ -193,6 +192,7 @@ var rollupFuncsKeepMetricGroup = map[string]bool{
 	"first_over_time":       true,
 	"last_over_time":        true,
 	"mode_over_time":        true,
+	"rollup_candlestick":    true,
 }
 
 func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
@@ -244,15 +244,17 @@ func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
 	return aggrFuncNames, nil
 }
 
-func getRollupArgIdx(funcName string) int {
-	funcName = strings.ToLower(funcName)
+func getRollupArgIdx(fe *metricsql.FuncExpr) int {
+	funcName := strings.ToLower(fe.Name)
 	if rollupFuncs[funcName] == nil {
-		logger.Panicf("BUG: getRollupArgIdx is called for non-rollup func %q", funcName)
+		logger.Panicf("BUG: getRollupArgIdx is called for non-rollup func %q", fe.Name)
 	}
 	switch funcName {
 	case "quantile_over_time", "aggr_over_time",
 		"hoeffding_bound_lower", "hoeffding_bound_upper":
 		return 1
+	case "quantiles_over_time":
+		return len(fe.Args) - 1
 	default:
 		return 0
 	}
@@ -268,16 +270,16 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en
 	}
 	newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig {
 		return &rollupConfig{
-			TagValue:          tagValue,
-			Func:              rf,
-			Start:             start,
-			End:               end,
-			Step:              step,
-			Window:            window,
-			MayAdjustWindow:   !rollupFuncsCannotAdjustWindow[name],
-			CanDropLastSample: name == "default_rollup",
-			LookbackDelta:     lookbackDelta,
-			Timestamps:        sharedTimestamps,
+			TagValue:        tagValue,
+			Func:            rf,
+			Start:           start,
+			End:             end,
+			Step:            step,
+			Window:          window,
+			MayAdjustWindow: rollupFuncsCanAdjustWindow[name],
+			LookbackDelta:   lookbackDelta,
+			Timestamps:      sharedTimestamps,
+			isDefaultRollup: name == "default_rollup",
 		}
 	}
 	appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig {
@@ -309,6 +311,24 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en
 		rcs = append(rcs, newRollupConfig(rollupClose, "close"))
 		rcs = append(rcs, newRollupConfig(rollupLow, "low"))
 		rcs = append(rcs, newRollupConfig(rollupHigh, "high"))
+	case "rollup_scrape_interval":
+		preFuncPrev := preFunc
+		preFunc = func(values []float64, timestamps []int64) {
+			preFuncPrev(values, timestamps)
+			// Calculate intervals in seconds between samples.
+			tsSecsPrev := nan
+			for i, ts := range timestamps {
+				tsSecs := float64(ts) / 1000
+				values[i] = tsSecs - tsSecsPrev
+				tsSecsPrev = tsSecs
+			}
+			if len(values) > 1 {
+				// Overwrite the first NaN interval with the second interval,
+				// So min, max and avg rollups could be calculated properly, since they don't expect to receive NaNs.
+				values[0] = values[1]
+			}
+		}
+		rcs = appendRollupConfigs(rcs)
 	case "aggr_over_time":
 		aggrFuncNames, err := getRollupAggrFuncNames(expr)
 		if err != nil {
@@ -399,15 +419,13 @@ type rollupConfig struct {
 	// when using window smaller than 2 x scrape_interval.
 	MayAdjustWindow bool
 
-	// Whether the last sample can be dropped during rollup calculations.
-	// The last sample can be dropped for `default_rollup()` function only.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748 .
-	CanDropLastSample bool
-
 	Timestamps []int64
 
 	// LoookbackDelta is the analog to `-query.lookback-delta` from Prometheus world.
 	LookbackDelta int64
+
+	// Whether default_rollup is used.
+	isDefaultRollup bool
 }
 
 var (
@@ -419,14 +437,16 @@ var (
 const maxSilenceInterval = 5 * 60 * 1000
 
 type timeseriesMap struct {
-	origin    *timeseries
-	labelName string
-	h         metrics.Histogram
-	m         map[string]*timeseries
+	origin *timeseries
+	h      metrics.Histogram
+	m      map[string]*timeseries
 }
 
 func newTimeseriesMap(funcName string, sharedTimestamps []int64, mnSrc *storage.MetricName) *timeseriesMap {
-	if strings.ToLower(funcName) != "histogram_over_time" {
+	funcName = strings.ToLower(funcName)
+	switch funcName {
+	case "histogram_over_time", "quantiles_over_time":
+	default:
 		return nil
 	}
 
@@ -436,13 +456,14 @@ func newTimeseriesMap(funcName string, sharedTimestamps []int64, mnSrc *storage.
 	}
 	var origin timeseries
 	origin.MetricName.CopyFrom(mnSrc)
-	origin.MetricName.ResetMetricGroup()
+	if !rollupFuncsKeepMetricGroup[funcName] {
+		origin.MetricName.ResetMetricGroup()
+	}
 	origin.Timestamps = sharedTimestamps
 	origin.Values = values
 	return &timeseriesMap{
-		origin:    &origin,
-		labelName: "vmrange",
-		m:         make(map[string]*timeseries),
+		origin: &origin,
+		m:      make(map[string]*timeseries),
 	}
 }
 
@@ -453,15 +474,15 @@ func (tsm *timeseriesMap) AppendTimeseriesTo(dst []*timeseries) []*timeseries {
 	return dst
 }
 
-func (tsm *timeseriesMap) GetOrCreateTimeseries(labelValue string) *timeseries {
+func (tsm *timeseriesMap) GetOrCreateTimeseries(labelName, labelValue string) *timeseries {
 	ts := tsm.m[labelValue]
 	if ts != nil {
 		return ts
 	}
 	ts = &timeseries{}
 	ts.CopyFromShallowTimestamps(tsm.origin)
-	ts.MetricName.RemoveTag(tsm.labelName)
-	ts.MetricName.AddTag(tsm.labelName, labelValue)
+	ts.MetricName.RemoveTag(labelName)
+	ts.MetricName.AddTag(labelName, labelValue)
 	tsm.m[labelValue] = ts
 	return ts
 }
@@ -516,7 +537,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	window := rc.Window
 	if window <= 0 {
 		window = rc.Step
-		if rc.CanDropLastSample && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
+		if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
 			// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
 			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 			window = rc.LookbackDelta
@@ -534,10 +555,6 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	j := 0
 	ni := 0
 	nj := 0
-	stalenessInterval := int64(float64(scrapeInterval) * 0.9)
-	// Do not drop trailing data points for queries, which return 2 or 1 point (aka instant queries).
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
-	canDropLastSample := rc.CanDropLastSample && len(rc.Timestamps) > 2
 	f := rc.Func
 	for _, tEnd := range rc.Timestamps {
 		tStart := tEnd - window
@@ -557,16 +574,6 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		}
 		rfa.values = values[i:j]
 		rfa.timestamps = timestamps[i:j]
-		if canDropLastSample && j == len(timestamps) && j > 0 && (tEnd-timestamps[j-1] > stalenessInterval || i == j && len(timestamps) == 1) {
-			// Drop trailing data points in the following cases:
-			// - if the distance between the last raw sample and tEnd exceeds stalenessInterval
-			// - if time series contains only a single raw sample
-			// This should prevent from double counting when a label changes in time series (for instance,
-			// during new deployment in K8S). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
-			rfa.prevValue = nan
-			rfa.values = nil
-			rfa.timestamps = nil
-		}
 		if i > 0 {
 			rfa.realPrevValue = values[i-1]
 		} else {
@@ -645,18 +652,20 @@ func getScrapeInterval(timestamps []int64) int64 {
 	}
 
 	// Estimate scrape interval as 0.6 quantile for the first 20 intervals.
-	h := histogram.GetFast()
 	tsPrev := timestamps[0]
 	timestamps = timestamps[1:]
 	if len(timestamps) > 20 {
 		timestamps = timestamps[:20]
 	}
+	a := getFloat64s()
+	intervals := a.A[:0]
 	for _, ts := range timestamps {
-		h.Update(float64(ts - tsPrev))
+		intervals = append(intervals, float64(ts-tsPrev))
 		tsPrev = ts
 	}
-	scrapeInterval := int64(h.Quantile(0.6))
-	histogram.PutFast(h)
+	scrapeInterval := int64(quantile(0.6, intervals))
+	a.A = intervals
+	putFloat64s(a)
 	if scrapeInterval <= 0 {
 		return int64(maxSilenceInterval)
 	}
@@ -844,37 +853,29 @@ func linearRegression(rfa *rollupFuncArg) (float64, float64) {
 	// before calling rollup funcs.
 	values := rfa.values
 	timestamps := rfa.timestamps
-	if len(values) == 0 {
-		return rfa.prevValue, 0
+	n := float64(len(values))
+	if n == 0 {
+		return nan, nan
+	}
+	if n == 1 {
+		return values[0], 0
 	}
 
 	// See https://en.wikipedia.org/wiki/Simple_linear_regression#Numerical_example
-	tFirst := rfa.prevTimestamp
-	vSum := rfa.prevValue
+	interceptTime := rfa.currTimestamp
+	vSum := float64(0)
 	tSum := float64(0)
 	tvSum := float64(0)
 	ttSum := float64(0)
-	n := 1.0
-	if math.IsNaN(rfa.prevValue) {
-		tFirst = timestamps[0]
-		vSum = 0
-		n = 0
-	}
 	for i, v := range values {
-		dt := float64(timestamps[i]-tFirst) / 1e3
+		dt := float64(timestamps[i]-interceptTime) / 1e3
 		vSum += v
 		tSum += dt
 		tvSum += dt * v
 		ttSum += dt * dt
 	}
-	n += float64(len(values))
-	if n == 1 {
-		return vSum, 0
-	}
-	k := (n*tvSum - tSum*vSum) / (n*ttSum - tSum*tSum)
-	v := (vSum - k*tSum) / n
-	// Adjust v to the last timestamp on the given time range.
-	v += k * (float64(timestamps[len(timestamps)-1]-tFirst) / 1e3)
+	k := (tvSum - tSum*vSum/n) / (ttSum - tSum*tSum/n)
+	v := vSum/n - k*tSum/n
 	return v, k
 }
 
@@ -1034,14 +1035,29 @@ func rollupHoeffdingBoundInternal(rfa *rollupFuncArg, phis []float64) (float64,
 	return bound, vAvg
 }
 
-func newRollupQuantile(args []interface{}) (rollupFunc, error) {
-	if err := expectRollupArgsNum(args, 2); err != nil {
-		return nil, err
+func newRollupQuantiles(args []interface{}) (rollupFunc, error) {
+	if len(args) < 3 {
+		return nil, fmt.Errorf("unexpected number of args: %d; want at least 3 args", len(args))
 	}
-	phis, err := getScalar(args[0], 0)
+	tssPhi, ok := args[0].([]*timeseries)
+	if !ok {
+		return nil, fmt.Errorf("unexpected type for phi arg: %T; want string", args[0])
+	}
+	phiLabel, err := getString(tssPhi, 0)
 	if err != nil {
 		return nil, err
 	}
+	phiArgs := args[1 : len(args)-1]
+	phis := make([]float64, len(phiArgs))
+	phiStrs := make([]string, len(phiArgs))
+	for i, phiArg := range phiArgs {
+		phiValues, err := getScalar(phiArg, i+1)
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain phi from arg #%d: %w", i+1, err)
+		}
+		phis[i] = phiValues[0]
+		phiStrs[i] = fmt.Sprintf("%g", phiValues[0])
+	}
 	rf := func(rfa *rollupFuncArg) float64 {
 		// There is no need in handling NaNs here, since they must be cleaned up
 		// before calling rollup funcs.
@@ -1053,13 +1069,34 @@ func newRollupQuantile(args []interface{}) (rollupFunc, error) {
 			// Fast path - only a single value.
 			return values[0]
 		}
-		hf := histogram.GetFast()
-		for _, v := range values {
-			hf.Update(v)
+		qs := getFloat64s()
+		qs.A = quantiles(qs.A[:0], phis, values)
+		idx := rfa.idx
+		tsm := rfa.tsm
+		for i, phiStr := range phiStrs {
+			ts := tsm.GetOrCreateTimeseries(phiLabel, phiStr)
+			ts.Values[idx] = qs.A[i]
 		}
+		putFloat64s(qs)
+		return nan
+	}
+	return rf, nil
+}
+
+func newRollupQuantile(args []interface{}) (rollupFunc, error) {
+	if err := expectRollupArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	phis, err := getScalar(args[0], 0)
+	if err != nil {
+		return nil, err
+	}
+	rf := func(rfa *rollupFuncArg) float64 {
+		// There is no need in handling NaNs here, since they must be cleaned up
+		// before calling rollup funcs.
+		values := rfa.values
 		phi := phis[rfa.idx]
-		qv := hf.Quantile(phi)
-		histogram.PutFast(hf)
+		qv := quantile(phi, values)
 		return qv
 	}
 	return rf, nil
@@ -1074,7 +1111,7 @@ func rollupHistogram(rfa *rollupFuncArg) float64 {
 	}
 	idx := rfa.idx
 	tsm.h.VisitNonZeroBuckets(func(vmrange string, count uint64) {
-		ts := tsm.GetOrCreateTimeseries(vmrange)
+		ts := tsm.GetOrCreateTimeseries("vmrange", vmrange)
 		ts.Values[idx] = float64(count)
 	})
 	return nan
@@ -1284,15 +1321,21 @@ func rollupAbsent(rfa *rollupFuncArg) float64 {
 	return nan
 }
 
+func rollupPresent(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	if len(rfa.values) > 0 {
+		return 1
+	}
+	return nan
+}
+
 func rollupCount(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		if math.IsNaN(rfa.prevValue) {
-			return nan
-		}
-		return 0
+		return nan
 	}
 	return float64(len(values))
 }
@@ -1309,14 +1352,11 @@ func rollupStdvar(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		if math.IsNaN(rfa.prevValue) {
-			return nan
-		}
-		return 0
+		return nan
 	}
 	if len(values) == 1 {
 		// Fast path.
-		return values[0]
+		return 0
 	}
 	var avg float64
 	var count float64
@@ -1344,7 +1384,7 @@ func rollupIncreasePure(rfa *rollupFuncArg) float64 {
 		prevValue = 0
 	}
 	if len(values) == 0 {
-		// Assume the counter didsn't change since prevValue.
+		// Assume the counter didn't change since prevValue.
 		return 0
 	}
 	return values[len(values)-1] - prevValue
@@ -1724,19 +1764,28 @@ func rollupModeOverTime(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 
 	// Copy rfa.values to a.A, since modeNoNaNs modifies a.A contents.
-	a := float64sPool.Get().(*float64s)
+	a := getFloat64s()
 	a.A = append(a.A[:0], rfa.values...)
 	result := modeNoNaNs(rfa.prevValue, a.A)
-	float64sPool.Put(a)
+	putFloat64s(a)
 	return result
 }
 
-var float64sPool = &sync.Pool{
-	New: func() interface{} {
-		return &float64s{}
-	},
+func getFloat64s() *float64s {
+	v := float64sPool.Get()
+	if v == nil {
+		v = &float64s{}
+	}
+	return v.(*float64s)
 }
 
+func putFloat64s(a *float64s) {
+	a.A = a.A[:0]
+	float64sPool.Put(a)
+}
+
+var float64sPool sync.Pool
+
 type float64s struct {
 	A []float64
 }
@@ -1814,11 +1863,20 @@ func rollupFirst(rfa *rollupFuncArg) float64 {
 	return values[0]
 }
 
-var rollupDefault = rollupLast
+func rollupDefault(rfa *rollupFuncArg) float64 {
+	values := rfa.values
+	if len(values) == 0 {
+		// Do not take into account rfa.prevValue, since it may lead
+		// to inconsistent results comparing to Prometheus on broken time series
+		// with irregular data points.
+		return nan
+	}
+	// Intentionally do not skip the possible last Prometheus staleness mark.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1526 .
+	return values[len(values)-1]
+}
 
 func rollupLast(rfa *rollupFuncArg) float64 {
-	// There is no need in handling NaNs here, since they must be cleaned up
-	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
 		// Do not take into account rfa.prevValue, since it may lead

app/vmselect/promql/rollup_result_cache.go

@@ -23,11 +23,18 @@ import (
 var (
 	cacheTimestampOffset = flag.Duration("search.cacheTimestampOffset", 5*time.Minute, "The maximum duration since the current time for response data, "+
 		"which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses "+
-		"due to time synchronization issues between VictoriaMetrics and data sources")
+		"due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset")
+	disableAutoCacheReset = flag.Bool("search.disableAutoCacheReset", false, "Whether to disable automatic response cache reset if a sample with timestamp "+
+		"outside -search.cacheTimestampOffset is inserted into VictoriaMetrics")
 )
 
 // ResetRollupResultCacheIfNeeded resets rollup result cache if mrs contains timestamps outside `now - search.cacheTimestampOffset`.
 func ResetRollupResultCacheIfNeeded(mrs []storage.MetricRow) {
+	if *disableAutoCacheReset {
+		// Do not reset response cache if -search.disableAutoCacheReset is set.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1570 .
+		return
+	}
 	checkRollupResultCacheResetOnce.Do(func() {
 		rollupResultResetMetricRowSample.Store(&storage.MetricRow{})
 		go checkRollupResultCacheReset()

app/vmselect/promql/rollup_test.go

@@ -335,14 +335,14 @@ func TestRollupQuantileOverTime(t *testing.T) {
 		testRollupFunc(t, "quantile_over_time", args, &me, vExpected)
 	}
 
-	f(-123, 12)
-	f(-0.5, 12)
+	f(-123, math.Inf(-1))
+	f(-0.5, math.Inf(-1))
 	f(0, 12)
-	f(0.1, 21)
+	f(0.1, 22.1)
 	f(0.5, 34)
-	f(0.9, 99)
+	f(0.9, 94.50000000000001)
 	f(1, 123)
-	f(234, 123)
+	f(234, math.Inf(+1))
 }
 
 func TestRollupPredictLinear(t *testing.T) {
@@ -357,10 +357,32 @@ func TestRollupPredictLinear(t *testing.T) {
 		testRollupFunc(t, "predict_linear", args, &me, vExpected)
 	}
 
-	f(0e-3, 30.382432471845043)
-	f(50e-3, 17.03950235614201)
-	f(100e-3, 3.696572240438975)
-	f(200e-3, -22.989287990967092)
+	f(0e-3, 65.07405077267295)
+	f(50e-3, 51.7311206569699)
+	f(100e-3, 38.38819054126685)
+	f(200e-3, 11.702330309860756)
+}
+
+func TestLinearRegression(t *testing.T) {
+	f := func(values []float64, timestamps []int64, expV, expK float64) {
+		t.Helper()
+		rfa := &rollupFuncArg{
+			values:        values,
+			timestamps:    timestamps,
+			currTimestamp: timestamps[0] + 100,
+		}
+		v, k := linearRegression(rfa)
+		if err := compareValues([]float64{v}, []float64{expV}); err != nil {
+			t.Fatalf("unexpected v err: %s", err)
+		}
+		if err := compareValues([]float64{k}, []float64{expK}); err != nil {
+			t.Fatalf("unexpected k err: %s", err)
+		}
+	}
+
+	f([]float64{1}, []int64{1}, math.NaN(), math.NaN())
+	f([]float64{1, 2}, []int64{100, 300}, 1.5, 5)
+	f([]float64{2, 4, 6, 8, 10}, []int64{100, 200, 300, 400, 500}, 4, 20)
 }
 
 func TestRollupHoltWinters(t *testing.T) {
@@ -448,7 +470,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("default_rollup", 34)
 	f("changes", 11)
 	f("delta", 34)
-	f("deriv", -266.85860231406065)
+	f("deriv", -266.85860231406093)
 	f("deriv_fast", -712)
 	f("idelta", 0)
 	f("increase", 398)
@@ -508,6 +530,7 @@ func TestRollupNewRollupFuncError(t *testing.T) {
 	f("holt_winters", nil)
 	f("predict_linear", nil)
 	f("quantile_over_time", nil)
+	f("quantiles_over_time", nil)
 
 	// Invalid arg type
 	scalarTs := []*timeseries{{
@@ -521,6 +544,7 @@ func TestRollupNewRollupFuncError(t *testing.T) {
 	f("predict_linear", []interface{}{123, 123})
 	f("predict_linear", []interface{}{me, 123})
 	f("quantile_over_time", []interface{}{123, 123})
+	f("quantiles_over_time", []interface{}{123, 123})
 }
 
 func TestRollupNoWindowNoPoints(t *testing.T) {
@@ -624,7 +648,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, 123, 34, nan}
+		valuesExpected := []float64{nan, nan, 123, 34, 32}
 		timestampsExpected := []int64{-50, 0, 50, 100, 150}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -731,7 +755,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 54, 44, nan}
+		valuesExpected := []float64{nan, 123, 54, 44, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -745,7 +769,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, nan}
+		valuesExpected := []float64{nan, 4, 4, 3, 1}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -759,7 +783,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 21, 12, 32, nan}
+		valuesExpected := []float64{nan, 21, 12, 32, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -773,7 +797,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 99, 44, nan}
+		valuesExpected := []float64{nan, 123, 99, 44, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -787,7 +811,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 222, 199, 110, nan}
+		valuesExpected := []float64{nan, 222, 199, 110, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -801,7 +825,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, -9, 22, nan}
+		valuesExpected := []float64{nan, 21, -9, 22, 0}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -829,7 +853,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.004, 0, 0, nan}
+		valuesExpected := []float64{nan, 0.004, 0, 0, 0.03}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -843,7 +867,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.031, 0.044, 0.04, nan}
+		valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -857,7 +881,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.031, 0.075, 0.115, nan}
+		valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -871,7 +895,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, nan}
+		valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -885,7 +909,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, nan}
+		valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -899,7 +923,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, nan}
+		valuesExpected := []float64{nan, 4, 4, 3, 0}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -927,7 +951,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 2, 2, 1, nan}
+		valuesExpected := []float64{nan, 2, 2, 1, 0}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -941,7 +965,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, nan}
+		valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -955,7 +979,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, nan}
+		valuesExpected := []float64{nan, -2879.310344827588, 127.87627310448904, -496.5831435079728, 0}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -983,7 +1007,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, nan}
+		valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -997,7 +1021,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, nan}
+		valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 0}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1011,7 +1035,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, nan}
+		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1025,7 +1049,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, nan}
+		valuesExpected := []float64{nan, 4, 4, 3, 1}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1039,7 +1063,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 7, 6, nan}
+		valuesExpected := []float64{nan, 4, 7, 6, 3}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1053,7 +1077,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 21, 34, 34, nan}
+		valuesExpected := []float64{nan, 21, 34, 34, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1067,7 +1091,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, nan}
+		valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, 2880}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1103,7 +1127,7 @@ func TestRollupBigNumberOfValues(t *testing.T) {
 		srcTimestamps[i] = int64(i / 2)
 	}
 	values := rc.Do(nil, srcValues, srcTimestamps)
-	valuesExpected := []float64{1, 4001, 8001, nan, nan, nan}
+	valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan}
 	timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000}
 	testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 }
@@ -1136,6 +1160,13 @@ func testRowsEqual(t *testing.T, values []float64, timestamps []int64, valuesExp
 			}
 			continue
 		}
+		if math.IsNaN(vExpected) {
+			if !math.IsNaN(v) {
+				t.Fatalf("unexpected value at values[%d]; got %f; want nan\nvalues=\n%v\nvaluesExpected=\n%v",
+					i, v, values, valuesExpected)
+			}
+			continue
+		}
 		if math.Abs(v-vExpected) > 1e-15 {
 			t.Fatalf("unexpected value at values[%d]; got %f; want %f\nvalues=\n%v\nvaluesExpected=\n%v",
 				i, v, vExpected, values, valuesExpected)

app/vmselect/promql/transform.go

@@ -14,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metricsql"
-	"github.com/valyala/histogram"
 )
 
 var transformFuncsKeepMetricGroup = map[string]bool{
@@ -44,13 +43,22 @@ var transformFuncs = map[string]transformFunc{
 	// See funcs accepting instant-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
 	"abs":                newTransformFuncOneArg(transformAbs),
 	"absent":             transformAbsent,
+	"acos":               newTransformFuncOneArg(transformAcos),
+	"acosh":              newTransformFuncOneArg(transformAcosh),
+	"asin":               newTransformFuncOneArg(transformAsin),
+	"asinh":              newTransformFuncOneArg(transformAsinh),
+	"atan":               newTransformFuncOneArg(transformAtan),
+	"atanh":              newTransformFuncOneArg(transformAtanh),
 	"ceil":               newTransformFuncOneArg(transformCeil),
 	"clamp":              transformClamp,
 	"clamp_max":          transformClampMax,
 	"clamp_min":          transformClampMin,
+	"cos":                newTransformFuncOneArg(transformCos),
+	"cosh":               newTransformFuncOneArg(transformCosh),
 	"day_of_month":       newTransformFuncDateTime(transformDayOfMonth),
 	"day_of_week":        newTransformFuncDateTime(transformDayOfWeek),
 	"days_in_month":      newTransformFuncDateTime(transformDaysInMonth),
+	"deg":                newTransformFuncOneArg(transformDeg),
 	"exp":                newTransformFuncOneArg(transformExp),
 	"floor":              newTransformFuncOneArg(transformFloor),
 	"histogram_quantile": transformHistogramQuantile,
@@ -62,68 +70,73 @@ var transformFuncs = map[string]transformFunc{
 	"log10":              newTransformFuncOneArg(transformLog10),
 	"minute":             newTransformFuncDateTime(transformMinute),
 	"month":              newTransformFuncDateTime(transformMonth),
+	"pi":                 transformPi,
+	"rad":                newTransformFuncOneArg(transformRad),
 	"round":              transformRound,
-	"sign":               transformSign,
 	"scalar":             transformScalar,
+	"sgn":                transformSgn,
+	"sin":                newTransformFuncOneArg(transformSin),
+	"sinh":               newTransformFuncOneArg(transformSinh),
 	"sort":               newTransformFuncSort(false),
 	"sort_desc":          newTransformFuncSort(true),
 	"sqrt":               newTransformFuncOneArg(transformSqrt),
+	"tan":                newTransformFuncOneArg(transformTan),
+	"tanh":               newTransformFuncOneArg(transformTanh),
 	"time":               transformTime,
 	// "timestamp" has been moved to rollup funcs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415
 	"vector": transformVector,
 	"year":   newTransformFuncDateTime(transformYear),
 
 	// New funcs
-	"label_set":          transformLabelSet,
-	"label_map":          transformLabelMap,
-	"label_uppercase":    transformLabelUppercase,
-	"label_lowercase":    transformLabelLowercase,
-	"label_del":          transformLabelDel,
-	"label_keep":         transformLabelKeep,
-	"label_copy":         transformLabelCopy,
-	"label_move":         transformLabelMove,
-	"label_transform":    transformLabelTransform,
-	"label_value":        transformLabelValue,
-	"label_match":        transformLabelMatch,
-	"label_mismatch":     transformLabelMismatch,
-	"union":              transformUnion,
-	"":                   transformUnion, // empty func is a synonym to union
-	"keep_last_value":    transformKeepLastValue,
-	"keep_next_value":    transformKeepNextValue,
-	"interpolate":        transformInterpolate,
-	"start":              newTransformFuncZeroArgs(transformStart),
-	"end":                newTransformFuncZeroArgs(transformEnd),
-	"step":               newTransformFuncZeroArgs(transformStep),
-	"running_sum":        newTransformFuncRunning(runningSum),
-	"running_max":        newTransformFuncRunning(runningMax),
-	"running_min":        newTransformFuncRunning(runningMin),
-	"running_avg":        newTransformFuncRunning(runningAvg),
-	"range_sum":          newTransformFuncRange(runningSum),
-	"range_max":          newTransformFuncRange(runningMax),
-	"range_min":          newTransformFuncRange(runningMin),
-	"range_avg":          newTransformFuncRange(runningAvg),
-	"range_first":        transformRangeFirst,
-	"range_last":         transformRangeLast,
-	"range_quantile":     transformRangeQuantile,
-	"smooth_exponential": transformSmoothExponential,
-	"remove_resets":      transformRemoveResets,
-	"rand":               newTransformRand(newRandFloat64),
-	"rand_normal":        newTransformRand(newRandNormFloat64),
-	"rand_exponential":   newTransformRand(newRandExpFloat64),
-	"pi":                 transformPi,
-	"sin":                newTransformFuncOneArg(transformSin),
-	"cos":                newTransformFuncOneArg(transformCos),
-	"asin":               newTransformFuncOneArg(transformAsin),
-	"acos":               newTransformFuncOneArg(transformAcos),
-	"prometheus_buckets": transformPrometheusBuckets,
-	"buckets_limit":      transformBucketsLimit,
-	"histogram_share":    transformHistogramShare,
-	"histogram_avg":      transformHistogramAvg,
-	"histogram_stdvar":   transformHistogramStdvar,
-	"histogram_stddev":   transformHistogramStddev,
-	"sort_by_label":      newTransformFuncSortByLabel(false),
-	"sort_by_label_desc": newTransformFuncSortByLabel(true),
-	"timezone_offset":    transformTimezoneOffset,
+	"label_set":           transformLabelSet,
+	"label_map":           transformLabelMap,
+	"label_uppercase":     transformLabelUppercase,
+	"label_lowercase":     transformLabelLowercase,
+	"label_del":           transformLabelDel,
+	"label_keep":          transformLabelKeep,
+	"label_copy":          transformLabelCopy,
+	"label_move":          transformLabelMove,
+	"label_transform":     transformLabelTransform,
+	"label_value":         transformLabelValue,
+	"label_match":         transformLabelMatch,
+	"label_mismatch":      transformLabelMismatch,
+	"union":               transformUnion,
+	"":                    transformUnion, // empty func is a synonym to union
+	"keep_last_value":     transformKeepLastValue,
+	"keep_next_value":     transformKeepNextValue,
+	"interpolate":         transformInterpolate,
+	"start":               newTransformFuncZeroArgs(transformStart),
+	"end":                 newTransformFuncZeroArgs(transformEnd),
+	"step":                newTransformFuncZeroArgs(transformStep),
+	"running_sum":         newTransformFuncRunning(runningSum),
+	"running_max":         newTransformFuncRunning(runningMax),
+	"running_min":         newTransformFuncRunning(runningMin),
+	"running_avg":         newTransformFuncRunning(runningAvg),
+	"range_sum":           newTransformFuncRange(runningSum),
+	"range_max":           newTransformFuncRange(runningMax),
+	"range_min":           newTransformFuncRange(runningMin),
+	"range_avg":           newTransformFuncRange(runningAvg),
+	"range_first":         transformRangeFirst,
+	"range_last":          transformRangeLast,
+	"range_quantile":      transformRangeQuantile,
+	"smooth_exponential":  transformSmoothExponential,
+	"remove_resets":       transformRemoveResets,
+	"rand":                newTransformRand(newRandFloat64),
+	"rand_normal":         newTransformRand(newRandNormFloat64),
+	"rand_exponential":    newTransformRand(newRandExpFloat64),
+	"prometheus_buckets":  transformPrometheusBuckets,
+	"buckets_limit":       transformBucketsLimit,
+	"histogram_share":     transformHistogramShare,
+	"histogram_avg":       transformHistogramAvg,
+	"histogram_stdvar":    transformHistogramStdvar,
+	"histogram_stddev":    transformHistogramStddev,
+	"sort_by_label":       newTransformFuncSortByLabel(false),
+	"sort_by_label_desc":  newTransformFuncSortByLabel(true),
+	"timezone_offset":     transformTimezoneOffset,
+	"bitmap_and":          newTransformBitmap(bitmapAnd),
+	"bitmap_or":           newTransformBitmap(bitmapOr),
+	"bitmap_xor":          newTransformBitmap(bitmapXor),
+	"histogram_quantiles": transformHistogramQuantiles,
 }
 
 func getTransformFunc(s string) transformFunc {
@@ -786,6 +799,43 @@ func stdvarForLeTimeseries(i int, xss []leTimeseries) float64 {
 	return stdvar
 }
 
+func transformHistogramQuantiles(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if len(args) < 3 {
+		return nil, fmt.Errorf("unexpected number of args: %d; expecting at least 3 args", len(args))
+	}
+	dstLabel, err := getString(args[0], 0)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain dstLabel: %w", err)
+	}
+	phiArgs := args[1 : len(args)-1]
+	tssOrig := args[len(args)-1]
+	// Calculate quantile individually per each phi.
+	var rvs []*timeseries
+	for _, phiArg := range phiArgs {
+		phiStr := fmt.Sprintf("%g", phiArg[0].Values[0])
+		tss := copyTimeseries(tssOrig)
+		tfaTmp := &transformFuncArg{
+			ec: tfa.ec,
+			fe: tfa.fe,
+			args: [][]*timeseries{
+				phiArg,
+				tss,
+			},
+		}
+		tssTmp, err := transformHistogramQuantile(tfaTmp)
+		if err != nil {
+			return nil, fmt.Errorf("cannot calculate quantile %s: %w", phiStr, err)
+		}
+		for _, ts := range tssTmp {
+			ts.MetricName.RemoveTag(dstLabel)
+			ts.MetricName.AddTag(dstLabel, phiStr)
+		}
+		rvs = append(rvs, tssTmp...)
+	}
+	return rvs, nil
+}
+
 func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
 	if len(args) < 2 || len(args) > 3 {
@@ -1137,23 +1187,26 @@ func transformRangeQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	}
 	phi := phis[0]
 	rvs := args[1]
-	hf := histogram.GetFast()
+	a := getFloat64s()
+	values := a.A[:0]
 	for _, ts := range rvs {
-		hf.Reset()
 		lastIdx := -1
-		values := ts.Values
-		for i, v := range values {
+		originValues := ts.Values
+		values = values[:0]
+		for i, v := range originValues {
 			if math.IsNaN(v) {
 				continue
 			}
-			hf.Update(v)
+			values = append(values, v)
 			lastIdx = i
 		}
 		if lastIdx >= 0 {
-			values[lastIdx] = hf.Quantile(phi)
+			sort.Float64s(values)
+			originValues[lastIdx] = quantileSorted(phi, values)
 		}
 	}
-	histogram.PutFast(hf)
+	a.A = values
+	putFloat64s(a)
 	setLastValues(rvs)
 	return rvs, nil
 }
@@ -1734,7 +1787,7 @@ func transformRound(tfa *transformFuncArg) ([]*timeseries, error) {
 	return doTransformValues(args[0], tf, tfa.fe)
 }
 
-func transformSign(tfa *transformFuncArg) ([]*timeseries, error) {
+func transformSgn(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
 	if err := expectTransformArgsNum(args, 1); err != nil {
 		return nil, err
@@ -1823,8 +1876,15 @@ func newTransformFuncSort(isDesc bool) transformFunc {
 			b := rvs[j].Values
 			n := len(a) - 1
 			for n >= 0 {
-				if !math.IsNaN(a[n]) && !math.IsNaN(b[n]) && a[n] != b[n] {
-					break
+				if !math.IsNaN(a[n]) {
+					if math.IsNaN(b[n]) {
+						return false
+					}
+					if a[n] != b[n] {
+						break
+					}
+				} else if !math.IsNaN(b[n]) {
+					return true
 				}
 				n--
 			}
@@ -1848,18 +1908,58 @@ func transformSin(v float64) float64 {
 	return math.Sin(v)
 }
 
+func transformSinh(v float64) float64 {
+	return math.Sinh(v)
+}
+
 func transformCos(v float64) float64 {
 	return math.Cos(v)
 }
 
+func transformCosh(v float64) float64 {
+	return math.Cosh(v)
+}
+
+func transformTan(v float64) float64 {
+	return math.Tan(v)
+}
+
+func transformTanh(v float64) float64 {
+	return math.Tanh(v)
+}
+
 func transformAsin(v float64) float64 {
 	return math.Asin(v)
 }
 
+func transformAsinh(v float64) float64 {
+	return math.Asinh(v)
+}
+
+func transformAtan(v float64) float64 {
+	return math.Atan(v)
+}
+
+func transformAtanh(v float64) float64 {
+	return math.Atanh(v)
+}
+
 func transformAcos(v float64) float64 {
 	return math.Acos(v)
 }
 
+func transformAcosh(v float64) float64 {
+	return math.Acosh(v)
+}
+
+func transformDeg(v float64) float64 {
+	return v * 180 / math.Pi
+}
+
+func transformRad(v float64) float64 {
+	return v * math.Pi / 180
+}
+
 func newTransformRand(newRandFunc func(r *rand.Rand) func() float64) transformFunc {
 	return func(tfa *transformFuncArg) ([]*timeseries, error) {
 		args := tfa.args
@@ -1907,6 +2007,37 @@ func transformPi(tfa *transformFuncArg) ([]*timeseries, error) {
 	return evalNumber(tfa.ec, math.Pi), nil
 }
 
+func bitmapAnd(a, b uint64) uint64 {
+	return a & b
+}
+
+func bitmapOr(a, b uint64) uint64 {
+	return a | b
+}
+
+func bitmapXor(a, b uint64) uint64 {
+	return a ^ b
+}
+
+func newTransformBitmap(bitmapFunc func(a, b uint64) uint64) func(tfa *transformFuncArg) ([]*timeseries, error) {
+	return func(tfa *transformFuncArg) ([]*timeseries, error) {
+		args := tfa.args
+		if err := expectTransformArgsNum(args, 2); err != nil {
+			return nil, err
+		}
+		ns, err := getScalar(args[1], 1)
+		if err != nil {
+			return nil, err
+		}
+		tf := func(values []float64) {
+			for i, v := range values {
+				values[i] = float64(bitmapFunc(uint64(v), uint64(ns[i])))
+			}
+		}
+		return doTransformValues(args[0], tf, tfa.fe)
+	}
+}
+
 func transformTimezoneOffset(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
 	if err := expectTransformArgsNum(args, 1); err != nil {
@@ -1975,6 +2106,17 @@ func transformEnd(tfa *transformFuncArg) float64 {
 	return float64(tfa.ec.End) / 1e3
 }
 
+// copyTimeseries returns a copy of tss.
+func copyTimeseries(tss []*timeseries) []*timeseries {
+	rvs := make([]*timeseries, len(tss))
+	for i, src := range tss {
+		var dst timeseries
+		dst.CopyFromShallowTimestamps(src)
+		rvs[i] = &dst
+	}
+	return rvs
+}
+
 // copyTimeseriesMetricNames returns a copy of tss with real copy of MetricNames,
 // but with shallow copy of Timestamps and Values if makeCopy is set.
 //
@@ -1992,7 +2134,7 @@ func copyTimeseriesMetricNames(tss []*timeseries, makeCopy bool) []*timeseries {
 	return rvs
 }
 
-// copyShallow returns a copy of arg with shallow copies of MetricNames,
+// copyTimeseriesShallow returns a copy of arg with shallow copies of MetricNames,
 // Timestamps and Values.
 func copyTimeseriesShallow(arg []*timeseries) []*timeseries {
 	rvs := make([]*timeseries, len(arg))

app/vmselect/promql/tzdata.go

@@ -1,3 +1,4 @@
+//go:build go1.15
 // +build go1.15
 
 package promql

app/vmselect/querystats/querystats.go

@@ -14,8 +14,7 @@ import (
 var (
 	lastQueriesCount = flag.Int("search.queryStats.lastQueriesCount", 20000, "Query stats for /api/v1/status/top_queries is tracked on this number of last queries. "+
 		"Zero value disables query stats tracking")
-	minQueryDuration = flag.Duration("search.queryStats.minQueryDuration", 0, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. "+
-		"Queries with lower duration are ignored in query stats")
+	minQueryDuration = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
 )
 
 var (

app/vmselect/vmui/asset-manifest.json

@@ -1,17 +1,19 @@
 {
   "files": {
-    "main.css": "./static/css/main.0ba440d3.chunk.css",
-    "main.js": "./static/js/main.ffd27a2f.chunk.js",
-    "runtime-main.js": "./static/js/runtime-main.50ad8b45.js",
-    "static/js/2.3cdac8ea.chunk.js": "./static/js/2.3cdac8ea.chunk.js",
-    "static/js/3.d52da3ae.chunk.js": "./static/js/3.d52da3ae.chunk.js",
+    "main.css": "./static/css/main.acc63211.chunk.css",
+    "main.js": "./static/js/main.fe86f8ba.chunk.js",
+    "runtime-main.js": "./static/js/runtime-main.b765a534.js",
+    "static/css/2.a684aa27.chunk.css": "./static/css/2.a684aa27.chunk.css",
+    "static/js/2.632b68e4.chunk.js": "./static/js/2.632b68e4.chunk.js",
+    "static/js/3.daeccd9c.chunk.js": "./static/js/3.daeccd9c.chunk.js",
     "index.html": "./index.html",
-    "static/js/2.3cdac8ea.chunk.js.LICENSE.txt": "./static/js/2.3cdac8ea.chunk.js.LICENSE.txt"
+    "static/js/2.632b68e4.chunk.js.LICENSE.txt": "./static/js/2.632b68e4.chunk.js.LICENSE.txt"
   },
   "entrypoints": [
-    "static/js/runtime-main.50ad8b45.js",
-    "static/js/2.3cdac8ea.chunk.js",
-    "static/css/main.0ba440d3.chunk.css",
-    "static/js/main.ffd27a2f.chunk.js"
+    "static/js/runtime-main.b765a534.js",
+    "static/css/2.a684aa27.chunk.css",
+    "static/js/2.632b68e4.chunk.js",
+    "static/css/main.acc63211.chunk.css",
+    "static/js/main.fe86f8ba.chunk.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><link href="./static/css/main.0ba440d3.chunk.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div><script>!function(e){function r(r){for(var n,i,a=r[0],c=r[1],l=r[2],s=0,p=[];s<a.length;s++)i=a[s],Object.prototype.hasOwnProperty.call(o,i)&&o[i]&&p.push(o[i][0]),o[i]=0;for(n in c)Object.prototype.hasOwnProperty.call(c,n)&&(e[n]=c[n]);for(f&&f(r);p.length;)p.shift()();return u.push.apply(u,l||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++){var c=t[a];0!==o[c]&&(n=!1)}n&&(u.splice(r--,1),e=i(i.s=t[0]))}return e}var n={},o={1:0},u=[];function i(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,i),t.l=!0,t.exports}i.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var u,a=document.createElement("script");a.charset="utf-8",a.timeout=120,i.nc&&a.setAttribute("nonce",i.nc),a.src=function(e){return i.p+"static/js/"+({}[e]||e)+"."+{3:"d52da3ae"}[e]+".chunk.js"}(e);var c=new Error;u=function(r){a.onerror=a.onload=null,clearTimeout(l);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),u=r&&r.target&&r.target.src;c.message="Loading chunk "+e+" failed.\n("+n+": "+u+")",c.name="ChunkLoadError",c.type=n,c.request=u,t[1](c)}o[e]=void 0}};var l=setTimeout((function(){u({type:"timeout",target:a})}),12e4);a.onerror=a.onload=u,document.head.appendChild(a)}return Promise.all(r)},i.m=e,i.c=n,i.d=function(e,r,t){i.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,r){if(1&r&&(e=i(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(i.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)i.d(t,n,function(r){return e[r]}.bind(null,n));return t},i.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(r,"a",r),r},i.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},i.p="./",i.oe=function(e){throw console.error(e),e};var a=this.webpackJsonpvmui=this.webpackJsonpvmui||[],c=a.push.bind(a);a.push=r,a=a.slice();for(var l=0;l<a.length;l++)r(a[l]);var f=c;t()}([])</script><script src="./static/js/2.3cdac8ea.chunk.js"></script><script src="./static/js/main.ffd27a2f.chunk.js"></script></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><link href="./static/css/2.a684aa27.chunk.css" rel="stylesheet"><link href="./static/css/main.acc63211.chunk.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div><script>!function(e){function r(r){for(var n,i,a=r[0],c=r[1],l=r[2],s=0,p=[];s<a.length;s++)i=a[s],Object.prototype.hasOwnProperty.call(o,i)&&o[i]&&p.push(o[i][0]),o[i]=0;for(n in c)Object.prototype.hasOwnProperty.call(c,n)&&(e[n]=c[n]);for(f&&f(r);p.length;)p.shift()();return u.push.apply(u,l||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++){var c=t[a];0!==o[c]&&(n=!1)}n&&(u.splice(r--,1),e=i(i.s=t[0]))}return e}var n={},o={1:0},u=[];function i(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,i),t.l=!0,t.exports}i.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var u,a=document.createElement("script");a.charset="utf-8",a.timeout=120,i.nc&&a.setAttribute("nonce",i.nc),a.src=function(e){return i.p+"static/js/"+({}[e]||e)+"."+{3:"daeccd9c"}[e]+".chunk.js"}(e);var c=new Error;u=function(r){a.onerror=a.onload=null,clearTimeout(l);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),u=r&&r.target&&r.target.src;c.message="Loading chunk "+e+" failed.\n("+n+": "+u+")",c.name="ChunkLoadError",c.type=n,c.request=u,t[1](c)}o[e]=void 0}};var l=setTimeout((function(){u({type:"timeout",target:a})}),12e4);a.onerror=a.onload=u,document.head.appendChild(a)}return Promise.all(r)},i.m=e,i.c=n,i.d=function(e,r,t){i.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,r){if(1&r&&(e=i(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(i.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)i.d(t,n,function(r){return e[r]}.bind(null,n));return t},i.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(r,"a",r),r},i.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},i.p="./",i.oe=function(e){throw console.error(e),e};var a=this.webpackJsonpvmui=this.webpackJsonpvmui||[],c=a.push.bind(a);a.push=r,a=a.slice();for(var l=0;l<a.length;l++)r(a[l]);var f=c;t()}([])</script><script src="./static/js/2.632b68e4.chunk.js"></script><script src="./static/js/main.fe86f8ba.chunk.js"></script></body></html>

app/vmselect/vmui/static/css/2.a684aa27.chunk.css

@@ -0,0 +1 @@
+.uplot,.uplot *,.uplot :after,.uplot :before{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";line-height:1.5;width:-webkit-min-content;width:-moz-min-content;width:min-content}.u-title{text-align:center;font-size:18px;font-weight:700}.u-wrap{position:relative;-webkit-user-select:none;-ms-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;position:relative;width:100%;height:100%}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{vertical-align:middle;display:inline-block}.u-legend .u-marker{width:1em;height:1em;margin-right:4px;background-clip:padding-box!important}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:rgba(0,0,0,.07)}.u-cursor-x,.u-cursor-y,.u-select{position:absolute;pointer-events:none}.u-cursor-x,.u-cursor-y{left:0;top:0;will-change:transform;z-index:100}.u-hz .u-cursor-x,.u-vt .u-cursor-y{height:100%;border-right:1px dashed #607d8b}.u-hz .u-cursor-y,.u-vt .u-cursor-x{width:100%;border-bottom:1px dashed #607d8b}.u-cursor-pt{position:absolute;top:0;left:0;border-radius:50%;border:0 solid;pointer-events:none;will-change:transform;z-index:100;background-clip:padding-box!important}.u-cursor-pt.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-select.u-off{display:none}