docs/CHANGELOG.md: cut v1.48.0 release

Users can specify -promscrape.config.strictParse if -promscrape.config shouldn't contain unknown config entries

.github/workflows/main.yml

@@ -14,9 +14,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Go
-        uses: actions/setup-go@master
+        uses: actions/setup-go@main
         with:
-          go-version: 1.14
+          go-version: 1.15
         id: go
       - name: Dependencies
         env:
@@ -43,7 +43,20 @@ jobs:
           make victoria-metrics-arm64
           make vmutils
           GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=freebsd go build -mod=vendor ./app/vmagent
+          GOOS=freebsd go build -mod=vendor ./app/vmalert
+          GOOS=freebsd go build -mod=vendor ./app/vmbackup
+          GOOS=freebsd go build -mod=vendor ./app/vmrestore
+          GOOS=openbsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=openbsd go build -mod=vendor ./app/vmagent
+          GOOS=openbsd go build -mod=vendor ./app/vmalert
+          GOOS=openbsd go build -mod=vendor ./app/vmbackup
+          GOOS=openbsd go build -mod=vendor ./app/vmrestore
           GOOS=darwin go build -mod=vendor ./app/victoria-metrics
+          GOOS=darwin go build -mod=vendor ./app/vmagent
+          GOOS=darwin go build -mod=vendor ./app/vmalert
+          GOOS=darwin go build -mod=vendor ./app/vmbackup
+          GOOS=darwin go build -mod=vendor ./app/vmrestore
       - name: Publish coverage
         uses: codecov/codecov-action@v1.0.6
         with:

Makefile

@@ -122,8 +122,8 @@ benchmark-pure:
 	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
 
 vendor-update:
-	GO111MODULE=on go get -u ./lib/...
-	GO111MODULE=on go get -u ./app/...
+	GO111MODULE=on go get -u -d ./lib/...
+	GO111MODULE=on go get -u -d ./app/...
 	GO111MODULE=on go mod tidy
 	GO111MODULE=on go mod vendor
 

app/victoria-metrics/Makefile

@@ -108,3 +108,10 @@ victoria-metrics-package-deb-rpm-all: \
         victoria-metrics-package-deb-arm64 \
         victoria-metrics-package-rpm \
         victoria-metrics-package-rpm-arm64
+
+### Packaging as snap
+victoria-metrics-package-snap:
+	which snapcraft || snap install snapcraft
+	which multipass || snap install multipass
+	snapcraft
+

app/victoria-metrics/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"os"
 	"time"
@@ -16,6 +17,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
@@ -24,6 +26,8 @@ var (
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
 		"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
 		"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
+	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
+		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
 )
 
 func main() {
@@ -33,6 +37,18 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 	cgroup.UpdateGOMAXPROCSToCPUQuota()
+
+	if promscrape.IsDryRun() {
+		*dryRun = true
+	}
+	if *dryRun {
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		}
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		return
+	}
+
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
@@ -66,6 +82,10 @@ func main() {
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "Single-node VictoriaMetrics. See docs at https://victoriametrics.github.io/")
+		return true
+	}
 	if vminsert.RequestHandler(w, r) {
 		return true
 	}

app/vmagent/README.md

@@ -25,7 +25,8 @@ to `vmagent` (like the ability to push metrics instead of pulling them). We did
   * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
   * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
-  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-json-line-format).
+  * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-native-format).
   * Data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
   * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
@@ -41,7 +42,7 @@ Just download `vmutils-*` archive from [releases page](https://github.com/Victor
 and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
 
 * `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary amount of remote storage systems.
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
 
@@ -62,6 +63,22 @@ Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://gi
 Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
 
 
+### Configuration update
+
+`vmagent` should be restarted in order to update config options set via command-line args.
+
+`vmagent` supports multiple approaches for reloading configs from updated config files such as `-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
+
+* Sending `SUGHUP` signal to `vmagent` process:
+  ```bash
+  kill -SIGHUP `pidof vmagent`
+  ```
+
+* Sending HTTP request to `http://vmagent:8429/-/reload` endpoint.
+
+There is also `-promscrape.configCheckInterval` command-line option, which can be used for automatic reloading configs from updated `-promscrape.config` file.
+
+
 ### Use cases
 
 
@@ -135,7 +152,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `ec2_sd_configs` - for scraping targets in Amazon EC2.
   See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
-  `vmagent` doesn't support `role_arn` config param yet.
+  `vmagent` doesn't support `profile` config param and aws credentials file yet.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
   `vmagent` provides the following additional functionality for `gce_sd_config`:
@@ -147,6 +164,13 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
 * `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
   See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
+* `openstack_sd_configs` - for scraping OpenStack targets.
+  See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
+  [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
+* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
+  See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
+* `eureka_sd_configs` - for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka).
+  See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
 
 File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
@@ -191,6 +215,7 @@ The relabeling can be defined in the following places:
 
 Read more about relabeling in the following articles:
 
+* [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
 * [Life of a label](https://www.robustperception.io/life-of-a-label)
 * [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
 * [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
@@ -205,7 +230,16 @@ either via `vmagent` itself or via Prometheus, so the exported metrics could be
 Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
 If you have suggestions, improvements or found a bug - feel free to open an issue on github or add review to the dashboard.
 
-`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+`vmagent` also exports target statuses at the following handlers:
+
+* `http://vmagent-host:8429/targets`. This handler returns human-readable plaintext status for every active target.
+This page is convenient to query from command line with `wget`, `curl` or similar tools.
+It accepts optional `show_original_labels=1` query arg, which shows the original labels per each target before applying relabeling.
+This information may be useful for debugging target relabeling.
+* `http://vmagent-host:8429/api/v1/targets`. This handler returns data compatible with [the corresponding page from Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/#targets).
+
+* `http://vmagent-host:8429/ready`. This handler returns http 200 status code when `vmagent` finishes initialization for all service_discovery configs.
+It may be useful for performing `vmagent` rolling update without scrape loss.
 
 
 ### Troubleshooting
@@ -216,7 +250,32 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
   since `vmagent` establishes at least a single TCP connection per each target.
 
 * When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
-  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
+  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`
+  and `http://vmagent-host:8429/api/v1/targets`.
+
+* The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
+  This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default up to `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value in order to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM, so big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if big number of scrape targets are dropped during relabeling.
+
+* If `vmagent` scrapes big number of targets, then `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
+  This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+
+* If `vmagent` scrapes targets with millions of metrics per each target (for instance, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
+  then it is recommended enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all the scrape targets
+  by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
+
+  ```yml
+  scrape_configs:
+  - job_name: 'big-federate'
+    stream_parse: true
+    static_configs:
+    - targets:
+      - big-prometeus1
+      - big-prometeus2
+    honor_labels: true
+    metrics_path: /federate
+    params:
+      'match[]': ['{__name__!=""}']
+  ```
 
 * It is recommended to increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
 
@@ -228,8 +287,13 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
   If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
 
+* By default `vmagent` masks `-remoteWrite.url` with `secret-url` values in logs and at `/metrics` page because
+  the url may contain sensitive information such as auth tokens or passwords.
+  Pass `-remoteWrite.showURL` command-line flag when starting `vmagent` in order to see all the valid urls.
+
 * If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports
-  or they use init container.
+  or they use init container. These errors can be either fixed or suppressed with `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag.
+  See available options below if you prefer fixing the root cause of the error:
 
   The following `relabel_configs` section may help determining `__meta_*` labels resulting in duplicate targets:
   ```yml
@@ -274,7 +338,7 @@ Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

app/vmagent/common/push_ctx.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // PushCtx is a context used for populating WriteRequest.
@@ -28,12 +29,7 @@ func (ctx *PushCtx) Reset() {
 	}
 	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
 
-	labels := ctx.Labels
-	for i := range labels {
-		label := &labels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(ctx.Labels)
 	ctx.Labels = ctx.Labels[:0]
 
 	ctx.Samples = ctx.Samples[:0]

app/vmagent/influx/request_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -62,6 +63,7 @@ func insertRows(db string, rows []parser.Row) error {
 	buf := ctx.buf[:0]
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Fields)
 		commonLabels = commonLabels[:0]
 		hasDBKey := false
 		for j := range r.Tags {
@@ -111,7 +113,6 @@ func insertRows(db string, rows []parser.Row) error {
 				Samples: samples[len(samples)-1:],
 			})
 		}
-		rowsTotal += len(r.Fields)
 	}
 	ctx.buf = buf
 	ctx.ctx.WriteRequest.Timeseries = tssDst
@@ -135,12 +136,8 @@ type pushCtx struct {
 func (ctx *pushCtx) reset() {
 	ctx.ctx.Reset()
 
-	commonLabels := ctx.commonLabels
-	for i := range commonLabels {
-		label := &commonLabels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(ctx.commonLabels)
+	ctx.commonLabels = ctx.commonLabels[:0]
 
 	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
 	ctx.buf = ctx.buf[:0]

app/vmagent/main.go

@@ -5,12 +5,15 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
@@ -28,6 +31,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -44,7 +48,8 @@ var (
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
 	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
-		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . See also -promscrape.config.dryRun")
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
+		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
 )
 
 var (
@@ -59,19 +64,24 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 	cgroup.UpdateGOMAXPROCSToCPUQuota()
 
-	if *dryRun {
-		if err := flag.Set("promscrape.config.strictParse", "true"); err != nil {
-			logger.Panicf("BUG: cannot set promscrape.config.strictParse=true: %s", err)
+	if promscrape.IsDryRun() {
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
+		return
+	}
+	if *dryRun {
 		if err := remotewrite.CheckRelabelConfigs(); err != nil {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := promscrape.CheckConfig(); err != nil {
-			logger.Fatalf("error when checking Prometheus config: %s", err)
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exitting with 0 status code")
 		return
@@ -80,6 +90,7 @@ func main() {
 	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
 	remotewrite.Init()
+	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
@@ -127,12 +138,17 @@ func main() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
+	common.StopUnmarshalWorkers()
 	remotewrite.Stop()
 
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.RequestURI == "/" {
+		fmt.Fprintf(w, "vmagent - see docs at https://victoriametrics.github.io/vmagent.html")
+		return true
+	}
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
 	case "/api/v1/write":
@@ -171,6 +187,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
@@ -188,14 +213,31 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain")
-		promscrape.WriteHumanReadableTargetsStatus(w)
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+		return true
+	case "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/ready":
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
+			http.Error(w, errMsg, http.StatusTooEarly)
+		} else {
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("OK"))
+		}
+		return true
 	}
 	return false
 }
@@ -213,12 +255,16 @@ var (
 	prometheusimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 	prometheusimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 
+	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
 
 	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
 
-	promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeTargetsRequests      = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )

app/vmagent/native/request_handler.go

@@ -0,0 +1,85 @@
+package native
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(block *parser.Block) error {
+			return insertRows(block, extraLabels)
+		})
+	})
+}
+
+func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	// Update rowsInserted and rowsPerInsert before actual inserting,
+	// since relabeling can prevent from inserting the rows.
+	rowsLen := len(block.Values)
+	rowsInserted.Add(rowsLen)
+	rowsPerInsert.Update(float64(rowsLen))
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	mn := &block.MetricName
+	labelsLen := len(labels)
+	labels = append(labels, prompbmarshal.Label{
+		Name:  "__name__",
+		Value: bytesutil.ToUnsafeString(mn.MetricGroup),
+	})
+	for j := range mn.Tags {
+		tag := &mn.Tags[j]
+		labels = append(labels, prompbmarshal.Label{
+			Name:  bytesutil.ToUnsafeString(tag.Key),
+			Value: bytesutil.ToUnsafeString(tag.Value),
+		})
+	}
+	labels = append(labels, extraLabels...)
+	values := block.Values
+	timestamps := block.Timestamps
+	if len(timestamps) != len(values) {
+		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+	}
+	samplesLen := len(samples)
+	for j, value := range values {
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     value,
+			Timestamp: timestamps[j],
+		})
+	}
+	tssDst = append(tssDst, prompbmarshal.TimeSeries{
+		Labels:  labels[labelsLen:],
+		Samples: samples[samplesLen:],
+	})
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	return nil
+}

app/vmagent/promremotewrite/request_handler.go

@@ -35,6 +35,7 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 	samples := ctx.Samples[:0]
 	for i := range timeseries {
 		ts := &timeseries[i]
+		rowsTotal += len(ts.Samples)
 		labelsLen := len(labels)
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
@@ -55,7 +56,6 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
-		rowsTotal += len(ts.Samples)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels

app/vmagent/remotewrite/client.go

@@ -44,7 +44,7 @@ var (
 )
 
 type client struct {
-	urlLabelValue  string
+	sanitizedURL   string
 	remoteWriteURL string
 	authHeader     string
 	fq             *persistentqueue.FastQueue
@@ -53,13 +53,14 @@ type client struct {
 	requestDuration *metrics.Histogram
 	requestsOKCount *metrics.Counter
 	errorsCount     *metrics.Counter
+	packetsDropped  *metrics.Counter
 	retriesCount    *metrics.Counter
 
 	wg     sync.WaitGroup
 	stopCh chan struct{}
 }
 
-func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue, concurrency int) *client {
+func newClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	tlsCfg, err := getTLSConfig(argIdx)
 	if err != nil {
 		logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
@@ -101,7 +102,7 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 		authHeader = "Bearer " + token
 	}
 	c := &client{
-		urlLabelValue:  urlLabelValue,
+		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authHeader:     authHeader,
 		fq:             fq,
@@ -111,10 +112,11 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 		},
 		stopCh: make(chan struct{}),
 	}
-	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
-	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
-	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
-	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
+	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -122,14 +124,14 @@ func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentq
 			c.runWorker()
 		}()
 	}
-	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 	return c
 }
 
 func (c *client) MustStop() {
 	close(c.stopCh)
 	c.wg.Wait()
-	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
 }
 
 func getTLSConfig(argIdx int) (*tls.Config, error) {
@@ -176,7 +178,7 @@ func (c *client) runWorker() {
 				// The block has been sent successfully.
 			case <-time.After(graceDuration):
 				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
-					len(block), c.remoteWriteURL, graceDuration.Seconds())
+					len(block), c.sanitizedURL, graceDuration.Seconds())
 			}
 			return
 		}
@@ -190,7 +192,7 @@ func (c *client) sendBlock(block []byte) {
 again:
 	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.remoteWriteURL, err)
+		logger.Panicf("BUG: unexected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
@@ -211,7 +213,7 @@ again:
 			retryDuration = time.Minute
 		}
 		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
-			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
+			len(block), c.sanitizedURL, err, retryDuration.Seconds())
 		t := time.NewTimer(retryDuration)
 		select {
 		case <-c.stopCh:
@@ -228,10 +230,20 @@ again:
 		c.requestsOKCount.Inc()
 		return
 	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
+	if statusCode == 409 {
+		// Just drop block on 409 status code like Prometheus does.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
+		body, _ := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		logger.Errorf("unexpected status code received when sending a block with size %d bytes to %q: #%d; dropping the block like Prometheus does; "+
+			"response body=%q", len(block), c.sanitizedURL, statusCode, body)
+		c.packetsDropped.Inc()
+		return
+	}
 
 	// Unexpected status code returned
 	retriesCount++
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
 	retryDuration *= 2
 	if retryDuration > time.Minute {
 		retryDuration = time.Minute
@@ -239,10 +251,10 @@ again:
 	body, err := ioutil.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
-		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.remoteWriteURL, retriesCount, err)
+		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
 	} else {
 		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
-			"re-sending the block in %.3f seconds", len(block), c.remoteWriteURL, retriesCount, statusCode, body, retryDuration.Seconds())
+			"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
 	}
 	t := time.NewTimer(retryDuration)
 	select {

app/vmagent/remotewrite/pendingseries.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -19,7 +20,7 @@ var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
 		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage. "+
 		"Minimum supported interval is 1 second")
-	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 32*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
 		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
 )
 
@@ -104,11 +105,7 @@ func (wr *writeRequest) reset() {
 	}
 	wr.tss = wr.tss[:0]
 
-	for i := range wr.labels {
-		label := &wr.labels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(wr.labels)
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
@@ -127,7 +124,7 @@ func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	for i := range src {
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
 		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
-		if len(tssDst) >= maxRowsPerBlock {
+		if len(wr.samples) >= maxRowsPerBlock {
 			wr.tss = tssDst
 			wr.flush()
 			tssDst = wr.tss

app/vmagent/remotewrite/relabel.go

@@ -117,12 +117,7 @@ type relabelCtx struct {
 }
 
 func (rctx *relabelCtx) reset() {
-	labels := rctx.labels
-	for i := range labels {
-		label := &labels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(rctx.labels)
 	rctx.labels = rctx.labels[:0]
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -24,7 +23,7 @@ var (
 		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
-	queues      = flag.Int("remoteWrite.queues", 1, "The number of concurrent queues to each -remoteWrite.url. Set more queues if a single queue "+
+	queues      = flag.Int("remoteWrite.queues", 4, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
@@ -46,6 +45,14 @@ var allRelabelConfigs atomic.Value
 // since it may lead to high memory usage due to big number of buffers.
 var maxQueues = runtime.GOMAXPROCS(-1) * 4
 
+// InitSecretFlags must be called after flag.Parse and before any logging.
+func InitSecretFlags() {
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		flagutil.RegisterSecretFlag("remoteWrite.url")
+	}
+}
+
 // Init initializes remotewrite.
 //
 // It must be called after flag.Parse().
@@ -61,10 +68,6 @@ func Init() {
 	if *queues <= 0 {
 		*queues = 1
 	}
-	if !*showRemoteWriteURL {
-		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
-		httpserver.RegisterSecretFlag("remoteWrite.url")
-	}
 	initLabelsGlobal()
 	rcs, err := loadRelabelConfigs()
 	if err != nil {
@@ -83,11 +86,11 @@ func Init() {
 		maxInmemoryBlocks = 2
 	}
 	for i, remoteWriteURL := range *remoteWriteURLs {
-		urlLabelValue := fmt.Sprintf("secret-url-%d", i+1)
+		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
 		if *showRemoteWriteURL {
-			urlLabelValue = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
+			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, urlLabelValue)
+		rwctx := newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 		rwctxs = append(rwctxs, rwctx)
 	}
 
@@ -154,10 +157,19 @@ func Push(wr *prompbmarshal.WriteRequest) {
 	tss := wr.Timeseries
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
+		samplesCount := 0
+		i := 0
+		for i < len(tss) {
+			samplesCount += len(tss[i].Samples)
+			i++
+			if samplesCount > maxRowsPerBlock {
+				break
+			}
+		}
 		tssBlock := tss
-		if len(tssBlock) > maxRowsPerBlock {
-			tssBlock = tss[:maxRowsPerBlock]
-			tss = tss[maxRowsPerBlock:]
+		if i < len(tss) {
+			tssBlock = tss[:i]
+			tss = tss[i:]
 		} else {
 			tss = nil
 		}
@@ -190,17 +202,17 @@ type remoteWriteCtx struct {
 	relabelMetricsDropped *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, urlLabelValue string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
 	h := xxhash.Sum64([]byte(remoteWriteURL))
 	path := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
-	fq := persistentqueue.MustOpenFastQueue(path, remoteWriteURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+	fq := persistentqueue.MustOpenFastQueue(path, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	c := newClient(argIdx, remoteWriteURL, urlLabelValue, fq, *queues)
+	c := newClient(argIdx, remoteWriteURL, sanitizedURL, fq, *queues)
 	pss := make([]*pendingSeries, *queues)
 	for i := range pss {
 		pss[i] = newPendingSeries(fq.MustWriteBlock)
@@ -211,7 +223,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL string, maxInmemoryBlocks int,
 		c:   c,
 		pss: pss,
 
-		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, urlLabelValue)),
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, sanitizedURL)),
 	}
 }
 

app/vmagent/vmimport/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
@@ -43,6 +44,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	samples := ctx.Samples[:0]
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Values)
 		labelsLen := len(labels)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
@@ -54,7 +56,9 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 		labels = append(labels, extraLabels...)
 		values := r.Values
 		timestamps := r.Timestamps
-		_ = timestamps[len(values)-1]
+		if len(timestamps) != len(values) {
+			logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+		}
 		samplesLen := len(samples)
 		for j, value := range values {
 			samples = append(samples, prompbmarshal.Sample{
@@ -66,7 +70,6 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
-		rowsTotal += len(values)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels

app/vmalert/README.md

@@ -11,6 +11,7 @@ rules against configured address.
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
  support;
 * Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Keeps the alerts [state on restarts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmalert#alerts-state-on-restarts);
 * Lightweight without extra dependencies.
 
 ### Limitations:
@@ -121,14 +122,6 @@ annotations:
   [ <labelname>: <tmpl_string> ]
 ``` 
 
-`vmalert` has no local storage and alerts state is stored in process memory. Hence, after reloading of `vmalert` process
-alerts state will be lost. To avoid this situation, `vmalert` may be configured via following flags:
-* `-remoteWrite.url` - URL to Victoria Metrics or VMInsert. `vmalert` will persist alerts state into the configured
-address in form of timeseries with name `ALERTS` via remote-write protocol.
-* `-remoteRead.url` - URL to Victoria Metrics or VMSelect. `vmalert` will try to restore alerts state from configured
-address by querying `ALERTS` timeseries.
-
-
 ##### Recording rules
 
 The syntax for recording rules is following:
@@ -147,6 +140,22 @@ labels:
 For recording rules to work `-remoteWrite.url` must specified.
 
 
+#### Alerts state on restarts
+
+`vmalert` has no local storage, so alerts state is stored in the process memory. Hence, after reloading of `vmalert` 
+the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
+* `-remoteWrite.url` - URL to VictoriaMetrics (Single) or VMInsert (Cluster). `vmalert` will persist alerts state 
+into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol. 
+These are regular time series and may be queried from VM just as any other time series. 
+The state stored to the configured address on every rule evaluation.
+* `-remoteRead.url` - URL to VictoriaMetrics (Single) or VMSelect (Cluster). `vmalert` will try to restore alerts state 
+from configured address by querying time series with name `ALERTS_FOR_STATE`.
+
+Both flags are required for the proper state restoring. Restore process may fail if time series are missing
+in configured `-remoteRead.url`, weren't updated in the last `1h` or received state doesn't match current `vmalert` 
+rules configuration.
+
+
 #### WEB
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
@@ -166,6 +175,10 @@ The shortlist of configuration flags is the following:
     	Optional basic auth password for -datasource.url
   -datasource.basicAuth.username string
     	Optional basic auth username for -datasource.url
+  -datasource.lookback duration
+        Lookback defines how far to look into past when evaluating queries. For example, if datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+  -datasource.maxIdleConnections int
+        Defines the number of idle (keep-alive connections) to configured datasource.Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state. (default 100)
   -datasource.tlsCAFile string
     	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default system CA is used
   -datasource.tlsCertFile string

app/vmalert/alerting.go

@@ -53,7 +53,7 @@ func newAlertingRule(group *Group, cfg config.Rule) *AlertingRule {
 		RuleID:      cfg.ID,
 		Name:        cfg.Alert,
 		Expr:        cfg.Expr,
-		For:         cfg.For,
+		For:         cfg.For.Duration(),
 		Labels:      cfg.Labels,
 		Annotations: cfg.Annotations,
 		GroupID:     group.ID(),
@@ -140,7 +140,17 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b
 	updated := make(map[uint64]struct{})
 	// update list of active alerts
 	for _, m := range qMetrics {
+		for k, v := range ar.Labels {
+			// apply extra labels
+			m.SetLabel(k, v)
+		}
+
 		h := hash(m)
+		if _, ok := updated[h]; ok {
+			// duplicate may be caused by extra labels
+			// conflicting with the metric labels
+			return nil, fmt.Errorf("labels %v: %w", m.Labels, errDuplicate)
+		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
 			if a.Value != m.Value {
@@ -258,25 +268,11 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifie
 }
 
 func (ar *AlertingRule) template(a *notifier.Alert) error {
-	// 1. template rule labels with data labels
-	rLabels, err := a.ExecTemplate(ar.Labels)
-	if err != nil {
-		return err
-	}
-
-	// 2. merge data labels and rule labels
-	// metric labels may be overridden by
-	// rule labels
-	for k, v := range rLabels {
-		a.Labels[k] = v
-	}
-
-	// 3. template merged labels
+	var err error
 	a.Labels, err = a.ExecTemplate(a.Labels)
 	if err != nil {
 		return err
 	}
-
 	a.Annotations, err = a.ExecTemplate(ar.Annotations)
 	return err
 }
@@ -403,7 +399,7 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
 	}
 
-	// Get the last datapoint in range via MetricsQL `last_over_time`.
+	// Get the last data point in range via MetricsQL `last_over_time`.
 	// We don't use plain PromQL since Prometheus doesn't support
 	// remote write protocol which is used for state persistence in vmalert.
 	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
@@ -417,13 +413,9 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		labels := m.Labels
 		m.Labels = make([]datasource.Label, 0)
 		// drop all extra labels, so hash key will
-		// be identical to timeseries received in Exec
+		// be identical to time series received in Exec
 		for _, l := range labels {
-			if l.Name == alertNameLabel {
-				continue
-			}
-			// drop all overridden labels
-			if _, ok := ar.Labels[l.Name]; ok {
+			if l.Name == alertNameLabel || l.Name == alertGroupNameLabel {
 				continue
 			}
 			m.Labels = append(m.Labels, l)
@@ -436,7 +428,7 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb
 		a.ID = hash(m)
 		a.State = notifier.StatePending
 		ar.alerts[a.ID] = a
-		logger.Infof("alert %q(%d) restored to state at %v", a.Name, a.ID, a.Start)
+		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.Start)
 	}
 	return nil
 }

app/vmalert/alerting_test.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"errors"
+	"strings"
 	"testing"
 	"time"
 
@@ -218,19 +220,6 @@ func TestAlertingRule_Exec(t *testing.T) {
 				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
 			},
 		},
-		{
-			newTestAlertingRule("duplicate", 0),
-			[][]datasource.Metric{
-				{
-					// metrics with the same labelset should result in one alert
-					metricWithLabels(t, "name", "foo", "type", "bar"),
-					metricWithLabels(t, "type", "bar", "name", "foo"),
-				},
-			},
-			map[uint64]*notifier.Alert{
-				hash(metricWithLabels(t, "name", "foo", "type", "bar")): {State: notifier.StateFiring},
-			},
-		},
 		{
 			newTestAlertingRule("for-pending", time.Minute),
 			[][]datasource.Metric{
@@ -355,6 +344,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
 					"__name__", alertForStateMetricName,
 					alertNameLabel, "",
+					alertGroupNameLabel, "groupID",
 					"foo", "bar",
 					"namespace", "baz",
 				),
@@ -375,7 +365,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 					alertNameLabel, "",
 					"foo", "bar",
 					"namespace", "baz",
-					// following pair supposed to be dropped
+					// extra labels set by rule
 					"source", "vm",
 				),
 			},
@@ -383,6 +373,7 @@ func TestAlertingRule_Restore(t *testing.T) {
 				hash(metricWithLabels(t,
 					"foo", "bar",
 					"namespace", "baz",
+					"source", "vm",
 				)): {State: notifier.StatePending,
 					Start: time.Now().Truncate(time.Hour)},
 			},
@@ -441,6 +432,38 @@ func TestAlertingRule_Restore(t *testing.T) {
 	}
 }
 
+func TestAlertingRule_Exec_Negative(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+
+	// successful attempt
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	_, err := ar.Exec(context.TODO(), fq, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// label `job` will collide with rule extra label and will make both time series equal
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
+	_, err = ar.Exec(context.TODO(), fq, false)
+	if !errors.Is(err, errDuplicate) {
+		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
+	}
+
+	fq.reset()
+
+	expErr := "connection reset by peer"
+	fq.setErr(errors.New(expErr))
+	_, err = ar.Exec(context.TODO(), fq, false)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), expErr) {
+		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+	}
+}
+
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 	r := newTestAlertingRule(name, 0)
 	r.Labels = make(map[string]string)

app/vmalert/config/config.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/metricsql"
@@ -41,7 +42,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 	b, err := yaml.Marshal(g)
 	if err != nil {
-		return fmt.Errorf("failed to marshal group configuration for checksum: %s", err)
+		return fmt.Errorf("failed to marshal group configuration for checksum: %w", err)
 	}
 	h := md5.New()
 	h.Write(b)
@@ -94,7 +95,7 @@ type Rule struct {
 	Record      string            `yaml:"record,omitempty"`
 	Alert       string            `yaml:"alert,omitempty"`
 	Expr        string            `yaml:"expr"`
-	For         time.Duration     `yaml:"for,omitempty"`
+	For         PromDuration      `yaml:"for,omitempty"`
 	Labels      map[string]string `yaml:"labels,omitempty"`
 	Annotations map[string]string `yaml:"annotations,omitempty"`
 
@@ -102,6 +103,37 @@ type Rule struct {
 	XXX map[string]interface{} `yaml:",inline"`
 }
 
+// PromDuration is Prometheus duration.
+type PromDuration struct {
+	milliseconds int64
+}
+
+// NewPromDuration returns PromDuration for given d.
+func NewPromDuration(d time.Duration) PromDuration {
+	return PromDuration{
+		milliseconds: d.Milliseconds(),
+	}
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler interface.
+func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	ms, err := metricsql.DurationValue(s, 0)
+	if err != nil {
+		return err
+	}
+	pd.milliseconds = ms
+	return nil
+}
+
+// Duration returns duration for pd.
+func (pd *PromDuration) Duration() time.Duration {
+	return time.Duration(pd.milliseconds) * time.Millisecond
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rule Rule
@@ -162,25 +194,32 @@ func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool)
 		}
 		fp = append(fp, matches...)
 	}
+	errGroup := new(utils.ErrGroup)
 	var groups []Group
 	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
 		gr, err := parseFile(file)
 		if err != nil {
-			return nil, fmt.Errorf("failed to parse file %q: %w", file, err)
+			errGroup.Add(fmt.Errorf("failed to parse file %q: %w", file, err))
+			continue
 		}
 		for _, g := range gr {
 			if err := g.Validate(validateAnnotations, validateExpressions); err != nil {
-				return nil, fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err)
+				errGroup.Add(fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err))
+				continue
 			}
 			if _, ok := uniqueGroups[g.Name]; ok {
-				return nil, fmt.Errorf("group name %q duplicate in file %q", g.Name, file)
+				errGroup.Add(fmt.Errorf("group name %q duplicate in file %q", g.Name, file))
+				continue
 			}
 			uniqueGroups[g.Name] = struct{}{}
 			g.File = file
 			groups = append(groups, g)
 		}
 	}
+	if err := errGroup.Err(); err != nil {
+		return nil, err
+	}
 	if len(groups) < 1 {
 		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
 	}

app/vmalert/config/config_test.go

@@ -270,7 +270,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: time.Minute},
+			Rule{Alert: "alert", Expr: "up == 1", For: NewPromDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},

app/vmalert/config/testdata/kube-good.rules

@@ -665,7 +665,7 @@
           /
         sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
         > 0.01
-      for: 15m
+      for: 1d
       labels:
         severity: critical
     - alert: KubeStateMetricsWatchErrors
@@ -1724,4 +1724,4 @@
         rate(prometheus_operator_node_address_lookup_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1
       for: 10m
       labels:
-        severity: warning
+        severity: warning

app/vmalert/datasource/datasource.go

@@ -17,6 +17,23 @@ type Metric struct {
 	Value     float64
 }
 
+// SetLabel adds or updates existing one label
+// by the given key and label
+func (m *Metric) SetLabel(key, value string) {
+	for i, l := range m.Labels {
+		if l.Name == key {
+			m.Labels[i].Value = value
+			return
+		}
+	}
+	m.AddLabel(key, value)
+}
+
+// AddLabel appends the given label to the label set
+func (m *Metric) AddLabel(key, value string) {
+	m.Labels = append(m.Labels, Label{Name: key, Value: value})
+}
+
 // Label represents metric's label
 type Label struct {
 	Name  string

app/vmalert/datasource/init.go

@@ -21,18 +21,23 @@ var (
 		"By default system CA is used")
 	tlsServerName = flag.String("datasource.tlsServerName", "", "Optional TLS server name to use for connections to -datasource.url. "+
 		"By default the server name from -datasource.url is used")
+
+	lookBack = flag.Duration("datasource.lookback", 0, "Lookback defines how far to look into past when evaluating queries. "+
+		"For example, if datasource.lookback=5m then param \"time\" with value now()-5m will be added to every query.")
+	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, "Defines the number of idle (keep-alive connections) to configured datasource."+
+		"Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state.")
 )
 
 // Init creates a Querier from provided flag values.
 func Init() (Querier, error) {
 	if *addr == "" {
-		flag.PrintDefaults()
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
 	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	tr.MaxIdleConns = *maxIdleConnections
 	c := &http.Client{Transport: tr}
-	return NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, c), nil
+	return NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, *lookBack, c), nil
 }

app/vmalert/datasource/vm.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 )
 
 type response struct {
@@ -36,7 +37,7 @@ func (r response) metrics() ([]Metric, error) {
 		}
 		m.Labels = nil
 		for k, v := range r.Data.Result[i].Labels {
-			m.Labels = append(m.Labels, Label{Name: k, Value: v})
+			m.AddLabel(k, v)
 		}
 		m.Timestamp = int64(res.TV[0].(float64))
 		m.Value = f
@@ -45,23 +46,25 @@ func (r response) metrics() ([]Metric, error) {
 	return ms, nil
 }
 
-const queryPath = "/api/v1/query?query="
-
 // VMStorage represents vmstorage entity with ability to read and write metrics
 type VMStorage struct {
 	c             *http.Client
 	queryURL      string
 	basicAuthUser string
 	basicAuthPass string
+	lookBack      time.Duration
 }
 
+const queryPath = "/api/v1/query?query="
+
 // NewVMStorage is a constructor for VMStorage
-func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, c *http.Client) *VMStorage {
+func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, c *http.Client) *VMStorage {
 	return &VMStorage{
 		c:             c,
 		basicAuthUser: basicAuthUser,
 		basicAuthPass: basicAuthPass,
 		queryURL:      strings.TrimSuffix(baseURL, "/") + queryPath,
+		lookBack:      lookBack,
 	}
 }
 
@@ -70,11 +73,16 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	const (
 		statusSuccess, statusError, rtVector = "success", "error", "vector"
 	)
-	req, err := http.NewRequest("POST", s.queryURL+url.QueryEscape(query), nil)
+	q := s.queryURL + url.QueryEscape(query)
+	if s.lookBack > 0 {
+		lookBack := time.Now().Add(-s.lookBack)
+		q += fmt.Sprintf("&time=%d", lookBack.Unix())
+	}
+	req, err := http.NewRequest("POST", q, nil)
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if s.basicAuthPass != "" {
 		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
 	}
@@ -85,7 +93,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(resp.Body)
-		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s with err %w. Reponse body %s", resp.StatusCode, req.URL, err, body)
+		return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
 	}
 	r := &response{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
@@ -98,7 +106,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
 		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
 	}
 	if r.Data.ResultType != rtVector {
-		return nil, fmt.Errorf("unknown restul type:%s. Expected vector", r.Data.ResultType)
+		return nil, fmt.Errorf("unknown result type:%s. Expected vector", r.Data.ResultType)
 	}
 	return r.metrics()
 }

app/vmalert/datasource/vm_test.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
+	"time"
 )
 
 var (
@@ -31,6 +33,13 @@ func TestVMSelectQuery(t *testing.T) {
 		if r.URL.Query().Get("query") != query {
 			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
+		timeParam := r.URL.Query().Get("time")
+		if timeParam == "" {
+			t.Errorf("expected 'time' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(timeParam, 10, 64); err != nil {
+			t.Errorf("failed to parse 'time' query param: %s", err)
+		}
 		switch c {
 		case 0:
 			conn, _, _ := w.(http.Hijacker).Hijack()
@@ -52,7 +61,7 @@ func TestVMSelectQuery(t *testing.T) {
 
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
-	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, srv.Client())
+	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, srv.Client())
 	if _, err := am.Query(ctx, query); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
@@ -66,7 +75,7 @@ func TestVMSelectQuery(t *testing.T) {
 		t.Fatalf("expected error status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
-		t.Fatalf("expected unkown status got nil")
+		t.Fatalf("expected unknown status got nil")
 	}
 	if _, err := am.Query(ctx, query); err == nil {
 		t.Fatalf("expected non-vector resultType error  got nil")
@@ -89,5 +98,4 @@ func TestVMSelectQuery(t *testing.T) {
 		m[0].Labels[0].Name != expected.Labels[0].Name {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
-
 }

app/vmalert/group_test.go

@@ -32,7 +32,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 0",
-				For:   time.Second,
+				For:   config.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"bar": "baz",
 				},
@@ -44,7 +44,7 @@ func TestUpdateWith(t *testing.T) {
 			[]config.Rule{{
 				Alert: "foo",
 				Expr:  "up > 10",
-				For:   time.Second,
+				For:   config.NewPromDuration(time.Second),
 				Labels: map[string]string{
 					"baz": "bar",
 				},
@@ -172,6 +172,11 @@ func TestGroupStart(t *testing.T) {
 		t.Fatalf("faield to create alert: %s", err)
 	}
 	alert1.State = notifier.StateFiring
+	// add external label
+	alert1.Labels["cluster"] = "east-1"
+	// add rule labels - see config/testdata/rules1-good.rules
+	alert1.Labels["label"] = "bar"
+	alert1.Labels["host"] = inst1
 	alert1.ID = hash(m1)
 
 	alert2, err := r.newAlert(m2, time.Now())
@@ -179,6 +184,11 @@ func TestGroupStart(t *testing.T) {
 		t.Fatalf("faield to create alert: %s", err)
 	}
 	alert2.State = notifier.StateFiring
+	// add external label
+	alert2.Labels["cluster"] = "east-1"
+	// add rule labels - see config/testdata/rules1-good.rules
+	alert2.Labels["label"] = "bar"
+	alert2.Labels["host"] = inst2
 	alert2.ID = hash(m2)
 
 	finished := make(chan struct{})

app/vmalert/main.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
@@ -47,6 +48,8 @@ eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+
+	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified.")
 )
 
 func main() {
@@ -58,6 +61,18 @@ func main() {
 	logger.Init()
 	cgroup.UpdateGOMAXPROCSToCPUQuota()
 
+	if *dryRun {
+		u, _ := url.Parse("https://victoriametrics.com/")
+		notifier.InitTemplateFunc(u)
+		groups, err := config.Parse(*rulePath, true, true)
+		if err != nil {
+			logger.Fatalf(err.Error())
+		}
+		if len(groups) == 0 {
+			logger.Fatalf("No rules for validation. Please specify path to file(s) with alerting and/or recording rules using `-rule` flag")
+		}
+		return
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 	manager, err := newManager(ctx)
 	if err != nil {

app/vmalert/notifier/alertmanager.go

@@ -28,7 +28,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error {
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	req = req.WithContext(ctx)
 	if am.basicAuthPass != "" {
 		req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass)

app/vmalert/notifier/init.go

@@ -26,7 +26,6 @@ var (
 // Init creates a Notifier object based on provided flags.
 func Init(gen AlertURLGenerator) ([]Notifier, error) {
 	if len(*addrs) == 0 {
-		flag.PrintDefaults()
 		return nil, fmt.Errorf("at least one `-notifier.url` must be set")
 	}
 

app/vmalert/recording.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"hash/fnv"
 	"sort"
@@ -79,8 +78,6 @@ func (rr *RecordingRule) Close() {
 	metrics.UnregisterMetric(rr.metrics.errors.name)
 }
 
-var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")
-
 // Exec executes RecordingRule expression via the given Querier.
 func (rr *RecordingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
 	if !series {

app/vmalert/remoteread/init.go

@@ -35,5 +35,5 @@ func Init() (datasource.Querier, error) {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	c := &http.Client{Transport: tr}
-	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, c), nil
+	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, 0, c), nil
 }

app/vmalert/rule.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
@@ -25,3 +26,5 @@ type Rule interface {
 	// such as metrics unregister
 	Close()
 }
+
+var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")

app/vmalert/web.go

@@ -40,7 +40,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		w.Write(data)
 		return true
 	case "/api/v1/alerts":
@@ -49,7 +49,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		w.Write(data)
 		return true
 	case "/-/reload":
@@ -67,7 +67,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		w.Write(data)
 		return true
 	}

app/vmauth/README.md

@@ -113,7 +113,7 @@ Run `make package-vmauth`. It builds `victoriametrics/vmauth:<PKG_TAG>` docker i
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

app/vmbackup/README.md

@@ -6,12 +6,12 @@ Supported storage systems for backups:
 
 * [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
-* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`
 
-Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+`vmbackup` supports incremental and full backups. Incremental backups created automatically if the destination path already contains data from the previous backup.
 Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
-data between the existing backup and new backup. This saves time and costs on data transfer.
+data between the existing backup and new backup. It saves time and costs on data transfer.
 
 Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
 
@@ -35,8 +35,8 @@ vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
   There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
 
 
@@ -49,13 +49,13 @@ with the following command:
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
 ```
 
-This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
 
 
 #### Incremental backups
 
-Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
-This saves time and network bandwidth costs when working with big backups:
+Incremental backups performed if `-dst` points to an already existing backup. In this case only new data uploaded to remote storage.
+It saves time and network bandwidth costs when working with big backups:
 
 ```
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
@@ -100,16 +100,16 @@ The backup algorithm is the following:
 2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
 3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
 4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
-   This are usually the biggest and the oldest files, which are shared between backups.
-5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+   These are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
 
-The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+The algorithm splits source files into 100 MB chunks in the backup. Each chunk stored as a separate file in the backup.
 Such splitting minimizes the amounts of data to re-transfer after temporary errors.
 
 `vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
 
 - All the files in the snapshot are immutable.
-- Old files are periodically merged into new files.
+- Old files periodically merged into new files.
 - Smaller files have higher probability to be merged.
 - Consecutive snapshots share many identical files.
 
@@ -129,7 +129,45 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 ### Advanced usage
 
-Run `vmbackup -help` in order to see all the available options:
+
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with the following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
+  You have to add a custom url endpoint via flag:
+```
+  # for minio
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+* Run `vmbackup -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -207,7 +245,7 @@ Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` dock
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

app/vmbackup/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -42,7 +43,7 @@ func main() {
 	cgroup.UpdateGOMAXPROCSToCPUQuota()
 
 	if len(*snapshotCreateURL) > 0 {
-		logger.Infof("%s", "Snapshots enabled")
+		logger.Infof("Snapshots enabled")
 		logger.Infof("Snapshot create url %s", *snapshotCreateURL)
 		if len(*snapshotDeleteURL) <= 0 {
 			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
@@ -54,17 +55,17 @@ func main() {
 
 		name, err := snapshot.Create(*snapshotCreateURL)
 		if err != nil {
-			logger.Fatalf("%s", err)
+			logger.Fatalf("cannot create snapshot: %s", err)
 		}
 		err = flag.Set("snapshotName", name)
 		if err != nil {
-			logger.Fatalf("Failed to set snapshotName flag: %v", err)
+			logger.Fatalf("cannot set snapshotName flag: %v", err)
 		}
 
 		defer func() {
 			err := snapshot.Delete(*snapshotDeleteURL, name)
 			if err != nil {
-				logger.Fatalf("%s", err)
+				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
 		}()
 	}
@@ -90,6 +91,9 @@ func main() {
 	if err := a.Run(); err != nil {
 		logger.Fatalf("cannot create backup: %s", err)
 	}
+	srcFS.MustStop()
+	dstFS.MustStop()
+	originFS.MustStop()
 }
 
 func usage() {
@@ -143,9 +147,9 @@ func newDstFS() (common.RemoteFS, error) {
 	return fs, nil
 }
 
-func newOriginFS() (common.RemoteFS, error) {
+func newOriginFS() (common.OriginFS, error) {
 	if len(*origin) == 0 {
-		return nil, nil
+		return &fsnil.FS{}, nil
 	}
 	fs, err := actions.NewRemoteFS(*origin)
 	if err != nil {

app/vmbackup/snapshot/snapshot.go

@@ -20,7 +20,7 @@ type snapshot struct {
 // Create creates a snapshot and the provided api endpoint and returns
 // the snapshot name
 func Create(createSnapshotURL string) (string, error) {
-	logger.Infof("%s", "Creating snapshot")
+	logger.Infof("Creating snapshot")
 	u, err := url.Parse(createSnapshotURL)
 	if err != nil {
 		return "", err

app/vminsert/influx/request_handler.go

@@ -65,6 +65,7 @@ func insertRows(db string, rows []parser.Row) error {
 	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Fields)
 		ic.Labels = ic.Labels[:0]
 		hasDBKey := false
 		for j := range r.Tags {
@@ -125,7 +126,6 @@ func insertRows(db string, rows []parser.Row) error {
 				}
 			}
 		}
-		rowsTotal += len(r.Fields)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vminsert/main.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheusimport"
@@ -24,6 +26,7 @@ import (
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -37,7 +40,7 @@ var (
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped")
 )
 
 var (
@@ -51,7 +54,7 @@ var (
 func Init() {
 	relabel.Init()
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
-
+	common.StartUnmarshalWorkers()
 	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
@@ -83,6 +86,7 @@ func Stop() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
+	common.StopUnmarshalWorkers()
 }
 
 // RequestHandler is a handler for Prometheus remote storage write API
@@ -125,6 +129,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
@@ -142,14 +155,31 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain")
-		promscrape.WriteHumanReadableTargetsStatus(w)
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
+		promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+		return true
+	case "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/ready":
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+			errMsg := fmt.Sprintf("waiting for scrape config to init targets, configs left: %d", rdy)
+			http.Error(w, errMsg, http.StatusTooEarly)
+		} else {
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("OK"))
+		}
+		return true
 	default:
 		// This is not our link
 		return false
@@ -169,12 +199,16 @@ var (
 	prometheusimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 	prometheusimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
 
+	nativeimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	nativeimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
 
 	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/query", protocol="influx"}`)
 
-	promscrapeTargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
+	promscrapeTargetsRequests      = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
+	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
 

app/vminsert/native/request_handler.go

@@ -0,0 +1,115 @@
+package native
+
+import (
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="native"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="native"}`)
+)
+
+// InsertHandler processes `/api/v1/import/native` request.
+func InsertHandler(req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(block *parser.Block) error {
+			return insertRows(block, extraLabels)
+		})
+	})
+}
+
+func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	// Update rowsInserted and rowsPerInsert before actual inserting,
+	// since relabeling can prevent from inserting the rows.
+	rowsLen := len(block.Values)
+	rowsInserted.Add(rowsLen)
+	rowsPerInsert.Update(float64(rowsLen))
+
+	ic := &ctx.Common
+	ic.Reset(rowsLen)
+	hasRelabeling := relabel.HasRelabeling()
+	mn := &block.MetricName
+	ic.Labels = ic.Labels[:0]
+	ic.AddLabelBytes(nil, mn.MetricGroup)
+	for j := range mn.Tags {
+		tag := &mn.Tags[j]
+		ic.AddLabelBytes(tag.Key, tag.Value)
+	}
+	for j := range extraLabels {
+		label := &extraLabels[j]
+		ic.AddLabel(label.Name, label.Value)
+	}
+	if hasRelabeling {
+		ic.ApplyRelabeling()
+	}
+	if len(ic.Labels) == 0 {
+		// Skip metric without labels.
+		return nil
+	}
+	ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
+	values := block.Values
+	timestamps := block.Timestamps
+	if len(timestamps) != len(values) {
+		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+	}
+	for j, value := range values {
+		timestamp := timestamps[j]
+		if err := ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value); err != nil {
+			return err
+		}
+	}
+	return ic.FlushBufs()
+}
+
+type pushCtx struct {
+	Common        common.InsertCtx
+	metricNameBuf []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Common.Reset(0)
+	ctx.metricNameBuf = ctx.metricNameBuf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/prompush/push.go

@@ -22,10 +22,19 @@ func Push(wr *prompbmarshal.WriteRequest) {
 	tss := wr.Timeseries
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		samplesCount := 0
+		i := 0
+		for i < len(tss) {
+			samplesCount += len(tss[i].Samples)
+			i++
+			if samplesCount > maxRowsPerBlock {
+				break
+			}
+		}
 		tssBlock := tss
-		if len(tssBlock) > maxRowsPerBlock {
-			tssBlock = tss[:maxRowsPerBlock]
-			tss = tss[maxRowsPerBlock:]
+		if i < len(tss) {
+			tssBlock = tss[:i]
+			tss = tss[i:]
 		} else {
 			tss = nil
 		}
@@ -42,6 +51,7 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 	rowsTotal := 0
 	for i := range tss {
 		ts := &tss[i]
+		rowsTotal += len(ts.Samples)
 		ctx.Labels = ctx.Labels[:0]
 		for j := range ts.Labels {
 			label := &ts.Labels[j]
@@ -62,7 +72,6 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 				return
 			}
 		}
-		rowsTotal += len(ts.Samples)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vminsert/promremotewrite/request_handler.go

@@ -36,6 +36,7 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 	hasRelabeling := relabel.HasRelabeling()
 	for i := range timeseries {
 		ts := &timeseries[i]
+		rowsTotal += len(ts.Samples)
 		ctx.Labels = ctx.Labels[:0]
 		srcLabels := ts.Labels
 		for _, srcLabel := range srcLabels {
@@ -50,14 +51,14 @@ func insertRows(timeseries []prompb.TimeSeries) error {
 		}
 		var metricNameRaw []byte
 		var err error
-		for i := range ts.Samples {
-			r := &ts.Samples[i]
+		samples := ts.Samples
+		for i := range samples {
+			r := &samples[i]
 			metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, r.Timestamp, r.Value)
 			if err != nil {
 				return err
 			}
 		}
-		rowsTotal += len(ts.Samples)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vminsert/relabel/relabel.go

@@ -69,12 +69,7 @@ type Ctx struct {
 
 // Reset resets ctx.
 func (ctx *Ctx) Reset() {
-	labels := ctx.tmpLabels
-	for i := range labels {
-		label := &labels[i]
-		label.Name = ""
-		label.Value = ""
-	}
+	promrelabel.CleanLabels(ctx.tmpLabels)
 	ctx.tmpLabels = ctx.tmpLabels[:0]
 }
 

app/vminsert/vmimport/request_handler.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
@@ -49,6 +50,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	hasRelabeling := relabel.HasRelabeling()
 	for i := range rows {
 		r := &rows[i]
+		rowsTotal += len(r.Values)
 		ic.Labels = ic.Labels[:0]
 		for j := range r.Tags {
 			tag := &r.Tags[j]
@@ -68,14 +70,15 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
 		values := r.Values
 		timestamps := r.Timestamps
-		_ = timestamps[len(values)-1]
+		if len(timestamps) != len(values) {
+			logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+		}
 		for j, value := range values {
 			timestamp := timestamps[j]
 			if err := ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value); err != nil {
 				return err
 			}
 		}
-		rowsTotal += len(values)
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))

app/vmrestore/README.md

@@ -3,7 +3,7 @@
 `vmrestore` restores data from backups created by [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
 VictoriaMetrics `v1.29.0` and newer versions must be used for working with the restored data.
 
-Restore process can be interrupted at any time. It is automatically resumed from the inerruption point
+Restore process can be interrupted at any time. It is automatically resumed from the interruption point
 when restarting `vmrestore` with the same args.
 
 
@@ -21,7 +21,7 @@ vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/r
 * `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
   to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
 
-The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup,
+The original `-storageDataPath` directory may contain old files. They will be substituted by the files from backup,
 i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/questions/476041/how-do-i-make-rsync-delete-files-that-have-been-deleted-from-the-source-folder).
 
 
@@ -33,7 +33,44 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 
 ### Advanced usage
 
-Run `vmrestore -help` in order to see all the available options:
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint.  It is possible to use `vmrestore` with s3 api compatible storages, like  minio, cloudian and other.
+  You have to add custom url endpoint with a flag:
+```
+  # for minio:
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+*  Run `vmrestore -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -104,7 +141,7 @@ Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` do
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

app/vmrestore/main.go

@@ -52,6 +52,8 @@ func main() {
 	if err := a.Run(); err != nil {
 		logger.Fatalf("cannot restore from backup: %s", err)
 	}
+	srcFS.MustStop()
+	dstFS.MustStop()
 }
 
 func usage() {

app/vmselect/bufferedwriter/bufferedwriter.go

@@ -0,0 +1,86 @@
+package bufferedwriter
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// Get returns buffered writer for the given w.
+//
+// The writer must be returned to the pool after use by calling Put().
+func Get(w io.Writer) *Writer {
+	v := writerPool.Get()
+	if v == nil {
+		v = &Writer{
+			// By default net/http.Server uses 4KB buffers, which are flushed to client with chunked responses.
+			// These buffers may result in visible overhead for responses exceeding a few megabytes.
+			// So allocate 64Kb buffers.
+			bw: bufio.NewWriterSize(w, 64*1024),
+		}
+	}
+	bw := v.(*Writer)
+	bw.bw.Reset(w)
+	return bw
+}
+
+// Put returns back bw to the pool.
+//
+// bw cannot be used after returning to the pool.
+func Put(bw *Writer) {
+	bw.reset()
+	writerPool.Put(bw)
+}
+
+var writerPool sync.Pool
+
+// Writer is buffered writer, which may be used in order to reduce overhead
+// when sending moderately big responses to http server.
+//
+// Writer methods can be called from concurrently running goroutines.
+// The writer remembers the first occurred error, which can be inspected with Error method.
+type Writer struct {
+	lock sync.Mutex
+	bw   *bufio.Writer
+	err  error
+}
+
+func (bw *Writer) reset() {
+	bw.bw.Reset(nil)
+	bw.err = nil
+}
+
+// Write writes p to bw.
+func (bw *Writer) Write(p []byte) (int, error) {
+	bw.lock.Lock()
+	defer bw.lock.Unlock()
+	if bw.err != nil {
+		return 0, bw.err
+	}
+	n, err := bw.bw.Write(p)
+	if err != nil {
+		bw.err = fmt.Errorf("cannot send %d bytes to client: %w", len(p), err)
+	}
+	return n, bw.err
+}
+
+// Flush flushes bw to the underlying writer.
+func (bw *Writer) Flush() error {
+	bw.lock.Lock()
+	defer bw.lock.Unlock()
+	if bw.err != nil {
+		return bw.err
+	}
+	if err := bw.bw.Flush(); err != nil {
+		bw.err = fmt.Errorf("cannot flush data to client: %w", err)
+	}
+	return bw.err
+}
+
+// Error returns the first occurred error in bw.
+func (bw *Writer) Error() error {
+	bw.lock.Lock()
+	defer bw.lock.Unlock()
+	return bw.err
+}

app/vmselect/graphite/metrics_api.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -74,24 +75,44 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 		MinTimestamp: from,
 		MaxTimestamp: until,
 	}
-	paths, err := metricsFind(tr, label, query, delimiter[0], deadline)
+	paths, err := metricsFind(tr, label, query, delimiter[0], false, deadline)
 	if err != nil {
 		return err
 	}
 	if leavesOnly {
 		paths = filterLeaves(paths, delimiter)
 	}
+	paths = deduplicatePaths(paths, delimiter)
 	sortPaths(paths, delimiter)
-	contentType := "application/json"
-	if jsonp != "" {
-		contentType = "text/javascript"
-	}
+	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
-	WriteMetricsFindResponse(w, paths, delimiter, format, wildcards, jsonp)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetricsFindResponse(bw, paths, delimiter, format, wildcards, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	metricsFindDuration.UpdateDuration(startTime)
 	return nil
 }
 
+func deduplicatePaths(paths []string, delimiter string) []string {
+	if len(paths) == 0 {
+		return nil
+	}
+	sort.Strings(paths)
+	dst := paths[:1]
+	for _, path := range paths[1:] {
+		prevPath := dst[len(dst)-1]
+		if path == prevPath {
+			// Skip duplicate path.
+			continue
+		}
+		dst = append(dst, path)
+	}
+	return dst
+}
+
 // MetricsExpandHandler implements /metrics/expand handler.
 //
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
@@ -133,7 +154,7 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 	}
 	m := make(map[string][]string, len(queries))
 	for _, query := range queries {
-		paths, err := metricsFind(tr, label, query, delimiter[0], deadline)
+		paths, err := metricsFind(tr, label, query, delimiter[0], true, deadline)
 		if err != nil {
 			return err
 		}
@@ -142,10 +163,7 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 		}
 		m[query] = paths
 	}
-	contentType := "application/json"
-	if jsonp != "" {
-		contentType = "text/javascript"
-	}
+	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
 	if groupByExpr {
 		for _, paths := range m {
@@ -168,7 +186,12 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
 		}
 	}
 	sortPaths(paths, delimiter)
-	WriteMetricsExpandResponseFlat(w, paths, jsonp)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetricsExpandResponseFlat(bw, paths, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	metricsExpandDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -186,40 +209,46 @@ func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
 	if err != nil {
 		return fmt.Errorf(`cannot obtain metric names: %w`, err)
 	}
-	contentType := "application/json"
-	if jsonp != "" {
-		contentType = "text/javascript"
-	}
+	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
-	WriteMetricsIndexResponse(w, metricNames, jsonp)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetricsIndexResponse(bw, metricNames, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	metricsIndexDuration.UpdateDuration(startTime)
 	return nil
 }
 
 // metricsFind searches for label values that match the given query.
-func metricsFind(tr storage.TimeRange, label, query string, delimiter byte, deadline searchutils.Deadline) ([]string, error) {
-	expandTail := strings.HasSuffix(query, "*")
-	for strings.HasSuffix(query, "*") {
-		query = query[:len(query)-1]
-	}
-	var results []string
+func metricsFind(tr storage.TimeRange, label, query string, delimiter byte, isExpand bool, deadline searchutils.Deadline) ([]string, error) {
 	n := strings.IndexAny(query, "*{[")
-	if n < 0 {
+	if n < 0 || n == len(query)-1 && strings.HasSuffix(query, "*") {
+		expandTail := n >= 0
+		if expandTail {
+			query = query[:len(query)-1]
+		}
 		suffixes, err := netstorage.GetTagValueSuffixes(tr, label, query, delimiter, deadline)
 		if err != nil {
 			return nil, err
 		}
-		if expandTail {
-			for _, suffix := range suffixes {
+		if len(suffixes) == 0 {
+			return nil, nil
+		}
+		if !expandTail && len(query) > 0 && query[len(query)-1] == delimiter {
+			return []string{query}, nil
+		}
+		results := make([]string, 0, len(suffixes))
+		for _, suffix := range suffixes {
+			if expandTail || len(suffix) == 0 || len(suffix) == 1 && suffix[0] == delimiter {
 				results = append(results, query+suffix)
 			}
-		} else if isFullMatch(query, suffixes, delimiter) {
-			results = append(results, query)
 		}
 		return results, nil
 	}
 	subquery := query[:n] + "*"
-	paths, err := metricsFind(tr, label, subquery, delimiter, deadline)
+	paths, err := metricsFind(tr, label, subquery, delimiter, isExpand, deadline)
 	if err != nil {
 		return nil, err
 	}
@@ -229,24 +258,32 @@ func metricsFind(tr storage.TimeRange, label, query string, delimiter byte, dead
 		tail = suffix[m+1:]
 		suffix = suffix[:m+1]
 	}
-	q := query[:n] + suffix
-	re, err := getRegexpForQuery(q, delimiter)
+	qPrefix := query[:n] + suffix
+	rePrefix, err := getRegexpForQuery(qPrefix, delimiter)
 	if err != nil {
-		return nil, fmt.Errorf("cannot convert query %q to regexp: %w", q, err)
-	}
-	if expandTail {
-		tail += "*"
+		return nil, fmt.Errorf("cannot convert query %q to regexp: %w", qPrefix, err)
 	}
+	results := make([]string, 0, len(paths))
 	for _, path := range paths {
-		if !re.MatchString(path) {
+		if !rePrefix.MatchString(path) {
+			continue
+		}
+		if tail == "" {
+			results = append(results, path)
 			continue
 		}
 		subquery := path + tail
-		tmp, err := metricsFind(tr, label, subquery, delimiter, deadline)
+		fullPaths, err := metricsFind(tr, label, subquery, delimiter, isExpand, deadline)
 		if err != nil {
 			return nil, err
 		}
-		results = append(results, tmp...)
+		if isExpand {
+			results = append(results, fullPaths...)
+		} else {
+			for _, fullPath := range fullPaths {
+				results = append(results, qPrefix+fullPath[len(path):])
+			}
+		}
 	}
 	return results, nil
 }
@@ -257,21 +294,6 @@ var (
 	metricsIndexDuration  = metrics.NewSummary(`vm_request_duration_seconds{path="/metrics/index.json"}`)
 )
 
-func isFullMatch(tagValuePrefix string, suffixes []string, delimiter byte) bool {
-	if len(suffixes) == 0 {
-		return false
-	}
-	if strings.LastIndexByte(tagValuePrefix, delimiter) == len(tagValuePrefix)-1 {
-		return true
-	}
-	for _, suffix := range suffixes {
-		if suffix == "" {
-			return true
-		}
-	}
-	return false
-}
-
 func addAutomaticVariants(query, delimiter string) string {
 	// See https://github.com/graphite-project/graphite-web/blob/bb9feb0e6815faa73f538af6ed35adea0fb273fd/webapp/graphite/metrics/views.py#L152
 	parts := strings.Split(query, delimiter)
@@ -317,7 +339,8 @@ func getRegexpForQuery(query string, delimiter byte) (*regexp.Regexp, error) {
 		return re.re, re.err
 	}
 	a := make([]string, 0, len(query))
-	tillNextDelimiter := "[^" + regexp.QuoteMeta(string([]byte{delimiter})) + "]*"
+	quotedDelimiter := regexp.QuoteMeta(string([]byte{delimiter}))
+	tillNextDelimiter := "[^" + quotedDelimiter + "]*"
 	for i := 0; i < len(query); i++ {
 		switch query[i] {
 		case '*':
@@ -351,6 +374,10 @@ func getRegexpForQuery(query string, delimiter byte) (*regexp.Regexp, error) {
 		}
 	}
 	s := strings.Join(a, "")
+	if !strings.HasSuffix(s, quotedDelimiter) {
+		s += quotedDelimiter + "?"
+	}
+	s = "^(?:" + s + ")$"
 	re, err := regexp.Compile(s)
 	regexpCache[k] = &regexpCacheEntry{
 		re:  re,
@@ -381,3 +408,10 @@ var regexpCache = make(map[regexpCacheKey]*regexpCacheEntry)
 var regexpCacheLock sync.Mutex
 
 const maxRegexpCacheSize = 10000
+
+func getContentType(jsonp string) string {
+	if jsonp == "" {
+		return "application/json; charset=utf-8"
+	}
+	return "text/javascript; charset=utf-8"
+}

app/vmselect/graphite/metrics_api_test.go

@@ -17,13 +17,17 @@ func TestGetRegexpForQuery(t *testing.T) {
 			t.Fatalf("unexpected regexp for query=%q, delimiter=%c; got %s; want %s", query, delimiter, reStr, reExpected)
 		}
 	}
-	f("", '.', "")
-	f("foobar", '.', "foobar")
-	f("*", '.', `[^\.]*`)
-	f("*", '_', `[^_]*`)
-	f("foo.*.bar", '.', `foo\.[^\.]*\.bar`)
-	f("fo*b{ar,aaa}[a-z]xx*.d", '.', `fo[^\.]*b(?:ar|aaa)[a-z]xx[^\.]*\.d`)
-	f("fo*b{ar,aaa}[a-z]xx*_d", '_', `fo[^_]*b(?:ar|aaa)[a-z]xx[^_]*_d`)
+	f("", '.', `^(?:\.?)$`)
+	f("foobar", '.', `^(?:foobar\.?)$`)
+	f("*", '.', `^(?:[^\.]*\.?)$`)
+	f("*", '_', `^(?:[^_]*_?)$`)
+	f("foo.*.bar", '.', `^(?:foo\.[^\.]*\.bar\.?)$`)
+	f("fo*b{ar,aaa}[a-z]xx*.d", '.', `^(?:fo[^\.]*b(?:ar|aaa)[a-z]xx[^\.]*\.d\.?)$`)
+	f("fo*b{ar,aaa}[a-z]xx*_d", '_', `^(?:fo[^_]*b(?:ar|aaa)[a-z]xx[^_]*_d_?)$`)
+	f("foo.[ab]*z", '.', `^(?:foo\.[ab][^\.]*z\.?)$`)
+	f("foo_[ab]*", '_', `^(?:foo_[ab][^_]*_?)$`)
+	f("foo_[ab]_", '_', `^(?:foo_[ab]_)$`)
+	f("foo.[ab].", '.', `^(?:foo\.[ab]\.)$`)
 }
 
 func TestSortPaths(t *testing.T) {

app/vmselect/graphite/metrics_find_response.qtpl

@@ -1,4 +1,5 @@
 {% import (
+	"sort"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -43,17 +44,44 @@ See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
 
 {% func metricsFindResponseTreeJSON(paths []string, delimiter string, addWildcards bool) %}
 [
+{% code
+	if len(paths) > 1 {
+		sort.Strings(paths)
+		// Substitute `path` and `path<delimiter>` with `path<delimiter><delimiter>`.
+		// Such path is treated specially during rendering - see code below for details.
+		dst := paths[:1]
+		for _, path := range paths[1:] {
+			prevPath := dst[len(dst)-1]
+			if len(path) == len(prevPath)+1 && strings.HasSuffix(path, delimiter) && strings.HasPrefix(path, prevPath) {
+				// The path is equivalent to <prevPath> + <delimiter>
+				// Overwrite the prevPath with <path> + <delimiter> as carbonapi does.
+				// I.e. the resulting path ends with double delimiter.
+				// Such path is treated specially during rendering - see metrics_find_response.qtpl for details.
+				dst[len(dst)-1] = path + delimiter
+				continue
+			}
+			dst = append(dst, path)
+		}
+		paths = dst
+	}
+%}
 	{% for i, path := range paths %}
 	{
 		{% code
+			id := path
 			allowChildren := "0"
 			isLeaf := "1"
-			if strings.HasSuffix(path, delimiter) {
+			if strings.HasSuffix(id, delimiter) {
+				if strings.HasSuffix(id[:len(id)-1], delimiter) {
+					// Special case when id ends with double delimiter.
+					// See the code above for details.
+					id = id[:len(id)-2]
+				}
 				allowChildren = "1"
 				isLeaf = "0"
 			}
 		%}
-		"id": {%q= path %},
+		"id": {%q= id %},
 		"text": {%= metricPathName(path, delimiter) %},
 		"allowChildren": {%s= allowChildren %},
 		"expandable": {%s= allowChildren %},

app/vmselect/graphite/metrics_find_response.qtpl.go

@@ -6,6 +6,7 @@ package graphite
 
 //line app/vmselect/graphite/metrics_find_response.qtpl:1
 import (
+	"sort"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -13,205 +14,232 @@ import (
 
 // MetricsFindResponse generates response for /metrics/find .See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:11
+//line app/vmselect/graphite/metrics_find_response.qtpl:12
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:11
+//line app/vmselect/graphite/metrics_find_response.qtpl:12
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:11
+//line app/vmselect/graphite/metrics_find_response.qtpl:12
 func StreamMetricsFindResponse(qw422016 *qt422016.Writer, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:12
-	if jsonp != "" {
-//line app/vmselect/graphite/metrics_find_response.qtpl:12
-		qw422016.N().S(jsonp)
-//line app/vmselect/graphite/metrics_find_response.qtpl:12
-		qw422016.N().S(`(`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:12
-	}
 //line app/vmselect/graphite/metrics_find_response.qtpl:13
-	switch format {
+	if jsonp != "" {
+//line app/vmselect/graphite/metrics_find_response.qtpl:13
+		qw422016.N().S(jsonp)
+//line app/vmselect/graphite/metrics_find_response.qtpl:13
+		qw422016.N().S(`(`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:13
+	}
 //line app/vmselect/graphite/metrics_find_response.qtpl:14
-	case "completer":
+	switch format {
 //line app/vmselect/graphite/metrics_find_response.qtpl:15
-		streammetricsFindResponseCompleter(qw422016, paths, delimiter, addWildcards)
+	case "completer":
 //line app/vmselect/graphite/metrics_find_response.qtpl:16
-	case "treejson":
+		streammetricsFindResponseCompleter(qw422016, paths, delimiter, addWildcards)
 //line app/vmselect/graphite/metrics_find_response.qtpl:17
-		streammetricsFindResponseTreeJSON(qw422016, paths, delimiter, addWildcards)
+	case "treejson":
 //line app/vmselect/graphite/metrics_find_response.qtpl:18
-	default:
+		streammetricsFindResponseTreeJSON(qw422016, paths, delimiter, addWildcards)
 //line app/vmselect/graphite/metrics_find_response.qtpl:19
+	default:
+//line app/vmselect/graphite/metrics_find_response.qtpl:20
 		logger.Panicf("BUG: unexpected format=%q", format)
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:20
-	}
 //line app/vmselect/graphite/metrics_find_response.qtpl:21
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:22
 	if jsonp != "" {
-//line app/vmselect/graphite/metrics_find_response.qtpl:21
+//line app/vmselect/graphite/metrics_find_response.qtpl:22
 		qw422016.N().S(`)`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:21
-	}
 //line app/vmselect/graphite/metrics_find_response.qtpl:22
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 func WriteMetricsFindResponse(qq422016 qtio422016.Writer, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	StreamMetricsFindResponse(qw422016, paths, delimiter, format, addWildcards, jsonp)
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 func MetricsFindResponse(paths []string, delimiter, format string, addWildcards bool, jsonp string) string {
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	WriteMetricsFindResponse(qb422016, paths, delimiter, format, addWildcards, jsonp)
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	qs422016 := string(qb422016.B)
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 	return qs422016
-//line app/vmselect/graphite/metrics_find_response.qtpl:22
+//line app/vmselect/graphite/metrics_find_response.qtpl:23
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:24
+//line app/vmselect/graphite/metrics_find_response.qtpl:25
 func streammetricsFindResponseCompleter(qw422016 *qt422016.Writer, paths []string, delimiter string, addWildcards bool) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:24
+//line app/vmselect/graphite/metrics_find_response.qtpl:25
 	qw422016.N().S(`{"metrics":[`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:27
+//line app/vmselect/graphite/metrics_find_response.qtpl:28
 	for i, path := range paths {
-//line app/vmselect/graphite/metrics_find_response.qtpl:27
+//line app/vmselect/graphite/metrics_find_response.qtpl:28
 		qw422016.N().S(`{"path":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:29
+//line app/vmselect/graphite/metrics_find_response.qtpl:30
 		qw422016.N().Q(path)
-//line app/vmselect/graphite/metrics_find_response.qtpl:29
+//line app/vmselect/graphite/metrics_find_response.qtpl:30
 		qw422016.N().S(`,"name":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:30
+//line app/vmselect/graphite/metrics_find_response.qtpl:31
 		streammetricPathName(qw422016, path, delimiter)
-//line app/vmselect/graphite/metrics_find_response.qtpl:30
+//line app/vmselect/graphite/metrics_find_response.qtpl:31
 		qw422016.N().S(`,"is_leaf":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:31
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
 		if strings.HasSuffix(path, delimiter) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:31
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
 			qw422016.N().S(`0`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:31
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
 		} else {
-//line app/vmselect/graphite/metrics_find_response.qtpl:31
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
 			qw422016.N().S(`1`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:31
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
 		}
-//line app/vmselect/graphite/metrics_find_response.qtpl:31
+//line app/vmselect/graphite/metrics_find_response.qtpl:32
 		qw422016.N().S(`}`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:33
-		if i+1 < len(paths) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:33
-			qw422016.N().S(`,`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:33
-		}
 //line app/vmselect/graphite/metrics_find_response.qtpl:34
-	}
+		if i+1 < len(paths) {
+//line app/vmselect/graphite/metrics_find_response.qtpl:34
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/metrics_find_response.qtpl:34
+		}
 //line app/vmselect/graphite/metrics_find_response.qtpl:35
+	}
+//line app/vmselect/graphite/metrics_find_response.qtpl:36
 	if addWildcards && len(paths) > 1 {
-//line app/vmselect/graphite/metrics_find_response.qtpl:35
+//line app/vmselect/graphite/metrics_find_response.qtpl:36
 		qw422016.N().S(`,{"name": "*"}`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:39
+//line app/vmselect/graphite/metrics_find_response.qtpl:40
 	}
-//line app/vmselect/graphite/metrics_find_response.qtpl:39
+//line app/vmselect/graphite/metrics_find_response.qtpl:40
 	qw422016.N().S(`]}`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 func writemetricsFindResponseCompleter(qq422016 qtio422016.Writer, paths []string, delimiter string, addWildcards bool) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	streammetricsFindResponseCompleter(qw422016, paths, delimiter, addWildcards)
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 func metricsFindResponseCompleter(paths []string, delimiter string, addWildcards bool) string {
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	writemetricsFindResponseCompleter(qb422016, paths, delimiter, addWildcards)
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	qs422016 := string(qb422016.B)
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 	return qs422016
-//line app/vmselect/graphite/metrics_find_response.qtpl:42
+//line app/vmselect/graphite/metrics_find_response.qtpl:43
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:44
+//line app/vmselect/graphite/metrics_find_response.qtpl:45
 func streammetricsFindResponseTreeJSON(qw422016 *qt422016.Writer, paths []string, delimiter string, addWildcards bool) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:44
+//line app/vmselect/graphite/metrics_find_response.qtpl:45
 	qw422016.N().S(`[`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:46
+//line app/vmselect/graphite/metrics_find_response.qtpl:48
+	if len(paths) > 1 {
+		sort.Strings(paths)
+		// Substitute `path` and `path<delimiter>` with `path<delimiter><delimiter>`.
+		// Such path is treated specially during rendering - see code below for details.
+		dst := paths[:1]
+		for _, path := range paths[1:] {
+			prevPath := dst[len(dst)-1]
+			if len(path) == len(prevPath)+1 && strings.HasSuffix(path, delimiter) && strings.HasPrefix(path, prevPath) {
+				// The path is equivalent to <prevPath> + <delimiter>
+				// Overwrite the prevPath with <path> + <delimiter> as carbonapi does.
+				// I.e. the resulting path ends with double delimiter.
+				// Such path is treated specially during rendering - see metrics_find_response.qtpl for details.
+				dst[len(dst)-1] = path + delimiter
+				continue
+			}
+			dst = append(dst, path)
+		}
+		paths = dst
+	}
+
+//line app/vmselect/graphite/metrics_find_response.qtpl:68
 	for i, path := range paths {
-//line app/vmselect/graphite/metrics_find_response.qtpl:46
+//line app/vmselect/graphite/metrics_find_response.qtpl:68
 		qw422016.N().S(`{`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:49
+//line app/vmselect/graphite/metrics_find_response.qtpl:71
+		id := path
 		allowChildren := "0"
 		isLeaf := "1"
-		if strings.HasSuffix(path, delimiter) {
+		if strings.HasSuffix(id, delimiter) {
+			if strings.HasSuffix(id[:len(id)-1], delimiter) {
+				// Special case when id ends with double delimiter.
+				// See the code above for details.
+				id = id[:len(id)-2]
+			}
 			allowChildren = "1"
 			isLeaf = "0"
 		}
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:55
+//line app/vmselect/graphite/metrics_find_response.qtpl:83
 		qw422016.N().S(`"id":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:56
-		qw422016.N().Q(path)
-//line app/vmselect/graphite/metrics_find_response.qtpl:56
+//line app/vmselect/graphite/metrics_find_response.qtpl:84
+		qw422016.N().Q(id)
+//line app/vmselect/graphite/metrics_find_response.qtpl:84
 		qw422016.N().S(`,"text":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:57
+//line app/vmselect/graphite/metrics_find_response.qtpl:85
 		streammetricPathName(qw422016, path, delimiter)
-//line app/vmselect/graphite/metrics_find_response.qtpl:57
+//line app/vmselect/graphite/metrics_find_response.qtpl:85
 		qw422016.N().S(`,"allowChildren":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:58
+//line app/vmselect/graphite/metrics_find_response.qtpl:86
 		qw422016.N().S(allowChildren)
-//line app/vmselect/graphite/metrics_find_response.qtpl:58
+//line app/vmselect/graphite/metrics_find_response.qtpl:86
 		qw422016.N().S(`,"expandable":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:59
+//line app/vmselect/graphite/metrics_find_response.qtpl:87
 		qw422016.N().S(allowChildren)
-//line app/vmselect/graphite/metrics_find_response.qtpl:59
+//line app/vmselect/graphite/metrics_find_response.qtpl:87
 		qw422016.N().S(`,"leaf":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:60
+//line app/vmselect/graphite/metrics_find_response.qtpl:88
 		qw422016.N().S(isLeaf)
-//line app/vmselect/graphite/metrics_find_response.qtpl:60
+//line app/vmselect/graphite/metrics_find_response.qtpl:88
 		qw422016.N().S(`}`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:62
+//line app/vmselect/graphite/metrics_find_response.qtpl:90
 		if i+1 < len(paths) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:62
+//line app/vmselect/graphite/metrics_find_response.qtpl:90
 			qw422016.N().S(`,`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:62
+//line app/vmselect/graphite/metrics_find_response.qtpl:90
 		}
-//line app/vmselect/graphite/metrics_find_response.qtpl:63
+//line app/vmselect/graphite/metrics_find_response.qtpl:91
 	}
-//line app/vmselect/graphite/metrics_find_response.qtpl:64
+//line app/vmselect/graphite/metrics_find_response.qtpl:92
 	if addWildcards && len(paths) > 1 {
-//line app/vmselect/graphite/metrics_find_response.qtpl:64
+//line app/vmselect/graphite/metrics_find_response.qtpl:92
 		qw422016.N().S(`,{`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:67
+//line app/vmselect/graphite/metrics_find_response.qtpl:95
 		path := paths[0]
 		for strings.HasSuffix(path, delimiter) {
 			path = path[:len(path)-1]
@@ -232,60 +260,60 @@ func streammetricsFindResponseTreeJSON(qw422016 *qt422016.Writer, paths []string
 			}
 		}
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:86
+//line app/vmselect/graphite/metrics_find_response.qtpl:114
 		qw422016.N().S(`"id":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:87
+//line app/vmselect/graphite/metrics_find_response.qtpl:115
 		qw422016.N().Q(id)
-//line app/vmselect/graphite/metrics_find_response.qtpl:87
+//line app/vmselect/graphite/metrics_find_response.qtpl:115
 		qw422016.N().S(`,"text": "*","allowChildren":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:89
+//line app/vmselect/graphite/metrics_find_response.qtpl:117
 		qw422016.N().S(allowChildren)
-//line app/vmselect/graphite/metrics_find_response.qtpl:89
+//line app/vmselect/graphite/metrics_find_response.qtpl:117
 		qw422016.N().S(`,"expandable":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:90
+//line app/vmselect/graphite/metrics_find_response.qtpl:118
 		qw422016.N().S(allowChildren)
-//line app/vmselect/graphite/metrics_find_response.qtpl:90
+//line app/vmselect/graphite/metrics_find_response.qtpl:118
 		qw422016.N().S(`,"leaf":`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:91
+//line app/vmselect/graphite/metrics_find_response.qtpl:119
 		qw422016.N().S(isLeaf)
-//line app/vmselect/graphite/metrics_find_response.qtpl:91
+//line app/vmselect/graphite/metrics_find_response.qtpl:119
 		qw422016.N().S(`}`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:93
+//line app/vmselect/graphite/metrics_find_response.qtpl:121
 	}
-//line app/vmselect/graphite/metrics_find_response.qtpl:93
+//line app/vmselect/graphite/metrics_find_response.qtpl:121
 	qw422016.N().S(`]`)
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 func writemetricsFindResponseTreeJSON(qq422016 qtio422016.Writer, paths []string, delimiter string, addWildcards bool) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	streammetricsFindResponseTreeJSON(qw422016, paths, delimiter, addWildcards)
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 func metricsFindResponseTreeJSON(paths []string, delimiter string, addWildcards bool) string {
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	writemetricsFindResponseTreeJSON(qb422016, paths, delimiter, addWildcards)
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	qs422016 := string(qb422016.B)
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 	return qs422016
-//line app/vmselect/graphite/metrics_find_response.qtpl:95
+//line app/vmselect/graphite/metrics_find_response.qtpl:123
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:97
+//line app/vmselect/graphite/metrics_find_response.qtpl:125
 func streammetricPathName(qw422016 *qt422016.Writer, path, delimiter string) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:99
+//line app/vmselect/graphite/metrics_find_response.qtpl:127
 	name := path
 	for strings.HasSuffix(name, delimiter) {
 		name = name[:len(name)-1]
@@ -294,33 +322,33 @@ func streammetricPathName(qw422016 *qt422016.Writer, path, delimiter string) {
 		name = name[n+1:]
 	}
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:107
+//line app/vmselect/graphite/metrics_find_response.qtpl:135
 	qw422016.N().Q(name)
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 func writemetricPathName(qq422016 qtio422016.Writer, path, delimiter string) {
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	streammetricPathName(qw422016, path, delimiter)
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 }
 
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 func metricPathName(path, delimiter string) string {
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	writemetricPathName(qb422016, path, delimiter)
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	qs422016 := string(qb422016.B)
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 	return qs422016
-//line app/vmselect/graphite/metrics_find_response.qtpl:108
+//line app/vmselect/graphite/metrics_find_response.qtpl:136
 }

app/vmselect/graphite/tag_values_response.qtpl

@@ -0,0 +1,20 @@
+{% stripspace %}
+
+Tags generates response for /tags/<tag_name> handler
+See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
+{% func TagValuesResponse(tagName string, tagValues []string) %}
+{
+	"tag":{%q= tagName %},
+	"values":[
+	{% for i, value := range tagValues %}
+		{
+			"count":1,
+			"value":{%q= value %}
+		}
+		{% if i+1 < len(tagValues) %},{% endif %}
+	{% endfor %}
+	]
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/tag_values_response.qtpl.go

@@ -0,0 +1,75 @@
+// Code generated by qtc from "tag_values_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// Tags generates response for /tags/<tag_name> handlerSee https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
+
+//line app/vmselect/graphite/tag_values_response.qtpl:5
+package graphite
+
+//line app/vmselect/graphite/tag_values_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/tag_values_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/tag_values_response.qtpl:5
+func StreamTagValuesResponse(qw422016 *qt422016.Writer, tagName string, tagValues []string) {
+//line app/vmselect/graphite/tag_values_response.qtpl:5
+	qw422016.N().S(`{"tag":`)
+//line app/vmselect/graphite/tag_values_response.qtpl:7
+	qw422016.N().Q(tagName)
+//line app/vmselect/graphite/tag_values_response.qtpl:7
+	qw422016.N().S(`,"values":[`)
+//line app/vmselect/graphite/tag_values_response.qtpl:9
+	for i, value := range tagValues {
+//line app/vmselect/graphite/tag_values_response.qtpl:9
+		qw422016.N().S(`{"count":1,"value":`)
+//line app/vmselect/graphite/tag_values_response.qtpl:12
+		qw422016.N().Q(value)
+//line app/vmselect/graphite/tag_values_response.qtpl:12
+		qw422016.N().S(`}`)
+//line app/vmselect/graphite/tag_values_response.qtpl:14
+		if i+1 < len(tagValues) {
+//line app/vmselect/graphite/tag_values_response.qtpl:14
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/tag_values_response.qtpl:14
+		}
+//line app/vmselect/graphite/tag_values_response.qtpl:15
+	}
+//line app/vmselect/graphite/tag_values_response.qtpl:15
+	qw422016.N().S(`]}`)
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+}
+
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+func WriteTagValuesResponse(qq422016 qtio422016.Writer, tagName string, tagValues []string) {
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	StreamTagValuesResponse(qw422016, tagName, tagValues)
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+}
+
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+func TagValuesResponse(tagName string, tagValues []string) string {
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	WriteTagValuesResponse(qb422016, tagName, tagValues)
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+	return qs422016
+//line app/vmselect/graphite/tag_values_response.qtpl:18
+}

app/vmselect/graphite/tags_api.go

@@ -0,0 +1,504 @@
+package graphite
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// TagsDelSeriesHandler implements /tags/delSeries handler.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb
+func TagsDelSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	paths := r.Form["path"]
+	totalDeleted := 0
+	var row graphiteparser.Row
+	var tagsPool []graphiteparser.Tag
+	ct := time.Now().UnixNano() / 1e6
+	for _, path := range paths {
+		var err error
+		tagsPool, err = row.UnmarshalMetricAndTags(path, tagsPool[:0])
+		if err != nil {
+			return fmt.Errorf("cannot parse path=%q: %w", path, err)
+		}
+		tfs := make([]storage.TagFilter, 0, 1+len(row.Tags))
+		tfs = append(tfs, storage.TagFilter{
+			Key:   nil,
+			Value: []byte(row.Metric),
+		})
+		for _, tag := range row.Tags {
+			tfs = append(tfs, storage.TagFilter{
+				Key:   []byte(tag.Key),
+				Value: []byte(tag.Value),
+			})
+		}
+		sq := storage.NewSearchQuery(0, ct, [][]storage.TagFilter{tfs})
+		n, err := netstorage.DeleteSeries(sq)
+		if err != nil {
+			return fmt.Errorf("cannot delete series for %q: %w", sq, err)
+		}
+		totalDeleted += n
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	if totalDeleted > 0 {
+		fmt.Fprintf(w, "true")
+	} else {
+		fmt.Fprintf(w, "false")
+	}
+	return nil
+}
+
+// TagsTagSeriesHandler implements /tags/tagSeries handler.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
+func TagsTagSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	return registerMetrics(startTime, w, r, false)
+}
+
+// TagsTagMultiSeriesHandler implements /tags/tagMultiSeries handler.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
+func TagsTagMultiSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	return registerMetrics(startTime, w, r, true)
+}
+
+func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request, isJSONResponse bool) error {
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	paths := r.Form["path"]
+	var row graphiteparser.Row
+	var labels []prompb.Label
+	var b []byte
+	var tagsPool []graphiteparser.Tag
+	mrs := make([]storage.MetricRow, len(paths))
+	ct := time.Now().UnixNano() / 1e6
+	canonicalPaths := make([]string, len(paths))
+	for i, path := range paths {
+		var err error
+		tagsPool, err = row.UnmarshalMetricAndTags(path, tagsPool[:0])
+		if err != nil {
+			return fmt.Errorf("cannot parse path=%q: %w", path, err)
+		}
+
+		// Construct canonical path according to https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
+		sort.Slice(row.Tags, func(i, j int) bool {
+			return row.Tags[i].Key < row.Tags[j].Key
+		})
+		b = append(b[:0], row.Metric...)
+		for _, tag := range row.Tags {
+			b = append(b, ';')
+			b = append(b, tag.Key...)
+			b = append(b, '=')
+			b = append(b, tag.Value...)
+		}
+		canonicalPaths[i] = string(b)
+
+		// Convert parsed metric and tags to labels.
+		labels = append(labels[:0], prompb.Label{
+			Name:  []byte("__name__"),
+			Value: []byte(row.Metric),
+		})
+		for _, tag := range row.Tags {
+			labels = append(labels, prompb.Label{
+				Name:  []byte(tag.Key),
+				Value: []byte(tag.Value),
+			})
+		}
+
+		// Put labels with the current timestamp to MetricRow
+		mr := &mrs[i]
+		mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
+		mr.Timestamp = ct
+	}
+	if err := vmstorage.RegisterMetricNames(mrs); err != nil {
+		return fmt.Errorf("cannot register paths: %w", err)
+	}
+
+	// Return response
+	contentType := "text/plain; charset=utf-8"
+	if isJSONResponse {
+		contentType = "application/json; charset=utf-8"
+	}
+	w.Header().Set("Content-Type", contentType)
+	WriteTagsTagMultiSeriesResponse(w, canonicalPaths, isJSONResponse)
+	if isJSONResponse {
+		tagsTagMultiSeriesDuration.UpdateDuration(startTime)
+	} else {
+		tagsTagSeriesDuration.UpdateDuration(startTime)
+	}
+	return nil
+}
+
+var (
+	tagsTagSeriesDuration      = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/tagSeries"}`)
+	tagsTagMultiSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/tagMultiSeries"}`)
+)
+
+// TagsAutoCompleteValuesHandler implements /tags/autoComplete/values endpoint from Graphite Tags API.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
+func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	limit, err := getInt(r, "limit")
+	if err != nil {
+		return err
+	}
+	if limit <= 0 {
+		// Use limit=100 by default. See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
+		limit = 100
+	}
+	tag := r.FormValue("tag")
+	if len(tag) == 0 {
+		return fmt.Errorf("missing `tag` query arg")
+	}
+	valuePrefix := r.FormValue("valuePrefix")
+	exprs := r.Form["expr"]
+	var tagValues []string
+	if len(exprs) == 0 {
+		// Fast path: there are no `expr` filters, so use netstorage.GetGraphiteTagValues.
+		// Escape special chars in tagPrefix as Graphite does.
+		// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228
+		filter := regexp.QuoteMeta(valuePrefix)
+		tagValues, err = netstorage.GetGraphiteTagValues(tag, filter, limit, deadline)
+		if err != nil {
+			return err
+		}
+	} else {
+		// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
+		sq, err := getSearchQueryForExprs(exprs)
+		if err != nil {
+			return err
+		}
+		mns, err := netstorage.SearchMetricNames(sq, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
+		}
+		m := make(map[string]struct{})
+		if tag == "name" {
+			tag = "__name__"
+		}
+		for _, mn := range mns {
+			tagValue := mn.GetTagValue(tag)
+			if len(tagValue) == 0 {
+				continue
+			}
+			m[string(tagValue)] = struct{}{}
+		}
+		if len(valuePrefix) > 0 {
+			for tagValue := range m {
+				if !strings.HasPrefix(tagValue, valuePrefix) {
+					delete(m, tagValue)
+				}
+			}
+		}
+		tagValues = make([]string, 0, len(m))
+		for tagValue := range m {
+			tagValues = append(tagValues, tagValue)
+		}
+		sort.Strings(tagValues)
+		if limit > 0 && limit < len(tagValues) {
+			tagValues = tagValues[:limit]
+		}
+	}
+
+	jsonp := r.FormValue("jsonp")
+	contentType := getContentType(jsonp)
+	w.Header().Set("Content-Type", contentType)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteTagsAutoCompleteResponse(bw, tagValues, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	tagsAutoCompleteValuesDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/autoComplete/values"}`)
+
+// TagsAutoCompleteTagsHandler implements /tags/autoComplete/tags endpoint from Graphite Tags API.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
+func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	limit, err := getInt(r, "limit")
+	if err != nil {
+		return err
+	}
+	if limit <= 0 {
+		// Use limit=100 by default. See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
+		limit = 100
+	}
+	tagPrefix := r.FormValue("tagPrefix")
+	exprs := r.Form["expr"]
+	var labels []string
+	if len(exprs) == 0 {
+		// Fast path: there are no `expr` filters, so use netstorage.GetGraphiteTags.
+
+		// Escape special chars in tagPrefix as Graphite does.
+		// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181
+		filter := regexp.QuoteMeta(tagPrefix)
+		labels, err = netstorage.GetGraphiteTags(filter, limit, deadline)
+		if err != nil {
+			return err
+		}
+	} else {
+		// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
+		sq, err := getSearchQueryForExprs(exprs)
+		if err != nil {
+			return err
+		}
+		mns, err := netstorage.SearchMetricNames(sq, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
+		}
+		m := make(map[string]struct{})
+		for _, mn := range mns {
+			m["name"] = struct{}{}
+			for _, tag := range mn.Tags {
+				m[string(tag.Key)] = struct{}{}
+			}
+		}
+		if len(tagPrefix) > 0 {
+			for label := range m {
+				if !strings.HasPrefix(label, tagPrefix) {
+					delete(m, label)
+				}
+			}
+		}
+		labels = make([]string, 0, len(m))
+		for label := range m {
+			labels = append(labels, label)
+		}
+		sort.Strings(labels)
+		if limit > 0 && limit < len(labels) {
+			labels = labels[:limit]
+		}
+	}
+
+	jsonp := r.FormValue("jsonp")
+	contentType := getContentType(jsonp)
+	w.Header().Set("Content-Type", contentType)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteTagsAutoCompleteResponse(bw, labels, jsonp)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	tagsAutoCompleteTagsDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/autoComplete/tags"}`)
+
+// TagsFindSeriesHandler implements /tags/findSeries endpoint from Graphite Tags API.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
+func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	limit, err := getInt(r, "limit")
+	if err != nil {
+		return err
+	}
+	exprs := r.Form["expr"]
+	if len(exprs) == 0 {
+		return fmt.Errorf("expecting at least one `expr` query arg")
+	}
+	sq, err := getSearchQueryForExprs(exprs)
+	if err != nil {
+		return err
+	}
+	mns, err := netstorage.SearchMetricNames(sq, deadline)
+	if err != nil {
+		return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
+	}
+	paths := getCanonicalPaths(mns)
+	if limit > 0 && limit < len(paths) {
+		paths = paths[:limit]
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteTagsFindSeriesResponse(bw, paths)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	tagsFindSeriesDuration.UpdateDuration(startTime)
+	return nil
+}
+
+func getCanonicalPaths(mns []storage.MetricName) []string {
+	paths := make([]string, 0, len(mns))
+	var b []byte
+	var tags []storage.Tag
+	for _, mn := range mns {
+		b = append(b[:0], mn.MetricGroup...)
+		tags = append(tags[:0], mn.Tags...)
+		sort.Slice(tags, func(i, j int) bool {
+			return string(tags[i].Key) < string(tags[j].Key)
+		})
+		for _, tag := range tags {
+			b = append(b, ';')
+			b = append(b, tag.Key...)
+			b = append(b, '=')
+			b = append(b, tag.Value...)
+		}
+		paths = append(paths, string(b))
+	}
+	sort.Strings(paths)
+	return paths
+}
+
+var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/findSeries"}`)
+
+// TagValuesHandler implements /tags/<tag_name> endpoint from Graphite Tags API.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
+func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	limit, err := getInt(r, "limit")
+	if err != nil {
+		return err
+	}
+	filter := r.FormValue("filter")
+	tagValues, err := netstorage.GetGraphiteTagValues(tagName, filter, limit, deadline)
+	if err != nil {
+		return err
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteTagValuesResponse(bw, tagName, tagValues)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	tagValuesDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var tagValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/<tag_name>"}`)
+
+// TagsHandler implements /tags endpoint from Graphite Tags API.
+//
+// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
+func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := searchutils.GetDeadlineForQuery(r, startTime)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %w", err)
+	}
+	limit, err := getInt(r, "limit")
+	if err != nil {
+		return err
+	}
+	filter := r.FormValue("filter")
+	labels, err := netstorage.GetGraphiteTags(filter, limit, deadline)
+	if err != nil {
+		return err
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteTagsResponse(bw, labels)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	tagsDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`)
+
+func getInt(r *http.Request, argName string) (int, error) {
+	argValue := r.FormValue(argName)
+	if len(argValue) == 0 {
+		return 0, nil
+	}
+	n, err := strconv.Atoi(argValue)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse %q=%q: %w", argName, argValue, err)
+	}
+	return n, nil
+}
+
+func getSearchQueryForExprs(exprs []string) (*storage.SearchQuery, error) {
+	tfs, err := exprsToTagFilters(exprs)
+	if err != nil {
+		return nil, err
+	}
+	ct := time.Now().UnixNano() / 1e6
+	sq := storage.NewSearchQuery(0, ct, [][]storage.TagFilter{tfs})
+	return sq, nil
+}
+
+func exprsToTagFilters(exprs []string) ([]storage.TagFilter, error) {
+	tfs := make([]storage.TagFilter, 0, len(exprs))
+	for _, expr := range exprs {
+		tf, err := parseFilterExpr(expr)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse `expr` query arg: %w", err)
+		}
+		tfs = append(tfs, *tf)
+	}
+	return tfs, nil
+}
+
+func parseFilterExpr(s string) (*storage.TagFilter, error) {
+	n := strings.Index(s, "=")
+	if n < 0 {
+		return nil, fmt.Errorf("missing tag value in filter expression %q", s)
+	}
+	tagName := s[:n]
+	tagValue := s[n+1:]
+	isNegative := false
+	if strings.HasSuffix(tagName, "!") {
+		isNegative = true
+		tagName = tagName[:len(tagName)-1]
+	}
+	if tagName == "name" {
+		tagName = ""
+	}
+	isRegexp := false
+	if strings.HasPrefix(tagValue, "~") {
+		isRegexp = true
+		tagValue = "^(?:" + tagValue[1:] + ").*"
+	}
+	return &storage.TagFilter{
+		Key:        []byte(tagName),
+		Value:      []byte(tagValue),
+		IsNegative: isNegative,
+		IsRegexp:   isRegexp,
+	}, nil
+}

app/vmselect/graphite/tags_autocomplete_response.qtpl

@@ -0,0 +1,16 @@
+{% stripspace %}
+
+TagsAutoCompleteResponse generates responses for /tags/autoComplete/{tags,values} handlers in Graphite Tags API.
+See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
+{% func TagsAutoCompleteResponse(ss []string, jsonp string) %}
+	{% if jsonp != "" %}{%s= jsonp %}({% endif %}
+	[
+		{% for i, s := range ss %}
+			{%q= s %}
+			{% if i+1 < len(ss) %},{% endif %}
+		{% endfor %}
+	]
+	{% if jsonp != "" %}){% endif %}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/tags_autocomplete_response.qtpl.go

@@ -0,0 +1,81 @@
+// Code generated by qtc from "tags_autocomplete_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// TagsAutoCompleteResponse generates responses for /tags/autoComplete/{tags,values} handlers in Graphite Tags API.See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
+
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
+package graphite
+
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5
+func StreamTagsAutoCompleteResponse(qw422016 *qt422016.Writer, ss []string, jsonp string) {
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
+	if jsonp != "" {
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
+		qw422016.N().S(jsonp)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
+		qw422016.N().S(`(`)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
+	}
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6
+	qw422016.N().S(`[`)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:8
+	for i, s := range ss {
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:9
+		qw422016.N().Q(s)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10
+		if i+1 < len(ss) {
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10
+		}
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:11
+	}
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:11
+	qw422016.N().S(`]`)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13
+	if jsonp != "" {
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13
+		qw422016.N().S(`)`)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13
+	}
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+}
+
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+func WriteTagsAutoCompleteResponse(qq422016 qtio422016.Writer, ss []string, jsonp string) {
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	StreamTagsAutoCompleteResponse(qw422016, ss, jsonp)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+}
+
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+func TagsAutoCompleteResponse(ss []string, jsonp string) string {
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	WriteTagsAutoCompleteResponse(qb422016, ss, jsonp)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+	return qs422016
+//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14
+}

app/vmselect/graphite/tags_find_series_response.qtpl

@@ -0,0 +1,12 @@
+{% stripspace %}
+
+{% func TagsFindSeriesResponse(paths []string) %}
+[
+	{% for i, path := range paths %}
+		{%q= path %}
+		{% if i+1 < len(paths) %},{% endif %}
+	{% endfor %}
+]
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/tags_find_series_response.qtpl.go

@@ -0,0 +1,65 @@
+// Code generated by qtc from "tags_find_series_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/graphite/tags_find_series_response.qtpl:3
+package graphite
+
+//line app/vmselect/graphite/tags_find_series_response.qtpl:3
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/tags_find_series_response.qtpl:3
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/tags_find_series_response.qtpl:3
+func StreamTagsFindSeriesResponse(qw422016 *qt422016.Writer, paths []string) {
+//line app/vmselect/graphite/tags_find_series_response.qtpl:3
+	qw422016.N().S(`[`)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:5
+	for i, path := range paths {
+//line app/vmselect/graphite/tags_find_series_response.qtpl:6
+		qw422016.N().Q(path)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:7
+		if i+1 < len(paths) {
+//line app/vmselect/graphite/tags_find_series_response.qtpl:7
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:7
+		}
+//line app/vmselect/graphite/tags_find_series_response.qtpl:8
+	}
+//line app/vmselect/graphite/tags_find_series_response.qtpl:8
+	qw422016.N().S(`]`)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+}
+
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+func WriteTagsFindSeriesResponse(qq422016 qtio422016.Writer, paths []string) {
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	StreamTagsFindSeriesResponse(qw422016, paths)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+}
+
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+func TagsFindSeriesResponse(paths []string) string {
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	WriteTagsFindSeriesResponse(qb422016, paths)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+	return qs422016
+//line app/vmselect/graphite/tags_find_series_response.qtpl:10
+}

app/vmselect/graphite/tags_response.qtpl

@@ -0,0 +1,16 @@
+{% stripspace %}
+
+Tags generates response for /tags handler
+See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
+{% func TagsResponse(tags []string) %}
+[
+	{% for i, tag := range tags %}
+		{
+			"tag":{%q= tag %}
+		}
+		{% if i+1 < len(tags) %},{% endif %}
+	{% endfor %}
+]
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/tags_response.qtpl.go

@@ -0,0 +1,71 @@
+// Code generated by qtc from "tags_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// Tags generates response for /tags handlerSee https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
+
+//line app/vmselect/graphite/tags_response.qtpl:5
+package graphite
+
+//line app/vmselect/graphite/tags_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/tags_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/tags_response.qtpl:5
+func StreamTagsResponse(qw422016 *qt422016.Writer, tags []string) {
+//line app/vmselect/graphite/tags_response.qtpl:5
+	qw422016.N().S(`[`)
+//line app/vmselect/graphite/tags_response.qtpl:7
+	for i, tag := range tags {
+//line app/vmselect/graphite/tags_response.qtpl:7
+		qw422016.N().S(`{"tag":`)
+//line app/vmselect/graphite/tags_response.qtpl:9
+		qw422016.N().Q(tag)
+//line app/vmselect/graphite/tags_response.qtpl:9
+		qw422016.N().S(`}`)
+//line app/vmselect/graphite/tags_response.qtpl:11
+		if i+1 < len(tags) {
+//line app/vmselect/graphite/tags_response.qtpl:11
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/tags_response.qtpl:11
+		}
+//line app/vmselect/graphite/tags_response.qtpl:12
+	}
+//line app/vmselect/graphite/tags_response.qtpl:12
+	qw422016.N().S(`]`)
+//line app/vmselect/graphite/tags_response.qtpl:14
+}
+
+//line app/vmselect/graphite/tags_response.qtpl:14
+func WriteTagsResponse(qq422016 qtio422016.Writer, tags []string) {
+//line app/vmselect/graphite/tags_response.qtpl:14
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/tags_response.qtpl:14
+	StreamTagsResponse(qw422016, tags)
+//line app/vmselect/graphite/tags_response.qtpl:14
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/tags_response.qtpl:14
+}
+
+//line app/vmselect/graphite/tags_response.qtpl:14
+func TagsResponse(tags []string) string {
+//line app/vmselect/graphite/tags_response.qtpl:14
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/tags_response.qtpl:14
+	WriteTagsResponse(qb422016, tags)
+//line app/vmselect/graphite/tags_response.qtpl:14
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/tags_response.qtpl:14
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/tags_response.qtpl:14
+	return qs422016
+//line app/vmselect/graphite/tags_response.qtpl:14
+}

app/vmselect/graphite/tags_tag_multi_series_response.qtpl

@@ -0,0 +1,14 @@
+{% stripspace %}
+
+TagsTagMultiSeriesResponse generates response for /tags/tagMultiSeries .
+See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
+{% func TagsTagMultiSeriesResponse(canonicalPaths []string, isJSONResponse bool) %}
+	{% if isJSONResponse %}[{% endif %}
+	{% for i, path := range canonicalPaths %}
+		{%q= path %}
+		{% if i+1 < len(canonicalPaths) %},{% endif %}
+	{% endfor %}
+	{% if isJSONResponse %}]{% endif %}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/graphite/tags_tag_multi_series_response.qtpl.go

@@ -0,0 +1,75 @@
+// Code generated by qtc from "tags_tag_multi_series_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// TagsTagMultiSeriesResponse generates response for /tags/tagMultiSeries .See https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb
+
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
+package graphite
+
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:5
+func StreamTagsTagMultiSeriesResponse(qw422016 *qt422016.Writer, canonicalPaths []string, isJSONResponse bool) {
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:6
+	if isJSONResponse {
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:6
+		qw422016.N().S(`[`)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:6
+	}
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:7
+	for i, path := range canonicalPaths {
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:8
+		qw422016.N().Q(path)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:9
+		if i+1 < len(canonicalPaths) {
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:9
+			qw422016.N().S(`,`)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:9
+		}
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:10
+	}
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:11
+	if isJSONResponse {
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:11
+		qw422016.N().S(`]`)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:11
+	}
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+}
+
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+func WriteTagsTagMultiSeriesResponse(qq422016 qtio422016.Writer, canonicalPaths []string, isJSONResponse bool) {
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	StreamTagsTagMultiSeriesResponse(qw422016, canonicalPaths, isJSONResponse)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+}
+
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+func TagsTagMultiSeriesResponse(canonicalPaths []string, isJSONResponse bool) string {
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	WriteTagsTagMultiSeriesResponse(qb422016, canonicalPaths, isJSONResponse)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	qs422016 := string(qb422016.B)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+	return qs422016
+//line app/vmselect/graphite/tags_tag_multi_series_response.qtpl:12
+}

app/vmselect/main.go

@@ -10,9 +10,12 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
@@ -20,10 +23,11 @@ import (
 )
 
 var (
-	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series")
+	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores. See also -search.maxQueueDuration")
-	maxQueueDuration  = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
+		"limit is reached; see also -search.maxQueryDuration")
 	resetCacheAuthKey = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
 )
 
@@ -43,6 +47,9 @@ func getDefaultMaxConcurrentRequests() int {
 
 // Init initializes vmselect
 func Init() {
+	tmpDirPath := *vmstorage.DataPath + "/tmp"
+	fs.RemoveDirContents(tmpDirPath)
+	netstorage.InitTmpBlocksDir(tmpDirPath)
 	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
 
 	concurrencyCh = make(chan struct{}, *maxConcurrentRequests)
@@ -77,7 +84,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
-		t := timerpool.Get(*maxQueueDuration)
+		d := searchutils.GetMaxQueryDuration(r)
+		if d > *maxQueueDuration {
+			d = *maxQueueDuration
+		}
+		t := timerpool.Get(d)
 		select {
 		case concurrencyCh <- struct{}{}:
 			timerpool.Put(t)
@@ -87,8 +98,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			concurrencyLimitTimeout.Inc()
 			err := &httpserver.ErrorWithStatusCode{
 				Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
-					"increase `-search.maxQueueDuration`, increase `-search.maxConcurrentRequests`, increase server capacity",
-					*maxConcurrentRequests, *maxQueueDuration),
+					"increase `-search.maxQueueDuration`; increase `-search.maxQueryDuration`; increase `-search.maxConcurrentRequests`; "+
+					"increase server capacity",
+					*maxConcurrentRequests, d),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
@@ -120,6 +132,16 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 	}
+	if strings.HasPrefix(path, "/tags/") && !isGraphiteTagsPath(path) {
+		tagName := r.URL.Path[len("/tags/"):]
+		graphiteTagValuesRequests.Inc()
+		if err := graphite.TagValuesHandler(startTime, tagName, w, r); err != nil {
+			graphiteTagValuesErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	}
 
 	switch path {
 	case "/api/v1/query":
@@ -196,6 +218,22 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
+	case "/api/v1/export/csv":
+		exportCSVRequests.Inc()
+		if err := prometheus.ExportCSVHandler(startTime, w, r); err != nil {
+			exportCSVErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/api/v1/export/native":
+		exportNativeRequests.Inc()
+		if err := prometheus.ExportNativeHandler(startTime, w, r); err != nil {
+			exportNativeErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
 	case "/federate":
 		federateRequests.Inc()
 		if err := prometheus.FederateHandler(startTime, w, r); err != nil {
@@ -231,22 +269,85 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
+	case "/tags/tagSeries":
+		graphiteTagsTagSeriesRequests.Inc()
+		if err := graphite.TagsTagSeriesHandler(startTime, w, r); err != nil {
+			graphiteTagsTagSeriesErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/tags/tagMultiSeries":
+		graphiteTagsTagMultiSeriesRequests.Inc()
+		if err := graphite.TagsTagMultiSeriesHandler(startTime, w, r); err != nil {
+			graphiteTagsTagMultiSeriesErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/tags":
+		graphiteTagsRequests.Inc()
+		if err := graphite.TagsHandler(startTime, w, r); err != nil {
+			graphiteTagsErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/tags/findSeries":
+		graphiteTagsFindSeriesRequests.Inc()
+		if err := graphite.TagsFindSeriesHandler(startTime, w, r); err != nil {
+			graphiteTagsFindSeriesErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/tags/autoComplete/tags":
+		graphiteTagsAutoCompleteTagsRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := graphite.TagsAutoCompleteTagsHandler(startTime, w, r); err != nil {
+			graphiteTagsAutoCompleteTagsErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/tags/autoComplete/values":
+		graphiteTagsAutoCompleteValuesRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := graphite.TagsAutoCompleteValuesHandler(startTime, w, r); err != nil {
+			graphiteTagsAutoCompleteValuesErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
+	case "/tags/delSeries":
+		graphiteTagsDelSeriesRequests.Inc()
+		authKey := r.FormValue("authKey")
+		if authKey != *deleteAuthKey {
+			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
+			return true
+		}
+		if err := graphite.TagsDelSeriesHandler(startTime, w, r); err != nil {
+			graphiteTagsDelSeriesErrors.Inc()
+			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		return true
 	case "/api/v1/rules":
 		// Return dumb placeholder
 		rulesRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
 		return true
 	case "/api/v1/alerts":
 		// Return dumb placehloder
 		alertsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
 		return true
 	case "/api/v1/metadata":
 		// Return dumb placeholder
 		metadataRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
@@ -268,10 +369,22 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
+func isGraphiteTagsPath(path string) bool {
+	switch path {
+	// See https://graphite.readthedocs.io/en/stable/tags.html for a list of Graphite Tags API paths.
+	// Do not include `/tags/<tag_name>` here, since this will fool the caller.
+	case "/tags/tagSeries", "/tags/tagMultiSeries", "/tags/findSeries",
+		"/tags/autoComplete/tags", "/tags/autoComplete/values", "/tags/delSeries":
+		return true
+	default:
+		return false
+	}
+}
+
 func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
 	logger.Warnf("error in %q: %s", r.RequestURI, err)
 
-	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	statusCode := http.StatusUnprocessableEntity
 	var esc *httpserver.ErrorWithStatusCode
 	if errors.As(err, &esc) {
@@ -314,6 +427,12 @@ var (
 	exportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/export"}`)
 	exportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/export"}`)
 
+	exportCSVRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/export/csv"}`)
+	exportCSVErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/export/csv"}`)
+
+	exportNativeRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/export/native"}`)
+	exportNativeErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/export/native"}`)
+
 	federateRequests = metrics.NewCounter(`vm_http_requests_total{path="/federate"}`)
 	federateErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/federate"}`)
 
@@ -326,6 +445,30 @@ var (
 	graphiteMetricsIndexRequests = metrics.NewCounter(`vm_http_requests_total{path="/metrics/index.json"}`)
 	graphiteMetricsIndexErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/metrics/index.json"}`)
 
+	graphiteTagsTagSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/tagSeries"}`)
+	graphiteTagsTagSeriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/tagSeries"}`)
+
+	graphiteTagsTagMultiSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/tagMultiSeries"}`)
+	graphiteTagsTagMultiSeriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/tagMultiSeries"}`)
+
+	graphiteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags"}`)
+	graphiteTagsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags"}`)
+
+	graphiteTagValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/<tag_name>"}`)
+	graphiteTagValuesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/<tag_name>"}`)
+
+	graphiteTagsFindSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/findSeries"}`)
+	graphiteTagsFindSeriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/findSeries"}`)
+
+	graphiteTagsAutoCompleteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/autoComplete/tags"}`)
+	graphiteTagsAutoCompleteTagsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/autoComplete/tags"}`)
+
+	graphiteTagsAutoCompleteValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/autoComplete/values"}`)
+	graphiteTagsAutoCompleteValuesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/autoComplete/values"}`)
+
+	graphiteTagsDelSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/delSeries"}`)
+	graphiteTagsDelSeriesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/delSeries"}`)
+
 	rulesRequests    = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
 	alertsRequests   = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
 	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)

app/vmselect/netstorage/netstorage.go

@@ -5,14 +5,15 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"regexp"
 	"runtime"
 	"sort"
 	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -57,6 +58,7 @@ type Results struct {
 
 	packedTimeseries []packedTimeseries
 	sr               *storage.Search
+	tbf              *tmpBlocksFile
 }
 
 // Len returns the number of results in rss.
@@ -72,15 +74,18 @@ func (rss *Results) Cancel() {
 func (rss *Results) mustClose() {
 	putStorageSearch(rss.sr)
 	rss.sr = nil
+	putTmpBlocksFile(rss.tbf)
+	rss.tbf = nil
 }
 
 var timeseriesWorkCh = make(chan *timeseriesWork, gomaxprocs*16)
 
 type timeseriesWork struct {
-	rss    *Results
-	pts    *packedTimeseries
-	f      func(rs *Result, workerID uint)
-	doneCh chan error
+	mustStop uint64
+	rss      *Results
+	pts      *packedTimeseries
+	f        func(rs *Result, workerID uint) error
+	doneCh   chan error
 
 	rowsProcessed int
 }
@@ -100,12 +105,19 @@ func timeseriesWorker(workerID uint) {
 			tsw.doneCh <- fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
 			continue
 		}
-		if err := tsw.pts.Unpack(&rs, rss.tr, rss.fetchData); err != nil {
+		if atomic.LoadUint64(&tsw.mustStop) != 0 {
+			tsw.doneCh <- nil
+			continue
+		}
+		if err := tsw.pts.Unpack(&rs, rss.tbf, rss.tr, rss.fetchData); err != nil {
 			tsw.doneCh <- fmt.Errorf("error during time series unpacking: %w", err)
 			continue
 		}
 		if len(rs.Timestamps) > 0 || !rss.fetchData {
-			tsw.f(&rs, workerID)
+			if err := tsw.f(&rs, workerID); err != nil {
+				tsw.doneCh <- err
+				continue
+			}
 		}
 		tsw.rowsProcessed = len(rs.Values)
 		tsw.doneCh <- nil
@@ -122,9 +134,10 @@ func timeseriesWorker(workerID uint) {
 //
 // f shouldn't hold references to rs after returning.
 // workerID is the id of the worker goroutine that calls f.
+// Data processing is immediately stopped if f returns non-nil error.
 //
 // rss becomes unusable after the call to RunParallel.
-func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
+func (rss *Results) RunParallel(f func(rs *Result, workerID uint) error) error {
 	defer rss.mustClose()
 
 	// Feed workers with work.
@@ -150,6 +163,10 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 			// Return just the first error, since other errors
 			// are likely duplicate the first error.
 			firstErr = err
+			// Notify all the the tsws that they shouldn't be executed.
+			for _, tsw := range tsws {
+				atomic.StoreUint64(&tsw.mustStop, 1)
+			}
 		}
 		rowsProcessedTotal += tsw.rowsProcessed
 	}
@@ -166,32 +183,32 @@ var gomaxprocs = runtime.GOMAXPROCS(-1)
 
 type packedTimeseries struct {
 	metricName string
-	brs        []storage.BlockRef
+	brs        []blockRef
 }
 
 var unpackWorkCh = make(chan *unpackWork, gomaxprocs*128)
 
 type unpackWorkItem struct {
-	br storage.BlockRef
+	br blockRef
 	tr storage.TimeRange
 }
 
 type unpackWork struct {
-	ws        []unpackWorkItem
-	fetchData bool
-	sbs       []*sortBlock
-	doneCh    chan error
+	tbf    *tmpBlocksFile
+	ws     []unpackWorkItem
+	sbs    []*sortBlock
+	doneCh chan error
 }
 
 func (upw *unpackWork) reset() {
+	upw.tbf = nil
 	ws := upw.ws
 	for i := range ws {
 		w := &ws[i]
-		w.br = storage.BlockRef{}
+		w.br = blockRef{}
 		w.tr = storage.TimeRange{}
 	}
 	upw.ws = upw.ws[:0]
-	upw.fetchData = false
 	sbs := upw.sbs
 	for i := range sbs {
 		sbs[i] = nil
@@ -202,10 +219,10 @@ func (upw *unpackWork) reset() {
 	}
 }
 
-func (upw *unpackWork) unpack() {
+func (upw *unpackWork) unpack(tmpBlock *storage.Block) {
 	for _, w := range upw.ws {
 		sb := getSortBlock()
-		if err := sb.unpackFrom(w.br, w.tr, upw.fetchData); err != nil {
+		if err := sb.unpackFrom(tmpBlock, upw.tbf, w.br, w.tr); err != nil {
 			putSortBlock(sb)
 			upw.doneCh <- fmt.Errorf("cannot unpack block: %w", err)
 			return
@@ -239,8 +256,9 @@ func init() {
 }
 
 func unpackWorker() {
+	var tmpBlock storage.Block
 	for upw := range unpackWorkCh {
-		upw.unpack()
+		upw.unpack(&tmpBlock)
 	}
 }
 
@@ -250,23 +268,27 @@ func unpackWorker() {
 var unpackBatchSize = 8 * runtime.GOMAXPROCS(-1)
 
 // Unpack unpacks pts to dst.
-func (pts *packedTimeseries) Unpack(dst *Result, tr storage.TimeRange, fetchData bool) error {
+func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.TimeRange, fetchData bool) error {
 	dst.reset()
-
 	if err := dst.MetricName.Unmarshal(bytesutil.ToUnsafeBytes(pts.metricName)); err != nil {
 		return fmt.Errorf("cannot unmarshal metricName %q: %w", pts.metricName, err)
 	}
+	if !fetchData {
+		// Do not spend resources on data reading and unpacking.
+		return nil
+	}
 
 	// Feed workers with work
-	upws := make([]*unpackWork, 0, 1+len(pts.brs)/unpackBatchSize)
+	brsLen := len(pts.brs)
+	upws := make([]*unpackWork, 0, 1+brsLen/unpackBatchSize)
 	upw := getUnpackWork()
-	upw.fetchData = fetchData
+	upw.tbf = tbf
 	for _, br := range pts.brs {
 		if len(upw.ws) >= unpackBatchSize {
 			unpackWorkCh <- upw
 			upws = append(upws, upw)
 			upw = getUnpackWork()
-			upw.fetchData = fetchData
+			upw.tbf = tbf
 		}
 		upw.ws = append(upw.ws, unpackWorkItem{
 			br: br,
@@ -278,7 +300,7 @@ func (pts *packedTimeseries) Unpack(dst *Result, tr storage.TimeRange, fetchData
 	pts.brs = pts.brs[:0]
 
 	// Wait until work is complete
-	sbs := make([]*sortBlock, 0, len(pts.brs))
+	sbs := make([]*sortBlock, 0, brsLen)
 	var firstErr error
 	for _, upw := range upws {
 		if err := <-upw.doneCh; err != nil && firstErr == nil {
@@ -372,52 +394,27 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
 var dedupsDuringSelect = metrics.NewCounter(`vm_deduplicated_samples_total{type="select"}`)
 
 type sortBlock struct {
-	// b is used as a temporary storage for unpacked rows before they
-	// go to Timestamps and Values.
-	b storage.Block
-
 	Timestamps []int64
 	Values     []float64
 	NextIdx    int
 }
 
 func (sb *sortBlock) reset() {
-	sb.b.Reset()
 	sb.Timestamps = sb.Timestamps[:0]
 	sb.Values = sb.Values[:0]
 	sb.NextIdx = 0
 }
 
-func (sb *sortBlock) unpackFrom(br storage.BlockRef, tr storage.TimeRange, fetchData bool) error {
-	br.MustReadBlock(&sb.b, fetchData)
-	if fetchData {
-		if err := sb.b.UnmarshalData(); err != nil {
-			return fmt.Errorf("cannot unmarshal block: %w", err)
-		}
+func (sb *sortBlock) unpackFrom(tmpBlock *storage.Block, tbf *tmpBlocksFile, br blockRef, tr storage.TimeRange) error {
+	tmpBlock.Reset()
+	brReal := tbf.MustReadBlockRefAt(br.partRef, br.addr)
+	brReal.MustReadBlock(tmpBlock, true)
+	if err := tmpBlock.UnmarshalData(); err != nil {
+		return fmt.Errorf("cannot unmarshal block: %w", err)
 	}
-	timestamps := sb.b.Timestamps()
-
-	// Skip timestamps smaller than tr.MinTimestamp.
-	i := 0
-	for i < len(timestamps) && timestamps[i] < tr.MinTimestamp {
-		i++
-	}
-
-	// Skip timestamps bigger than tr.MaxTimestamp.
-	j := len(timestamps)
-	for j > i && timestamps[j-1] > tr.MaxTimestamp {
-		j--
-	}
-	skippedRows := sb.b.RowsCount() - (j - i)
+	sb.Timestamps, sb.Values = tmpBlock.AppendRowsWithTimeRangeFilter(sb.Timestamps[:0], sb.Values[:0], tr)
+	skippedRows := tmpBlock.RowsCount() - len(sb.Timestamps)
 	metricRowsSkipped.Add(skippedRows)
-
-	// Copy the remaining values.
-	if i == j {
-		return nil
-	}
-	values := sb.b.Values()
-	sb.Timestamps = append(sb.Timestamps, timestamps[i:j]...)
-	sb.Values = decimal.AppendDecimalToFloat(sb.Values, values[i:j], sb.b.Scale())
 	return nil
 }
 
@@ -457,6 +454,55 @@ func DeleteSeries(sq *storage.SearchQuery) (int, error) {
 	return vmstorage.DeleteMetrics(tfss)
 }
 
+// GetLabelsOnTimeRange returns labels for the given tr until the given deadline.
+func GetLabelsOnTimeRange(tr storage.TimeRange, deadline searchutils.Deadline) ([]string, error) {
+	if deadline.Exceeded() {
+		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
+	}
+	labels, err := vmstorage.SearchTagKeysOnTimeRange(tr, *maxTagKeysPerSearch, deadline.Deadline())
+	if err != nil {
+		return nil, fmt.Errorf("error during labels search on time range: %w", err)
+	}
+	// Substitute "" with "__name__"
+	for i := range labels {
+		if labels[i] == "" {
+			labels[i] = "__name__"
+		}
+	}
+	// Sort labels like Prometheus does
+	sort.Strings(labels)
+	return labels, nil
+}
+
+// GetGraphiteTags returns Graphite tags until the given deadline.
+func GetGraphiteTags(filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
+	if deadline.Exceeded() {
+		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
+	}
+	labels, err := GetLabels(deadline)
+	if err != nil {
+		return nil, err
+	}
+	// Substitute "__name__" with "name" for Graphite compatibility
+	for i := range labels {
+		if labels[i] == "__name__" {
+			labels[i] = "name"
+			sort.Strings(labels)
+			break
+		}
+	}
+	if len(filter) > 0 {
+		labels, err = applyGraphiteRegexpFilter(filter, labels)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if limit > 0 && limit < len(labels) {
+		labels = labels[:limit]
+	}
+	return labels, nil
+}
+
 // GetLabels returns labels until the given deadline.
 func GetLabels(deadline searchutils.Deadline) ([]string, error) {
 	if deadline.Exceeded() {
@@ -466,20 +512,60 @@ func GetLabels(deadline searchutils.Deadline) ([]string, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error during labels search: %w", err)
 	}
-
 	// Substitute "" with "__name__"
 	for i := range labels {
 		if labels[i] == "" {
 			labels[i] = "__name__"
 		}
 	}
-
 	// Sort labels like Prometheus does
 	sort.Strings(labels)
-
 	return labels, nil
 }
 
+// GetLabelValuesOnTimeRange returns label values for the given labelName on the given tr
+// until the given deadline.
+func GetLabelValuesOnTimeRange(labelName string, tr storage.TimeRange, deadline searchutils.Deadline) ([]string, error) {
+	if deadline.Exceeded() {
+		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
+	}
+	if labelName == "__name__" {
+		labelName = ""
+	}
+	// Search for tag values
+	labelValues, err := vmstorage.SearchTagValuesOnTimeRange([]byte(labelName), tr, *maxTagValuesPerSearch, deadline.Deadline())
+	if err != nil {
+		return nil, fmt.Errorf("error during label values search on time range for labelName=%q: %w", labelName, err)
+	}
+	// Sort labelValues like Prometheus does
+	sort.Strings(labelValues)
+	return labelValues, nil
+}
+
+// GetGraphiteTagValues returns tag values for the given tagName until the given deadline.
+func GetGraphiteTagValues(tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
+	if deadline.Exceeded() {
+		return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
+	}
+	if tagName == "name" {
+		tagName = ""
+	}
+	tagValues, err := GetLabelValues(tagName, deadline)
+	if err != nil {
+		return nil, err
+	}
+	if len(filter) > 0 {
+		tagValues, err = applyGraphiteRegexpFilter(filter, tagValues)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if limit > 0 && limit < len(tagValues) {
+		tagValues = tagValues[:limit]
+	}
+	return tagValues, nil
+}
+
 // GetLabelValues returns label values for the given labelName
 // until the given deadline.
 func GetLabelValues(labelName string, deadline searchutils.Deadline) ([]string, error) {
@@ -489,16 +575,13 @@ func GetLabelValues(labelName string, deadline searchutils.Deadline) ([]string,
 	if labelName == "__name__" {
 		labelName = ""
 	}
-
 	// Search for tag values
 	labelValues, err := vmstorage.SearchTagValues([]byte(labelName), *maxTagValuesPerSearch, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during label values search for labelName=%q: %w", labelName, err)
 	}
-
 	// Sort labelValues like Prometheus does
 	sort.Strings(labelValues)
-
 	return labelValues, nil
 }
 
@@ -586,7 +669,142 @@ func putStorageSearch(sr *storage.Search) {
 
 var ssPool sync.Pool
 
-// ProcessSearchQuery performs sq on storage nodes until the given deadline.
+// ExportBlocks searches for time series matching sq and calls f for each found block.
+//
+// f is called in parallel from multiple goroutines.
+// Data processing is immediately stopped if f returns non-nil error.
+// It is the responsibility of f to call b.UnmarshalData before reading timestamps and values from the block.
+// It is the responsibility of f to filter blocks according to the given tr.
+func ExportBlocks(sq *storage.SearchQuery, deadline searchutils.Deadline, f func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error) error {
+	if deadline.Exceeded() {
+		return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
+	}
+	tfss, err := setupTfss(sq.TagFilterss)
+	if err != nil {
+		return err
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: sq.MinTimestamp,
+		MaxTimestamp: sq.MaxTimestamp,
+	}
+	if err := vmstorage.CheckTimeRange(tr); err != nil {
+		return err
+	}
+
+	vmstorage.WG.Add(1)
+	defer vmstorage.WG.Done()
+
+	sr := getStorageSearch()
+	defer putStorageSearch(sr)
+	sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
+
+	// Start workers that call f in parallel on available CPU cores.
+	gomaxprocs := runtime.GOMAXPROCS(-1)
+	workCh := make(chan *exportWork, gomaxprocs*8)
+	var (
+		errGlobal     error
+		errGlobalLock sync.Mutex
+		mustStop      uint32
+	)
+	var wg sync.WaitGroup
+	wg.Add(gomaxprocs)
+	for i := 0; i < gomaxprocs; i++ {
+		go func() {
+			defer wg.Done()
+			for xw := range workCh {
+				if err := f(&xw.mn, &xw.b, tr); err != nil {
+					errGlobalLock.Lock()
+					if errGlobal != nil {
+						errGlobal = err
+						atomic.StoreUint32(&mustStop, 1)
+					}
+					errGlobalLock.Unlock()
+				}
+				xw.reset()
+				exportWorkPool.Put(xw)
+			}
+		}()
+	}
+
+	// Feed workers with work
+	blocksRead := 0
+	for sr.NextMetricBlock() {
+		blocksRead++
+		if deadline.Exceeded() {
+			return fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
+		}
+		if atomic.LoadUint32(&mustStop) != 0 {
+			break
+		}
+		xw := exportWorkPool.Get().(*exportWork)
+		if err := xw.mn.Unmarshal(sr.MetricBlockRef.MetricName); err != nil {
+			return fmt.Errorf("cannot unmarshal metricName for block #%d: %w", blocksRead, err)
+		}
+		sr.MetricBlockRef.BlockRef.MustReadBlock(&xw.b, true)
+		workCh <- xw
+	}
+	close(workCh)
+
+	// Wait for workers to finish.
+	wg.Wait()
+
+	// Check errors.
+	err = sr.Error()
+	if err == nil {
+		err = errGlobal
+	}
+	if err != nil {
+		if errors.Is(err, storage.ErrDeadlineExceeded) {
+			return fmt.Errorf("timeout exceeded during the query: %s", deadline.String())
+		}
+		return fmt.Errorf("search error after reading %d data blocks: %w", blocksRead, err)
+	}
+	return nil
+}
+
+type exportWork struct {
+	mn storage.MetricName
+	b  storage.Block
+}
+
+func (xw *exportWork) reset() {
+	xw.mn.Reset()
+	xw.b.Reset()
+}
+
+var exportWorkPool = &sync.Pool{
+	New: func() interface{} {
+		return &exportWork{}
+	},
+}
+
+// SearchMetricNames returns all the metric names matching sq until the given deadline.
+func SearchMetricNames(sq *storage.SearchQuery, deadline searchutils.Deadline) ([]storage.MetricName, error) {
+	if deadline.Exceeded() {
+		return nil, fmt.Errorf("timeout exceeded before starting to search metric names: %s", deadline.String())
+	}
+
+	// Setup search.
+	tfss, err := setupTfss(sq.TagFilterss)
+	if err != nil {
+		return nil, err
+	}
+	tr := storage.TimeRange{
+		MinTimestamp: sq.MinTimestamp,
+		MaxTimestamp: sq.MaxTimestamp,
+	}
+	if err := vmstorage.CheckTimeRange(tr); err != nil {
+		return nil, err
+	}
+
+	mns, err := vmstorage.SearchMetricNames(tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
+	if err != nil {
+		return nil, fmt.Errorf("cannot find metric names: %w", err)
+	}
+	return mns, nil
+}
+
+// ProcessSearchQuery performs sq until the given deadline.
 //
 // Results.RunParallel or Results.Cancel must be called on the returned Results.
 func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline searchutils.Deadline) (*Results, error) {
@@ -612,18 +830,31 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 
 	sr := getStorageSearch()
 	maxSeriesCount := sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
-
-	m := make(map[string][]storage.BlockRef, maxSeriesCount)
+	m := make(map[string][]blockRef, maxSeriesCount)
 	orderedMetricNames := make([]string, 0, maxSeriesCount)
 	blocksRead := 0
+	tbf := getTmpBlocksFile()
+	var buf []byte
 	for sr.NextMetricBlock() {
 		blocksRead++
 		if deadline.Exceeded() {
+			putTmpBlocksFile(tbf)
+			putStorageSearch(sr)
 			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
+		buf = sr.MetricBlockRef.BlockRef.Marshal(buf[:0])
+		addr, err := tbf.WriteBlockRefData(buf)
+		if err != nil {
+			putTmpBlocksFile(tbf)
+			putStorageSearch(sr)
+			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
+		}
 		metricName := sr.MetricBlockRef.MetricName
 		brs := m[string(metricName)]
-		brs = append(brs, *sr.MetricBlockRef.BlockRef)
+		brs = append(brs, blockRef{
+			partRef: sr.MetricBlockRef.BlockRef.PartRef(),
+			addr:    addr,
+		})
 		if len(brs) > 1 {
 			// An optimization: do not allocate a string for already existing metricName key in m
 			m[string(metricName)] = brs
@@ -635,11 +866,18 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 		}
 	}
 	if err := sr.Error(); err != nil {
+		putTmpBlocksFile(tbf)
+		putStorageSearch(sr)
 		if errors.Is(err, storage.ErrDeadlineExceeded) {
 			return nil, fmt.Errorf("timeout exceeded during the query: %s", deadline.String())
 		}
 		return nil, fmt.Errorf("search error after reading %d data blocks: %w", blocksRead, err)
 	}
+	if err := tbf.Finalize(); err != nil {
+		putTmpBlocksFile(tbf)
+		putStorageSearch(sr)
+		return nil, fmt.Errorf("cannot finalize temporary file: %w", err)
+	}
 
 	var rss Results
 	rss.tr = tr
@@ -654,9 +892,15 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 	}
 	rss.packedTimeseries = pts
 	rss.sr = sr
+	rss.tbf = tbf
 	return &rss, nil
 }
 
+type blockRef struct {
+	partRef storage.PartRef
+	addr    tmpBlockAddr
+}
+
 func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error) {
 	tfss := make([]*storage.TagFilters, 0, len(tagFilterss))
 	for _, tagFilters := range tagFilterss {
@@ -672,3 +916,20 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
 	}
 	return tfss, nil
 }
+
+func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
+	// Anchor filter regexp to the beginning of the string as Graphite does.
+	// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
+	filter = "^(?:" + filter + ")"
+	re, err := regexp.Compile(filter)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err)
+	}
+	dst := ss[:0]
+	for _, s := range ss {
+		if re.MatchString(s) {
+			dst = append(dst, s)
+		}
+	}
+	return dst, nil
+}

app/vmselect/netstorage/tmp_blocks_file.go

@@ -0,0 +1,185 @@
+package netstorage
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// InitTmpBlocksDir initializes directory to store temporary search results.
+//
+// It stores data in system-defined temporary directory if tmpDirPath is empty.
+func InitTmpBlocksDir(tmpDirPath string) {
+	if len(tmpDirPath) == 0 {
+		tmpDirPath = os.TempDir()
+	}
+	tmpBlocksDir = tmpDirPath + "/searchResults"
+	fs.MustRemoveAll(tmpBlocksDir)
+	if err := fs.MkdirAllIfNotExist(tmpBlocksDir); err != nil {
+		logger.Panicf("FATAL: cannot create %q: %s", tmpBlocksDir, err)
+	}
+}
+
+var tmpBlocksDir string
+
+func maxInmemoryTmpBlocksFile() int {
+	mem := memory.Allowed()
+	maxLen := mem / 1024
+	if maxLen < 64*1024 {
+		return 64 * 1024
+	}
+	if maxLen > 4*1024*1024 {
+		return 4 * 1024 * 1024
+	}
+	return maxLen
+}
+
+var _ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
+	return float64(maxInmemoryTmpBlocksFile())
+})
+
+type tmpBlocksFile struct {
+	buf []byte
+
+	f *os.File
+	r *fs.ReaderAt
+
+	offset uint64
+}
+
+func getTmpBlocksFile() *tmpBlocksFile {
+	v := tmpBlocksFilePool.Get()
+	if v == nil {
+		return &tmpBlocksFile{
+			buf: make([]byte, 0, maxInmemoryTmpBlocksFile()),
+		}
+	}
+	return v.(*tmpBlocksFile)
+}
+
+func putTmpBlocksFile(tbf *tmpBlocksFile) {
+	tbf.MustClose()
+	tbf.buf = tbf.buf[:0]
+	tbf.f = nil
+	tbf.r = nil
+	tbf.offset = 0
+	tmpBlocksFilePool.Put(tbf)
+}
+
+var tmpBlocksFilePool sync.Pool
+
+type tmpBlockAddr struct {
+	offset uint64
+	size   int
+}
+
+func (addr tmpBlockAddr) String() string {
+	return fmt.Sprintf("offset %d, size %d", addr.offset, addr.size)
+}
+
+var (
+	tmpBlocksFilesCreated = metrics.NewCounter(`vm_tmp_blocks_files_created_total`)
+	_                     = metrics.NewGauge(`vm_tmp_blocks_files_directory_free_bytes`, func() float64 {
+		return float64(fs.MustGetFreeSpace(tmpBlocksDir))
+	})
+)
+
+// WriteBlockRefData writes br to tbf.
+//
+// It returns errors since the operation may fail on space shortage
+// and this must be handled.
+func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
+	var addr tmpBlockAddr
+	addr.offset = tbf.offset
+	addr.size = len(b)
+	tbf.offset += uint64(addr.size)
+	if len(tbf.buf)+len(b) <= cap(tbf.buf) {
+		// Fast path - the data fits tbf.buf
+		tbf.buf = append(tbf.buf, b...)
+		return addr, nil
+	}
+
+	// Slow path: flush the data from tbf.buf to file.
+	if tbf.f == nil {
+		f, err := ioutil.TempFile(tmpBlocksDir, "")
+		if err != nil {
+			return addr, err
+		}
+		tbf.f = f
+		tmpBlocksFilesCreated.Inc()
+	}
+	_, err := tbf.f.Write(tbf.buf)
+	tbf.buf = append(tbf.buf[:0], b...)
+	if err != nil {
+		return addr, fmt.Errorf("cannot write block to %q: %w", tbf.f.Name(), err)
+	}
+	return addr, nil
+}
+
+func (tbf *tmpBlocksFile) Finalize() error {
+	if tbf.f == nil {
+		return nil
+	}
+	fname := tbf.f.Name()
+	if _, err := tbf.f.Write(tbf.buf); err != nil {
+		return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
+	}
+	tbf.buf = tbf.buf[:0]
+	r := fs.MustOpenReaderAt(fname)
+	// Hint the OS that the file is read almost sequentiallly.
+	// This should reduce the number of disk seeks, which is important
+	// for HDDs.
+	r.MustFadviseSequentialRead(true)
+	tbf.r = r
+	return nil
+}
+
+func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBlockAddr) storage.BlockRef {
+	var buf []byte
+	if tbf.f == nil {
+		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]
+	} else {
+		bb := tmpBufPool.Get()
+		defer tmpBufPool.Put(bb)
+		bb.B = bytesutil.Resize(bb.B, addr.size)
+		tbf.r.MustReadAt(bb.B, int64(addr.offset))
+		buf = bb.B
+	}
+	var br storage.BlockRef
+	if err := br.Init(partRef, buf); err != nil {
+		logger.Panicf("FATAL: cannot initialize BlockRef: %s", err)
+	}
+	return br
+}
+
+var tmpBufPool bytesutil.ByteBufferPool
+
+func (tbf *tmpBlocksFile) MustClose() {
+	if tbf.f == nil {
+		return
+	}
+	if tbf.r != nil {
+		// tbf.r could be nil if Finalize wasn't called.
+		tbf.r.MustClose()
+	}
+	fname := tbf.f.Name()
+
+	// Remove the file at first, then close it.
+	// This way the OS shouldn't try to flush file contents to storage
+	// on close.
+	if err := os.Remove(fname); err != nil {
+		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
+	}
+	if err := tbf.f.Close(); err != nil {
+		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
+	}
+	tbf.f = nil
+}

app/vmselect/prometheus/export.qtpl

@@ -1,30 +1,102 @@
 {% import (
+	"bytes"
+	"strings"
+	"time"
+
 	"github.com/valyala/quicktemplate"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 ) %}
 
 {% stripspace %}
 
-{% func ExportPrometheusLine(rs *netstorage.Result) %}
-	{% if len(rs.Timestamps) == 0 %}{% return %}{% endif %}
+{% func ExportCSVLine(xb *exportBlock, fieldNames []string) %}
+	{% if len(xb.timestamps) == 0 || len(fieldNames) == 0 %}{% return %}{% endif %}
+	{% for i, timestamp := range xb.timestamps %}
+		{% code value := xb.values[i] %}
+		{%= exportCSVField(xb.mn, fieldNames[0], timestamp, value) %}
+		{% for _, fieldName := range fieldNames[1:] %}
+			,
+			{%= exportCSVField(xb.mn, fieldName, timestamp, value) %}
+		{% endfor %}
+		{% newline %}
+	{% endfor %}
+{% endfunc %}
+
+{% func exportCSVField(mn *storage.MetricName, fieldName string, timestamp int64, value float64) %}
+	{% if fieldName == "__value__" %}
+		{%f= value %}
+		{% return %}
+	{% endif %}
+	{% if fieldName == "__timestamp__" %}
+		{%dl timestamp %}
+		{% return %}
+	{% endif %}
+	{% if strings.HasPrefix(fieldName, "__timestamp__:") %}
+		{% code timeFormat := fieldName[len("__timestamp__:"):] %}
+		{% switch timeFormat %}
+		{% case "unix_s" %}
+			{%dl= timestamp/1000 %}
+		{% case "unix_ms" %}
+			{%dl= timestamp %}
+		{% case "unix_ns" %}
+			{%dl= timestamp*1e6 %}
+		{% case "rfc3339" %}
+			{% code
+				bb := quicktemplate.AcquireByteBuffer()
+				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], time.RFC3339)
+			%}
+			{%z= bb.B %}
+			{% code
+				quicktemplate.ReleaseByteBuffer(bb)
+			%}
+		{% default %}
+			{% if strings.HasPrefix(timeFormat, "custom:") %}
+				{% code
+					layout := timeFormat[len("custom:"):]
+					bb := quicktemplate.AcquireByteBuffer()
+					bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], layout)
+				%}
+				{% if bytes.ContainsAny(bb.B, `"`+",\n") %}
+					{%qz bb.B %}
+				{% else %}
+					{%z= bb.B %}
+				{% endif %}
+				{% code
+					quicktemplate.ReleaseByteBuffer(bb)
+				%}
+			{% else %}
+				Unsupported timeFormat={%s= timeFormat %}
+			{% endif %}
+		{% endswitch %}
+		{% return %}
+	{% endif %}
+	{% code v := mn.GetTagValue(fieldName) %}
+	{% if bytes.ContainsAny(v, `"`+",\n") %}
+		{%qz= v %}
+	{% else %}
+		{%z= v %}
+	{% endif %}
+{% endfunc %}
+
+{% func ExportPrometheusLine(xb *exportBlock) %}
+	{% if len(xb.timestamps) == 0 %}{% return %}{% endif %}
 	{% code bb := quicktemplate.AcquireByteBuffer() %}
-	{% code writeprometheusMetricName(bb, &rs.MetricName) %}
-	{% for i, ts := range rs.Timestamps %}
+	{% code writeprometheusMetricName(bb, xb.mn) %}
+	{% for i, ts := range xb.timestamps %}
 		{%z= bb.B %}{% space %}
-		{%f= rs.Values[i] %}{% space %}
+		{%f= xb.values[i] %}{% space %}
 		{%dl= ts %}{% newline %}
 	{% endfor %}
 	{% code quicktemplate.ReleaseByteBuffer(bb) %}
 {% endfunc %}
 
-{% func ExportJSONLine(rs *netstorage.Result) %}
-	{% if len(rs.Timestamps) == 0 %}{% return %}{% endif %}
+{% func ExportJSONLine(xb *exportBlock) %}
+	{% if len(xb.timestamps) == 0 %}{% return %}{% endif %}
 	{
-		"metric":{%= metricNameObject(&rs.MetricName) %},
+		"metric":{%= metricNameObject(xb.mn) %},
 		"values":[
-			{% if len(rs.Values) > 0 %}
-				{% code values := rs.Values %}
+			{% if len(xb.values) > 0 %}
+				{% code values := xb.values %}
 				{%f= values[0] %}
 				{% code values = values[1:] %}
 				{% for _, v := range values %}
@@ -33,8 +105,8 @@
 			{% endif %}
 		],
 		"timestamps":[
-			{% if len(rs.Timestamps) > 0 %}
-				{% code timestamps := rs.Timestamps %}
+			{% if len(xb.timestamps) > 0 %}
+				{% code timestamps := xb.timestamps %}
 				{%dl= timestamps[0] %}
 				{% code timestamps = timestamps[1:] %}
 				{% for _, ts := range timestamps %}
@@ -45,10 +117,10 @@
 	}{% newline %}
 {% endfunc %}
 
-{% func ExportPromAPILine(rs *netstorage.Result) %}
+{% func ExportPromAPILine(xb *exportBlock) %}
 {
-	"metric": {%= metricNameObject(&rs.MetricName) %},
-	"values": {%= valuesWithTimestamps(rs.Values, rs.Timestamps) %}
+	"metric": {%= metricNameObject(xb.mn) %},
+	"values": {%= valuesWithTimestamps(xb.values, xb.timestamps) %}
 }
 {% endfunc %}
 

app/vmselect/prometheus/export.qtpl.go

@@ -6,380 +6,566 @@ package prometheus
 
 //line app/vmselect/prometheus/export.qtpl:1
 import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"bytes"
+	"strings"
+	"time"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/export.qtpl:9
+//line app/vmselect/prometheus/export.qtpl:12
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/export.qtpl:9
+//line app/vmselect/prometheus/export.qtpl:12
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/export.qtpl:9
-func StreamExportPrometheusLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:10
-	if len(rs.Timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:10
-		return
-//line app/vmselect/prometheus/export.qtpl:10
-	}
-//line app/vmselect/prometheus/export.qtpl:11
-	bb := quicktemplate.AcquireByteBuffer()
-
 //line app/vmselect/prometheus/export.qtpl:12
-	writeprometheusMetricName(bb, &rs.MetricName)
-
+func StreamExportCSVLine(qw422016 *qt422016.Writer, xb *exportBlock, fieldNames []string) {
 //line app/vmselect/prometheus/export.qtpl:13
-	for i, ts := range rs.Timestamps {
+	if len(xb.timestamps) == 0 || len(fieldNames) == 0 {
+//line app/vmselect/prometheus/export.qtpl:13
+		return
+//line app/vmselect/prometheus/export.qtpl:13
+	}
 //line app/vmselect/prometheus/export.qtpl:14
-		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:14
-		qw422016.N().S(` `)
+	for i, timestamp := range xb.timestamps {
 //line app/vmselect/prometheus/export.qtpl:15
-		qw422016.N().F(rs.Values[i])
-//line app/vmselect/prometheus/export.qtpl:15
-		qw422016.N().S(` `)
-//line app/vmselect/prometheus/export.qtpl:16
-		qw422016.N().DL(ts)
+		value := xb.values[i]
+
 //line app/vmselect/prometheus/export.qtpl:16
+		streamexportCSVField(qw422016, xb.mn, fieldNames[0], timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:17
+		for _, fieldName := range fieldNames[1:] {
+//line app/vmselect/prometheus/export.qtpl:17
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:19
+			streamexportCSVField(qw422016, xb.mn, fieldName, timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:20
+		}
+//line app/vmselect/prometheus/export.qtpl:21
 		qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/export.qtpl:17
+//line app/vmselect/prometheus/export.qtpl:22
 	}
-//line app/vmselect/prometheus/export.qtpl:18
-	quicktemplate.ReleaseByteBuffer(bb)
-
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 }
 
-//line app/vmselect/prometheus/export.qtpl:19
-func WriteExportPrometheusLine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+func WriteExportCSVLine(qq422016 qtio422016.Writer, xb *exportBlock, fieldNames []string) {
+//line app/vmselect/prometheus/export.qtpl:23
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:19
-	StreamExportPrometheusLine(qw422016, rs)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+	StreamExportCSVLine(qw422016, xb, fieldNames)
+//line app/vmselect/prometheus/export.qtpl:23
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 }
 
-//line app/vmselect/prometheus/export.qtpl:19
-func ExportPrometheusLine(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+func ExportCSVLine(xb *exportBlock, fieldNames []string) string {
+//line app/vmselect/prometheus/export.qtpl:23
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:19
-	WriteExportPrometheusLine(qb422016, rs)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
+	WriteExportCSVLine(qb422016, xb, fieldNames)
+//line app/vmselect/prometheus/export.qtpl:23
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:19
+//line app/vmselect/prometheus/export.qtpl:23
 }
 
-//line app/vmselect/prometheus/export.qtpl:21
-func StreamExportJSONLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:22
-	if len(rs.Timestamps) == 0 {
-//line app/vmselect/prometheus/export.qtpl:22
-		return
-//line app/vmselect/prometheus/export.qtpl:22
-	}
-//line app/vmselect/prometheus/export.qtpl:22
-	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:24
-	streammetricNameObject(qw422016, &rs.MetricName)
-//line app/vmselect/prometheus/export.qtpl:24
-	qw422016.N().S(`,"values":[`)
+//line app/vmselect/prometheus/export.qtpl:25
+func streamexportCSVField(qw422016 *qt422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
 //line app/vmselect/prometheus/export.qtpl:26
-	if len(rs.Values) > 0 {
+	if fieldName == "__value__" {
 //line app/vmselect/prometheus/export.qtpl:27
-		values := rs.Values
-
+		qw422016.N().F(value)
 //line app/vmselect/prometheus/export.qtpl:28
-		qw422016.N().F(values[0])
+		return
 //line app/vmselect/prometheus/export.qtpl:29
-		values = values[1:]
-
+	}
 //line app/vmselect/prometheus/export.qtpl:30
-		for _, v := range values {
-//line app/vmselect/prometheus/export.qtpl:30
-			qw422016.N().S(`,`)
+	if fieldName == "__timestamp__" {
 //line app/vmselect/prometheus/export.qtpl:31
-			qw422016.N().F(v)
+		qw422016.N().DL(timestamp)
 //line app/vmselect/prometheus/export.qtpl:32
-		}
+		return
 //line app/vmselect/prometheus/export.qtpl:33
 	}
-//line app/vmselect/prometheus/export.qtpl:33
-	qw422016.N().S(`],"timestamps":[`)
+//line app/vmselect/prometheus/export.qtpl:34
+	if strings.HasPrefix(fieldName, "__timestamp__:") {
+//line app/vmselect/prometheus/export.qtpl:35
+		timeFormat := fieldName[len("__timestamp__:"):]
+
 //line app/vmselect/prometheus/export.qtpl:36
-	if len(rs.Timestamps) > 0 {
+		switch timeFormat {
 //line app/vmselect/prometheus/export.qtpl:37
-		timestamps := rs.Timestamps
-
+		case "unix_s":
 //line app/vmselect/prometheus/export.qtpl:38
-		qw422016.N().DL(timestamps[0])
+			qw422016.N().DL(timestamp / 1000)
 //line app/vmselect/prometheus/export.qtpl:39
-		timestamps = timestamps[1:]
-
+		case "unix_ms":
 //line app/vmselect/prometheus/export.qtpl:40
-		for _, ts := range timestamps {
-//line app/vmselect/prometheus/export.qtpl:40
-			qw422016.N().S(`,`)
+			qw422016.N().DL(timestamp)
 //line app/vmselect/prometheus/export.qtpl:41
-			qw422016.N().DL(ts)
+		case "unix_ns":
 //line app/vmselect/prometheus/export.qtpl:42
-		}
+			qw422016.N().DL(timestamp * 1e6)
 //line app/vmselect/prometheus/export.qtpl:43
-	}
-//line app/vmselect/prometheus/export.qtpl:43
-	qw422016.N().S(`]}`)
+		case "rfc3339":
 //line app/vmselect/prometheus/export.qtpl:45
-	qw422016.N().S(`
-`)
-//line app/vmselect/prometheus/export.qtpl:46
-}
-
-//line app/vmselect/prometheus/export.qtpl:46
-func WriteExportJSONLine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:46
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:46
-	StreamExportJSONLine(qw422016, rs)
-//line app/vmselect/prometheus/export.qtpl:46
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:46
-}
-
-//line app/vmselect/prometheus/export.qtpl:46
-func ExportJSONLine(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/export.qtpl:46
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:46
-	WriteExportJSONLine(qb422016, rs)
-//line app/vmselect/prometheus/export.qtpl:46
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:46
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:46
-	return qs422016
-//line app/vmselect/prometheus/export.qtpl:46
-}
+			bb := quicktemplate.AcquireByteBuffer()
+			bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], time.RFC3339)
 
 //line app/vmselect/prometheus/export.qtpl:48
-func StreamExportPromAPILine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:48
-	qw422016.N().S(`{"metric":`)
-//line app/vmselect/prometheus/export.qtpl:50
-	streammetricNameObject(qw422016, &rs.MetricName)
-//line app/vmselect/prometheus/export.qtpl:50
-	qw422016.N().S(`,"values":`)
-//line app/vmselect/prometheus/export.qtpl:51
-	streamvaluesWithTimestamps(qw422016, rs.Values, rs.Timestamps)
-//line app/vmselect/prometheus/export.qtpl:51
-	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/export.qtpl:53
-}
-
-//line app/vmselect/prometheus/export.qtpl:53
-func WriteExportPromAPILine(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/export.qtpl:53
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:53
-	StreamExportPromAPILine(qw422016, rs)
-//line app/vmselect/prometheus/export.qtpl:53
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:53
-}
-
-//line app/vmselect/prometheus/export.qtpl:53
-func ExportPromAPILine(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/export.qtpl:53
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:53
-	WriteExportPromAPILine(qb422016, rs)
-//line app/vmselect/prometheus/export.qtpl:53
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:53
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:53
-	return qs422016
-//line app/vmselect/prometheus/export.qtpl:53
-}
-
-//line app/vmselect/prometheus/export.qtpl:55
-func StreamExportPromAPIResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:55
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vmselect/prometheus/export.qtpl:61
-	bb, ok := <-resultsCh
-
-//line app/vmselect/prometheus/export.qtpl:62
-	if ok {
-//line app/vmselect/prometheus/export.qtpl:63
-		qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:64
-		quicktemplate.ReleaseByteBuffer(bb)
-
-//line app/vmselect/prometheus/export.qtpl:65
-		for bb := range resultsCh {
-//line app/vmselect/prometheus/export.qtpl:65
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/export.qtpl:66
 			qw422016.N().Z(bb.B)
-//line app/vmselect/prometheus/export.qtpl:67
+//line app/vmselect/prometheus/export.qtpl:50
 			quicktemplate.ReleaseByteBuffer(bb)
 
+//line app/vmselect/prometheus/export.qtpl:52
+		default:
+//line app/vmselect/prometheus/export.qtpl:53
+			if strings.HasPrefix(timeFormat, "custom:") {
+//line app/vmselect/prometheus/export.qtpl:55
+				layout := timeFormat[len("custom:"):]
+				bb := quicktemplate.AcquireByteBuffer()
+				bb.B = time.Unix(timestamp/1000, (timestamp%1000)*1e6).AppendFormat(bb.B[:0], layout)
+
+//line app/vmselect/prometheus/export.qtpl:59
+				if bytes.ContainsAny(bb.B, `"`+",\n") {
+//line app/vmselect/prometheus/export.qtpl:60
+					qw422016.E().QZ(bb.B)
+//line app/vmselect/prometheus/export.qtpl:61
+				} else {
+//line app/vmselect/prometheus/export.qtpl:62
+					qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:63
+				}
+//line app/vmselect/prometheus/export.qtpl:65
+				quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:67
+			} else {
+//line app/vmselect/prometheus/export.qtpl:67
+				qw422016.N().S(`Unsupported timeFormat=`)
 //line app/vmselect/prometheus/export.qtpl:68
+				qw422016.N().S(timeFormat)
+//line app/vmselect/prometheus/export.qtpl:69
+			}
+//line app/vmselect/prometheus/export.qtpl:70
 		}
-//line app/vmselect/prometheus/export.qtpl:69
+//line app/vmselect/prometheus/export.qtpl:71
+		return
+//line app/vmselect/prometheus/export.qtpl:72
 	}
-//line app/vmselect/prometheus/export.qtpl:69
-	qw422016.N().S(`]}}`)
 //line app/vmselect/prometheus/export.qtpl:73
-}
-
-//line app/vmselect/prometheus/export.qtpl:73
-func WriteExportPromAPIResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:73
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:73
-	StreamExportPromAPIResponse(qw422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:73
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:73
-}
-
-//line app/vmselect/prometheus/export.qtpl:73
-func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
-//line app/vmselect/prometheus/export.qtpl:73
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:73
-	WriteExportPromAPIResponse(qb422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:73
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:73
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:73
-	return qs422016
-//line app/vmselect/prometheus/export.qtpl:73
-}
+	v := mn.GetTagValue(fieldName)
 
+//line app/vmselect/prometheus/export.qtpl:74
+	if bytes.ContainsAny(v, `"`+",\n") {
 //line app/vmselect/prometheus/export.qtpl:75
-func StreamExportStdResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+		qw422016.N().QZ(v)
 //line app/vmselect/prometheus/export.qtpl:76
-	for bb := range resultsCh {
+	} else {
 //line app/vmselect/prometheus/export.qtpl:77
-		qw422016.N().Z(bb.B)
+		qw422016.N().Z(v)
 //line app/vmselect/prometheus/export.qtpl:78
-		quicktemplate.ReleaseByteBuffer(bb)
+	}
+//line app/vmselect/prometheus/export.qtpl:79
+}
 
 //line app/vmselect/prometheus/export.qtpl:79
-	}
-//line app/vmselect/prometheus/export.qtpl:80
-}
-
-//line app/vmselect/prometheus/export.qtpl:80
-func WriteExportStdResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
-//line app/vmselect/prometheus/export.qtpl:80
+func writeexportCSVField(qq422016 qtio422016.Writer, mn *storage.MetricName, fieldName string, timestamp int64, value float64) {
+//line app/vmselect/prometheus/export.qtpl:79
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:80
-	StreamExportStdResponse(qw422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
+	streamexportCSVField(qw422016, mn, fieldName, timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:79
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 }
 
-//line app/vmselect/prometheus/export.qtpl:80
-func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
+func exportCSVField(mn *storage.MetricName, fieldName string, timestamp int64, value float64) string {
+//line app/vmselect/prometheus/export.qtpl:79
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:80
-	WriteExportStdResponse(qb422016, resultsCh)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
+	writeexportCSVField(qb422016, mn, fieldName, timestamp, value)
+//line app/vmselect/prometheus/export.qtpl:79
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:80
+//line app/vmselect/prometheus/export.qtpl:79
 }
 
+//line app/vmselect/prometheus/export.qtpl:81
+func StreamExportPrometheusLine(qw422016 *qt422016.Writer, xb *exportBlock) {
 //line app/vmselect/prometheus/export.qtpl:82
-func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+	if len(xb.timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:82
+		return
+//line app/vmselect/prometheus/export.qtpl:82
+	}
 //line app/vmselect/prometheus/export.qtpl:83
-	qw422016.N().Z(mn.MetricGroup)
+	bb := quicktemplate.AcquireByteBuffer()
+
 //line app/vmselect/prometheus/export.qtpl:84
-	if len(mn.Tags) > 0 {
-//line app/vmselect/prometheus/export.qtpl:84
-		qw422016.N().S(`{`)
+	writeprometheusMetricName(bb, xb.mn)
+
+//line app/vmselect/prometheus/export.qtpl:85
+	for i, ts := range xb.timestamps {
 //line app/vmselect/prometheus/export.qtpl:86
-		tags := mn.Tags
-
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:86
+		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:87
-		qw422016.N().Z(tags[0].Key)
+		qw422016.N().F(xb.values[i])
 //line app/vmselect/prometheus/export.qtpl:87
-		qw422016.N().S(`=`)
-//line app/vmselect/prometheus/export.qtpl:87
-		qw422016.N().QZ(tags[0].Value)
+		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:88
-		tags = tags[1:]
-
+		qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:88
+		qw422016.N().S(`
+`)
 //line app/vmselect/prometheus/export.qtpl:89
-		for i := range tags {
+	}
 //line app/vmselect/prometheus/export.qtpl:90
-			tag := &tags[i]
+	quicktemplate.ReleaseByteBuffer(bb)
 
-//line app/vmselect/prometheus/export.qtpl:90
-			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:91
-			qw422016.N().Z(tag.Key)
+}
+
 //line app/vmselect/prometheus/export.qtpl:91
-			qw422016.N().S(`=`)
+func WriteExportPrometheusLine(qq422016 qtio422016.Writer, xb *exportBlock) {
 //line app/vmselect/prometheus/export.qtpl:91
-			qw422016.N().QZ(tag.Value)
-//line app/vmselect/prometheus/export.qtpl:92
-		}
-//line app/vmselect/prometheus/export.qtpl:92
-		qw422016.N().S(`}`)
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:91
+	StreamExportPrometheusLine(qw422016, xb)
+//line app/vmselect/prometheus/export.qtpl:91
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:91
+}
+
+//line app/vmselect/prometheus/export.qtpl:91
+func ExportPrometheusLine(xb *exportBlock) string {
+//line app/vmselect/prometheus/export.qtpl:91
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:91
+	WriteExportPrometheusLine(qb422016, xb)
+//line app/vmselect/prometheus/export.qtpl:91
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:91
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:91
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:91
+}
+
+//line app/vmselect/prometheus/export.qtpl:93
+func StreamExportJSONLine(qw422016 *qt422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:94
+	if len(xb.timestamps) == 0 {
+//line app/vmselect/prometheus/export.qtpl:94
+		return
 //line app/vmselect/prometheus/export.qtpl:94
 	}
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:94
+	qw422016.N().S(`{"metric":`)
+//line app/vmselect/prometheus/export.qtpl:96
+	streammetricNameObject(qw422016, xb.mn)
+//line app/vmselect/prometheus/export.qtpl:96
+	qw422016.N().S(`,"values":[`)
+//line app/vmselect/prometheus/export.qtpl:98
+	if len(xb.values) > 0 {
+//line app/vmselect/prometheus/export.qtpl:99
+		values := xb.values
+
+//line app/vmselect/prometheus/export.qtpl:100
+		qw422016.N().F(values[0])
+//line app/vmselect/prometheus/export.qtpl:101
+		values = values[1:]
+
+//line app/vmselect/prometheus/export.qtpl:102
+		for _, v := range values {
+//line app/vmselect/prometheus/export.qtpl:102
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:103
+			qw422016.N().F(v)
+//line app/vmselect/prometheus/export.qtpl:104
+		}
+//line app/vmselect/prometheus/export.qtpl:105
+	}
+//line app/vmselect/prometheus/export.qtpl:105
+	qw422016.N().S(`],"timestamps":[`)
+//line app/vmselect/prometheus/export.qtpl:108
+	if len(xb.timestamps) > 0 {
+//line app/vmselect/prometheus/export.qtpl:109
+		timestamps := xb.timestamps
+
+//line app/vmselect/prometheus/export.qtpl:110
+		qw422016.N().DL(timestamps[0])
+//line app/vmselect/prometheus/export.qtpl:111
+		timestamps = timestamps[1:]
+
+//line app/vmselect/prometheus/export.qtpl:112
+		for _, ts := range timestamps {
+//line app/vmselect/prometheus/export.qtpl:112
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:113
+			qw422016.N().DL(ts)
+//line app/vmselect/prometheus/export.qtpl:114
+		}
+//line app/vmselect/prometheus/export.qtpl:115
+	}
+//line app/vmselect/prometheus/export.qtpl:115
+	qw422016.N().S(`]}`)
+//line app/vmselect/prometheus/export.qtpl:117
+	qw422016.N().S(`
+`)
+//line app/vmselect/prometheus/export.qtpl:118
 }
 
-//line app/vmselect/prometheus/export.qtpl:95
-func writeprometheusMetricName(qq422016 qtio422016.Writer, mn *storage.MetricName) {
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+func WriteExportJSONLine(qq422016 qtio422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:118
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/export.qtpl:95
-	streamprometheusMetricName(qw422016, mn)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+	StreamExportJSONLine(qw422016, xb)
+//line app/vmselect/prometheus/export.qtpl:118
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
 }
 
-//line app/vmselect/prometheus/export.qtpl:95
-func prometheusMetricName(mn *storage.MetricName) string {
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+func ExportJSONLine(xb *exportBlock) string {
+//line app/vmselect/prometheus/export.qtpl:118
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/export.qtpl:95
-	writeprometheusMetricName(qb422016, mn)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+	WriteExportJSONLine(qb422016, xb)
+//line app/vmselect/prometheus/export.qtpl:118
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
 	return qs422016
-//line app/vmselect/prometheus/export.qtpl:95
+//line app/vmselect/prometheus/export.qtpl:118
+}
+
+//line app/vmselect/prometheus/export.qtpl:120
+func StreamExportPromAPILine(qw422016 *qt422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:120
+	qw422016.N().S(`{"metric":`)
+//line app/vmselect/prometheus/export.qtpl:122
+	streammetricNameObject(qw422016, xb.mn)
+//line app/vmselect/prometheus/export.qtpl:122
+	qw422016.N().S(`,"values":`)
+//line app/vmselect/prometheus/export.qtpl:123
+	streamvaluesWithTimestamps(qw422016, xb.values, xb.timestamps)
+//line app/vmselect/prometheus/export.qtpl:123
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/export.qtpl:125
+}
+
+//line app/vmselect/prometheus/export.qtpl:125
+func WriteExportPromAPILine(qq422016 qtio422016.Writer, xb *exportBlock) {
+//line app/vmselect/prometheus/export.qtpl:125
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:125
+	StreamExportPromAPILine(qw422016, xb)
+//line app/vmselect/prometheus/export.qtpl:125
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:125
+}
+
+//line app/vmselect/prometheus/export.qtpl:125
+func ExportPromAPILine(xb *exportBlock) string {
+//line app/vmselect/prometheus/export.qtpl:125
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:125
+	WriteExportPromAPILine(qb422016, xb)
+//line app/vmselect/prometheus/export.qtpl:125
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:125
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:125
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:125
+}
+
+//line app/vmselect/prometheus/export.qtpl:127
+func StreamExportPromAPIResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:127
+	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vmselect/prometheus/export.qtpl:133
+	bb, ok := <-resultsCh
+
+//line app/vmselect/prometheus/export.qtpl:134
+	if ok {
+//line app/vmselect/prometheus/export.qtpl:135
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:136
+		quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:137
+		for bb := range resultsCh {
+//line app/vmselect/prometheus/export.qtpl:137
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:138
+			qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:139
+			quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:140
+		}
+//line app/vmselect/prometheus/export.qtpl:141
+	}
+//line app/vmselect/prometheus/export.qtpl:141
+	qw422016.N().S(`]}}`)
+//line app/vmselect/prometheus/export.qtpl:145
+}
+
+//line app/vmselect/prometheus/export.qtpl:145
+func WriteExportPromAPIResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:145
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:145
+	StreamExportPromAPIResponse(qw422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:145
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:145
+}
+
+//line app/vmselect/prometheus/export.qtpl:145
+func ExportPromAPIResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
+//line app/vmselect/prometheus/export.qtpl:145
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:145
+	WriteExportPromAPIResponse(qb422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:145
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:145
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:145
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:145
+}
+
+//line app/vmselect/prometheus/export.qtpl:147
+func StreamExportStdResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:148
+	for bb := range resultsCh {
+//line app/vmselect/prometheus/export.qtpl:149
+		qw422016.N().Z(bb.B)
+//line app/vmselect/prometheus/export.qtpl:150
+		quicktemplate.ReleaseByteBuffer(bb)
+
+//line app/vmselect/prometheus/export.qtpl:151
+	}
+//line app/vmselect/prometheus/export.qtpl:152
+}
+
+//line app/vmselect/prometheus/export.qtpl:152
+func WriteExportStdResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
+//line app/vmselect/prometheus/export.qtpl:152
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:152
+	StreamExportStdResponse(qw422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:152
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:152
+}
+
+//line app/vmselect/prometheus/export.qtpl:152
+func ExportStdResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
+//line app/vmselect/prometheus/export.qtpl:152
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:152
+	WriteExportStdResponse(qb422016, resultsCh)
+//line app/vmselect/prometheus/export.qtpl:152
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:152
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:152
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:152
+}
+
+//line app/vmselect/prometheus/export.qtpl:154
+func streamprometheusMetricName(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/export.qtpl:155
+	qw422016.N().Z(mn.MetricGroup)
+//line app/vmselect/prometheus/export.qtpl:156
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/export.qtpl:156
+		qw422016.N().S(`{`)
+//line app/vmselect/prometheus/export.qtpl:158
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/export.qtpl:159
+		qw422016.N().Z(tags[0].Key)
+//line app/vmselect/prometheus/export.qtpl:159
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/export.qtpl:159
+		qw422016.N().QZ(tags[0].Value)
+//line app/vmselect/prometheus/export.qtpl:160
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/export.qtpl:161
+		for i := range tags {
+//line app/vmselect/prometheus/export.qtpl:162
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/export.qtpl:162
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/export.qtpl:163
+			qw422016.N().Z(tag.Key)
+//line app/vmselect/prometheus/export.qtpl:163
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/export.qtpl:163
+			qw422016.N().QZ(tag.Value)
+//line app/vmselect/prometheus/export.qtpl:164
+		}
+//line app/vmselect/prometheus/export.qtpl:164
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/export.qtpl:166
+	}
+//line app/vmselect/prometheus/export.qtpl:167
+}
+
+//line app/vmselect/prometheus/export.qtpl:167
+func writeprometheusMetricName(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/export.qtpl:167
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/export.qtpl:167
+	streamprometheusMetricName(qw422016, mn)
+//line app/vmselect/prometheus/export.qtpl:167
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/export.qtpl:167
+}
+
+//line app/vmselect/prometheus/export.qtpl:167
+func prometheusMetricName(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/export.qtpl:167
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/export.qtpl:167
+	writeprometheusMetricName(qb422016, mn)
+//line app/vmselect/prometheus/export.qtpl:167
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/export.qtpl:167
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/export.qtpl:167
+	return qs422016
+//line app/vmselect/prometheus/export.qtpl:167
 }

app/vmselect/prometheus/prometheus.go

@@ -8,12 +8,16 @@ import (
 	"runtime"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -35,7 +39,7 @@ var (
 	maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+
 		"By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+
 		"Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+
-		"See also '-search.maxLookback' flag, which has the same meanining due to historical reasons")
+		"See also '-search.maxLookback' flag, which has the same meaning due to historical reasons")
 )
 
 // Default step used if not set.
@@ -74,44 +78,201 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 	if err != nil {
 		return err
 	}
-	sq := &storage.SearchQuery{
-		MinTimestamp: start,
-		MaxTimestamp: end,
-		TagFilterss:  tagFilterss,
-	}
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
 	rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 
-	resultsCh := make(chan *quicktemplate.ByteBuffer)
-	doneCh := make(chan error)
-	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
-			bb := quicktemplate.AcquireByteBuffer()
-			WriteFederate(bb, rs)
-			resultsCh <- bb
-		})
-		close(resultsCh)
-		doneCh <- err
-	}()
-
-	w.Header().Set("Content-Type", "text/plain")
-	for bb := range resultsCh {
-		w.Write(bb.B)
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+		if err := bw.Error(); err != nil {
+			return err
+		}
+		bb := quicktemplate.AcquireByteBuffer()
+		WriteFederate(bb, rs)
+		_, err := bw.Write(bb.B)
 		quicktemplate.ReleaseByteBuffer(bb)
-	}
-
-	err = <-doneCh
+		return err
+	})
 	if err != nil {
 		return fmt.Errorf("error during data fetching: %w", err)
 	}
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	federateDuration.UpdateDuration(startTime)
 	return nil
 }
 
 var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/federate"}`)
 
+// ExportCSVHandler exports data in CSV format from /api/v1/export/csv
+func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	ct := startTime.UnixNano() / 1e6
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse request form values: %w", err)
+	}
+	format := r.FormValue("format")
+	if len(format) == 0 {
+		return fmt.Errorf("missing `format` arg; see https://victoriametrics.github.io/#how-to-export-csv-data")
+	}
+	fieldNames := strings.Split(format, ",")
+	matches := r.Form["match[]"]
+	if len(matches) == 0 {
+		// Maintain backwards compatibility
+		match := r.FormValue("match")
+		if len(match) == 0 {
+			return fmt.Errorf("missing `match[]` arg")
+		}
+		matches = []string{match}
+	}
+	start, err := searchutils.GetTime(r, "start", 0)
+	if err != nil {
+		return err
+	}
+	end, err := searchutils.GetTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
+	deadline := searchutils.GetDeadlineForExport(r, startTime)
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return err
+	}
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
+	w.Header().Set("Content-Type", "text/csv; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+
+	resultsCh := make(chan *quicktemplate.ByteBuffer, runtime.GOMAXPROCS(-1))
+	doneCh := make(chan error)
+	go func() {
+		err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
+			if err := bw.Error(); err != nil {
+				return err
+			}
+			if err := b.UnmarshalData(); err != nil {
+				return fmt.Errorf("cannot unmarshal block during export: %s", err)
+			}
+			xb := exportBlockPool.Get().(*exportBlock)
+			xb.mn = mn
+			xb.timestamps, xb.values = b.AppendRowsWithTimeRangeFilter(xb.timestamps[:0], xb.values[:0], tr)
+			if len(xb.timestamps) > 0 {
+				bb := quicktemplate.AcquireByteBuffer()
+				WriteExportCSVLine(bb, xb, fieldNames)
+				resultsCh <- bb
+			}
+			xb.reset()
+			exportBlockPool.Put(xb)
+			return nil
+		})
+		close(resultsCh)
+		doneCh <- err
+	}()
+	// Consume all the data from resultsCh.
+	for bb := range resultsCh {
+		// Do not check for error in bw.Write, since this error is checked inside netstorage.ExportBlocks above.
+		_, _ = bw.Write(bb.B)
+		quicktemplate.ReleaseByteBuffer(bb)
+	}
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	err = <-doneCh
+	if err != nil {
+		return fmt.Errorf("error during exporting data to csv: %w", err)
+	}
+	exportCSVDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/csv"}`)
+
+// ExportNativeHandler exports data in native format from /api/v1/export/native.
+func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	ct := startTime.UnixNano() / 1e6
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse request form values: %w", err)
+	}
+	matches := r.Form["match[]"]
+	if len(matches) == 0 {
+		// Maintain backwards compatibility
+		match := r.FormValue("match")
+		if len(match) == 0 {
+			return fmt.Errorf("missing `match[]` arg")
+		}
+		matches = []string{match}
+	}
+	start, err := searchutils.GetTime(r, "start", 0)
+	if err != nil {
+		return err
+	}
+	end, err := searchutils.GetTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
+	deadline := searchutils.GetDeadlineForExport(r, startTime)
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return err
+	}
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
+	w.Header().Set("Content-Type", "VictoriaMetrics/native")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+
+	// Marshal tr
+	trBuf := make([]byte, 0, 16)
+	trBuf = encoding.MarshalInt64(trBuf, start)
+	trBuf = encoding.MarshalInt64(trBuf, end)
+	_, _ = bw.Write(trBuf)
+
+	// Marshal native blocks.
+	err = netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
+		if err := bw.Error(); err != nil {
+			return err
+		}
+		dstBuf := bbPool.Get()
+		tmpBuf := bbPool.Get()
+		dst := dstBuf.B
+		tmp := tmpBuf.B
+
+		// Marshal mn
+		tmp = mn.Marshal(tmp[:0])
+		dst = encoding.MarshalUint32(dst, uint32(len(tmp)))
+		dst = append(dst, tmp...)
+
+		// Marshal b
+		tmp = b.MarshalPortable(tmp[:0])
+		dst = encoding.MarshalUint32(dst, uint32(len(tmp)))
+		dst = append(dst, tmp...)
+
+		tmpBuf.B = tmp
+		bbPool.Put(tmpBuf)
+
+		_, err := bw.Write(dst)
+
+		dstBuf.B = dst
+		bbPool.Put(dstBuf)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	exportNativeDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var exportNativeDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/native"}`)
+
+var bbPool bytesutil.ByteBufferPool
+
 // ExportHandler exports data in raw format from /api/v1/export.
 func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := startTime.UnixNano() / 1e6
@@ -137,11 +298,12 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	}
 	format := r.FormValue("format")
 	maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line")))
+	reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
 	deadline := searchutils.GetDeadlineForExport(r, startTime)
 	if start >= end {
 		end = start + defaultStep
 	}
-	if err := exportHandler(w, matches, start, end, format, maxRowsPerLine, deadline); err != nil {
+	if err := exportHandler(w, matches, start, end, format, maxRowsPerLine, reduceMemUsage, deadline); err != nil {
 		return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %w", matches, start, end, err)
 	}
 	exportDuration.UpdateDuration(startTime)
@@ -150,17 +312,34 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 
 var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export"}`)
 
-func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, maxRowsPerLine int, deadline searchutils.Deadline) error {
+func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, maxRowsPerLine int, reduceMemUsage bool, deadline searchutils.Deadline) error {
 	writeResponseFunc := WriteExportStdResponse
-	writeLineFunc := func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
+	writeLineFunc := func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
 		bb := quicktemplate.AcquireByteBuffer()
-		WriteExportJSONLine(bb, rs)
+		WriteExportJSONLine(bb, xb)
 		resultsCh <- bb
 	}
+	contentType := "application/stream+json; charset=utf-8"
+	if format == "prometheus" {
+		contentType = "text/plain; charset=utf-8"
+		writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
+			bb := quicktemplate.AcquireByteBuffer()
+			WriteExportPrometheusLine(bb, xb)
+			resultsCh <- bb
+		}
+	} else if format == "promapi" {
+		writeResponseFunc = WriteExportPromAPIResponse
+		writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
+			bb := quicktemplate.AcquireByteBuffer()
+			WriteExportPromAPILine(bb, xb)
+			resultsCh <- bb
+		}
+	}
 	if maxRowsPerLine > 0 {
-		writeLineFunc = func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
-			valuesOrig := rs.Values
-			timestampsOrig := rs.Timestamps
+		writeLineFuncOrig := writeLineFunc
+		writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
+			valuesOrig := xb.values
+			timestampsOrig := xb.timestamps
 			values := valuesOrig
 			timestamps := timestampsOrig
 			for len(values) > 0 {
@@ -177,30 +356,12 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 					values = nil
 					timestamps = nil
 				}
-				rs.Values = valuesChunk
-				rs.Timestamps = timestampsChunk
-				bb := quicktemplate.AcquireByteBuffer()
-				WriteExportJSONLine(bb, rs)
-				resultsCh <- bb
+				xb.values = valuesChunk
+				xb.timestamps = timestampsChunk
+				writeLineFuncOrig(xb, resultsCh)
 			}
-			rs.Values = valuesOrig
-			rs.Timestamps = timestampsOrig
-		}
-	}
-	contentType := "application/stream+json"
-	if format == "prometheus" {
-		contentType = "text/plain"
-		writeLineFunc = func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
-			bb := quicktemplate.AcquireByteBuffer()
-			WriteExportPrometheusLine(bb, rs)
-			resultsCh <- bb
-		}
-	} else if format == "promapi" {
-		writeResponseFunc = WriteExportPromAPIResponse
-		writeLineFunc = func(rs *netstorage.Result, resultsCh chan<- *quicktemplate.ByteBuffer) {
-			bb := quicktemplate.AcquireByteBuffer()
-			WriteExportPromAPILine(bb, rs)
-			resultsCh <- bb
+			xb.values = valuesOrig
+			xb.timestamps = timestampsOrig
 		}
 	}
 
@@ -208,33 +369,63 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 	if err != nil {
 		return err
 	}
-	sq := &storage.SearchQuery{
-		MinTimestamp: start,
-		MaxTimestamp: end,
-		TagFilterss:  tagFilterss,
-	}
-	rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
-	if err != nil {
-		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
-	}
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
+	w.Header().Set("Content-Type", contentType)
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
 
 	resultsCh := make(chan *quicktemplate.ByteBuffer, runtime.GOMAXPROCS(-1))
 	doneCh := make(chan error)
-	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
-			writeLineFunc(rs, resultsCh)
-		})
-		close(resultsCh)
-		doneCh <- err
-	}()
+	if !reduceMemUsage {
+		rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
+		}
+		go func() {
+			err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+				if err := bw.Error(); err != nil {
+					return err
+				}
+				xb := exportBlockPool.Get().(*exportBlock)
+				xb.mn = &rs.MetricName
+				xb.timestamps = rs.Timestamps
+				xb.values = rs.Values
+				writeLineFunc(xb, resultsCh)
+				xb.reset()
+				exportBlockPool.Put(xb)
+				return nil
+			})
+			close(resultsCh)
+			doneCh <- err
+		}()
+	} else {
+		go func() {
+			err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
+				if err := bw.Error(); err != nil {
+					return err
+				}
+				if err := b.UnmarshalData(); err != nil {
+					return fmt.Errorf("cannot unmarshal block during export: %s", err)
+				}
+				xb := exportBlockPool.Get().(*exportBlock)
+				xb.mn = mn
+				xb.timestamps, xb.values = b.AppendRowsWithTimeRangeFilter(xb.timestamps[:0], xb.values[:0], tr)
+				if len(xb.timestamps) > 0 {
+					writeLineFunc(xb, resultsCh)
+				}
+				xb.reset()
+				exportBlockPool.Put(xb)
+				return nil
+			})
+			close(resultsCh)
+			doneCh <- err
+		}()
+	}
 
-	w.Header().Set("Content-Type", contentType)
-	writeResponseFunc(w, resultsCh)
-
-	// Consume all the data from resultsCh in the event writeResponseFunc
-	// fails to consume all the data.
-	for bb := range resultsCh {
-		quicktemplate.ReleaseByteBuffer(bb)
+	// writeResponseFunc must consume all the data from resultsCh.
+	writeResponseFunc(bw, resultsCh)
+	if err := bw.Flush(); err != nil {
+		return err
 	}
 	err = <-doneCh
 	if err != nil {
@@ -243,6 +434,24 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 	return nil
 }
 
+type exportBlock struct {
+	mn         *storage.MetricName
+	timestamps []int64
+	values     []float64
+}
+
+func (xb *exportBlock) reset() {
+	xb.mn = nil
+	xb.timestamps = xb.timestamps[:0]
+	xb.values = xb.values[:0]
+}
+
+var exportBlockPool = &sync.Pool{
+	New: func() interface{} {
+		return &exportBlock{}
+	},
+}
+
 // DeleteHandler processes /api/v1/admin/tsdb/delete_series prometheus API request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#delete-series
@@ -261,9 +470,7 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	sq := &storage.SearchQuery{
-		TagFilterss: tagFilterss,
-	}
+	sq := storage.NewSearchQuery(0, 0, tagFilterss)
 	deletedCount, err := netstorage.DeleteSeries(sq)
 	if err != nil {
 		return fmt.Errorf("cannot delete time series matching %q: %w", matches, err)
@@ -286,11 +493,31 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
 	var labelValues []string
-	if len(r.Form["match[]"]) == 0 && len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
-		var err error
-		labelValues, err = netstorage.GetLabelValues(labelName, deadline)
-		if err != nil {
-			return fmt.Errorf(`cannot obtain label values for %q: %w`, labelName, err)
+	if len(r.Form["match[]"]) == 0 {
+		if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
+			var err error
+			labelValues, err = netstorage.GetLabelValues(labelName, deadline)
+			if err != nil {
+				return fmt.Errorf(`cannot obtain label values for %q: %w`, labelName, err)
+			}
+		} else {
+			ct := startTime.UnixNano() / 1e6
+			end, err := searchutils.GetTime(r, "end", ct)
+			if err != nil {
+				return err
+			}
+			start, err := searchutils.GetTime(r, "start", end-defaultStep)
+			if err != nil {
+				return err
+			}
+			tr := storage.TimeRange{
+				MinTimestamp: start,
+				MaxTimestamp: end,
+			}
+			labelValues, err = netstorage.GetLabelValuesOnTimeRange(labelName, tr, deadline)
+			if err != nil {
+				return fmt.Errorf(`cannot obtain label values on time range for %q: %w`, labelName, err)
+			}
 		}
 	} else {
 		// Extended functionality that allows filtering by label filters and time range
@@ -316,8 +543,13 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr
 		}
 	}
 
-	w.Header().Set("Content-Type", "application/json")
-	WriteLabelValuesResponse(w, labelValues)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteLabelValuesResponse(bw, labelValues)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	labelValuesDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -346,31 +578,41 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
 	if start >= end {
 		end = start + defaultStep
 	}
-	sq := &storage.SearchQuery{
-		MinTimestamp: start,
-		MaxTimestamp: end,
-		TagFilterss:  tagFilterss,
-	}
-	rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
-	if err != nil {
-		return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
-	}
-
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
 	m := make(map[string]struct{})
-	var mLock sync.Mutex
-	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
-		labelValue := rs.MetricName.GetTagValue(labelName)
-		if len(labelValue) == 0 {
-			return
+	if end-start > 24*3600*1000 {
+		// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
+		mns, err := netstorage.SearchMetricNames(sq, deadline)
+		if err != nil {
+			return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
+		}
+		for _, mn := range mns {
+			labelValue := mn.GetTagValue(labelName)
+			if len(labelValue) == 0 {
+				continue
+			}
+			m[string(labelValue)] = struct{}{}
+		}
+	} else {
+		rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
+		if err != nil {
+			return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
+		}
+		var mLock sync.Mutex
+		err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+			labelValue := rs.MetricName.GetTagValue(labelName)
+			if len(labelValue) == 0 {
+				return nil
+			}
+			mLock.Lock()
+			m[string(labelValue)] = struct{}{}
+			mLock.Unlock()
+			return nil
+		})
+		if err != nil {
+			return nil, fmt.Errorf("error when data fetching: %w", err)
 		}
-		mLock.Lock()
-		m[string(labelValue)] = struct{}{}
-		mLock.Unlock()
-	})
-	if err != nil {
-		return nil, fmt.Errorf("error when data fetching: %w", err)
 	}
-
 	labelValues := make([]string, 0, len(m))
 	for labelValue := range m {
 		labelValues = append(labelValues, labelValue)
@@ -388,8 +630,13 @@ func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	if err != nil {
 		return fmt.Errorf(`cannot obtain label entries: %w`, err)
 	}
-	w.Header().Set("Content-Type", "application/json")
-	WriteLabelsCountResponse(w, labelEntries)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteLabelsCountResponse(bw, labelEntries)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	labelsCountDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -434,8 +681,13 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
 	if err != nil {
 		return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
 	}
-	w.Header().Set("Content-Type", "application/json")
-	WriteTSDBStatusResponse(w, status)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteTSDBStatusResponse(bw, status)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	tsdbStatusDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -451,11 +703,31 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		return fmt.Errorf("cannot parse form values: %w", err)
 	}
 	var labels []string
-	if len(r.Form["match[]"]) == 0 && len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
-		var err error
-		labels, err = netstorage.GetLabels(deadline)
-		if err != nil {
-			return fmt.Errorf("cannot obtain labels: %w", err)
+	if len(r.Form["match[]"]) == 0 {
+		if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
+			var err error
+			labels, err = netstorage.GetLabels(deadline)
+			if err != nil {
+				return fmt.Errorf("cannot obtain labels: %w", err)
+			}
+		} else {
+			ct := startTime.UnixNano() / 1e6
+			end, err := searchutils.GetTime(r, "end", ct)
+			if err != nil {
+				return err
+			}
+			start, err := searchutils.GetTime(r, "start", end-defaultStep)
+			if err != nil {
+				return err
+			}
+			tr := storage.TimeRange{
+				MinTimestamp: start,
+				MaxTimestamp: end,
+			}
+			labels, err = netstorage.GetLabelsOnTimeRange(tr, deadline)
+			if err != nil {
+				return fmt.Errorf("cannot obtain labels on time range: %w", err)
+			}
 		}
 	} else {
 		// Extended functionality that allows filtering by label filters and time range
@@ -479,8 +751,13 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	w.Header().Set("Content-Type", "application/json")
-	WriteLabelsResponse(w, labels)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteLabelsResponse(bw, labels)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	labelsDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -496,32 +773,41 @@ func labelsWithMatches(matches []string, start, end int64, deadline searchutils.
 	if start >= end {
 		end = start + defaultStep
 	}
-	sq := &storage.SearchQuery{
-		MinTimestamp: start,
-		MaxTimestamp: end,
-		TagFilterss:  tagFilterss,
-	}
-	rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
-	if err != nil {
-		return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
-	}
-
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
 	m := make(map[string]struct{})
-	var mLock sync.Mutex
-	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
-		mLock.Lock()
-		tags := rs.MetricName.Tags
-		for i := range tags {
-			t := &tags[i]
-			m[string(t.Key)] = struct{}{}
+	if end-start > 24*3600*1000 {
+		// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
+		mns, err := netstorage.SearchMetricNames(sq, deadline)
+		if err != nil {
+			return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
+		}
+		for _, mn := range mns {
+			for _, tag := range mn.Tags {
+				m[string(tag.Key)] = struct{}{}
+			}
+		}
+		if len(mns) > 0 {
+			m["__name__"] = struct{}{}
+		}
+	} else {
+		rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
+		if err != nil {
+			return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
+		}
+		var mLock sync.Mutex
+		err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+			mLock.Lock()
+			for _, tag := range rs.MetricName.Tags {
+				m[string(tag.Key)] = struct{}{}
+			}
+			m["__name__"] = struct{}{}
+			mLock.Unlock()
+			return nil
+		})
+		if err != nil {
+			return nil, fmt.Errorf("error when data fetching: %w", err)
 		}
-		m["__name__"] = struct{}{}
-		mLock.Unlock()
-	})
-	if err != nil {
-		return nil, fmt.Errorf("error when data fetching: %w", err)
 	}
-
 	labels := make([]string, 0, len(m))
 	for label := range m {
 		labels = append(labels, label)
@@ -539,8 +825,13 @@ func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %w", err)
 	}
-	w.Header().Set("Content-Type", "application/json")
-	WriteSeriesCountResponse(w, n)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteSeriesCountResponse(bw, n)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	seriesCountDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -581,35 +872,60 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	if start >= end {
 		end = start + defaultStep
 	}
-	sq := &storage.SearchQuery{
-		MinTimestamp: start,
-		MaxTimestamp: end,
-		TagFilterss:  tagFilterss,
+	sq := storage.NewSearchQuery(start, end, tagFilterss)
+	if end-start > 24*3600*1000 {
+		// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
+		mns, err := netstorage.SearchMetricNames(sq, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		bw := bufferedwriter.Get(w)
+		defer bufferedwriter.Put(bw)
+		resultsCh := make(chan *quicktemplate.ByteBuffer)
+		go func() {
+			for i := range mns {
+				bb := quicktemplate.AcquireByteBuffer()
+				writemetricNameObject(bb, &mns[i])
+				resultsCh <- bb
+			}
+			close(resultsCh)
+		}()
+		// WriteSeriesResponse must consume all the data from resultsCh.
+		WriteSeriesResponse(bw, resultsCh)
+		if err := bw.Flush(); err != nil {
+			return err
+		}
+		seriesDuration.UpdateDuration(startTime)
+		return nil
 	}
 	rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
 	resultsCh := make(chan *quicktemplate.ByteBuffer)
 	doneCh := make(chan error)
 	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
+			if err := bw.Error(); err != nil {
+				return err
+			}
 			bb := quicktemplate.AcquireByteBuffer()
 			writemetricNameObject(bb, &rs.MetricName)
 			resultsCh <- bb
+			return nil
 		})
 		close(resultsCh)
 		doneCh <- err
 	}()
-
-	w.Header().Set("Content-Type", "application/json")
-	WriteSeriesResponse(w, resultsCh)
-
-	// Consume all the data from resultsCh in the event WriteSeriesResponse
-	// fails to consume all the data.
-	for bb := range resultsCh {
-		quicktemplate.ReleaseByteBuffer(bb)
+	// WriteSeriesResponse must consume all the data from resultsCh.
+	WriteSeriesResponse(bw, resultsCh)
+	if err := bw.Flush(); err != nil {
+		return err
 	}
 	err = <-doneCh
 	if err != nil {
@@ -650,12 +966,6 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	if len(query) > maxQueryLen.N {
 		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
 	}
-	queryOffset := getLatencyOffsetMilliseconds()
-	if !searchutils.GetBool(r, "nocache") && ct-start < queryOffset {
-		// Adjust start time only if `nocache` arg isn't set.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
-		start = ct - queryOffset
-	}
 	if childQuery, windowStr, offsetStr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
 		window, err := parsePositiveDuration(windowStr, step)
 		if err != nil {
@@ -668,7 +978,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		start -= offset
 		end := start
 		start = end - window
-		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", 0, deadline); err != nil {
+		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", 0, false, deadline); err != nil {
 			return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %w", childQuery, start, end, err)
 		}
 		queryDuration.UpdateDuration(startTime)
@@ -700,6 +1010,16 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 		return nil
 	}
 
+	queryOffset := getLatencyOffsetMilliseconds()
+	if !searchutils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset {
+		// Adjust start time only if `nocache` arg isn't set.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
+		startPrev := start
+		start = ct - queryOffset
+		queryOffset = startPrev - start
+	} else {
+		queryOffset = 0
+	}
 	ec := promql.EvalConfig{
 		Start:            start,
 		End:              start,
@@ -712,9 +1032,22 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
 	if err != nil {
 		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %w", query, start, step, err)
 	}
+	if queryOffset > 0 {
+		for i := range result {
+			timestamps := result[i].Timestamps
+			for j := range timestamps {
+				timestamps[j] += queryOffset
+			}
+		}
+	}
 
-	w.Header().Set("Content-Type", "application/json")
-	WriteQueryResponse(w, result)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteQueryResponse(bw, result)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	queryDuration.UpdateDuration(startTime)
 	return nil
 }
@@ -807,8 +1140,13 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string,
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153
 	result = removeEmptyValuesAndTimeseries(result)
 
-	w.Header().Set("Content-Type", "application/json")
-	WriteQueryRangeResponse(w, result)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteQueryRangeResponse(bw, result)
+	if err := bw.Flush(); err != nil {
+		return err
+	}
 	return nil
 }
 

app/vmselect/promql/aggr.go

@@ -69,7 +69,9 @@ func newAggrFunc(afe func(tss []*timeseries) []*timeseries) aggrFunc {
 		if err != nil {
 			return nil, err
 		}
-		return aggrFuncExt(afe, tss, &afa.ae.Modifier, afa.ae.Limit, false)
+		return aggrFuncExt(func(tss []*timeseries, modififer *metricsql.ModifierExpr) []*timeseries {
+			return afe(tss)
+		}, tss, &afa.ae.Modifier, afa.ae.Limit, false)
 	}
 }
 
@@ -98,7 +100,8 @@ func removeGroupTags(metricName *storage.MetricName, modifier *metricsql.Modifie
 	}
 }
 
-func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeseries, modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) ([]*timeseries, error) {
+func aggrFuncExt(afe func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries, argOrig []*timeseries,
+	modifier *metricsql.ModifierExpr, maxSeries int, keepOriginal bool) ([]*timeseries, error) {
 	arg := copyTimeseriesMetricNames(argOrig, keepOriginal)
 
 	// Perform grouping.
@@ -124,7 +127,7 @@ func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeserie
 	dstTssCount := 0
 	rvs := make([]*timeseries, 0, len(m))
 	for _, tss := range m {
-		rv := afe(tss)
+		rv := afe(tss, modifier)
 		rvs = append(rvs, rv...)
 		srcTssCount += len(tss)
 		dstTssCount += len(rv)
@@ -141,7 +144,7 @@ func aggrFuncAny(afa *aggrFuncArg) ([]*timeseries, error) {
 	if err != nil {
 		return nil, err
 	}
-	afe := func(tss []*timeseries) []*timeseries {
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		return tss[:1]
 	}
 	limit := afa.ae.Limit
@@ -178,10 +181,11 @@ func aggrFuncSum(tss []*timeseries) []*timeseries {
 		sum := float64(0)
 		count := 0
 		for _, ts := range tss {
-			if math.IsNaN(ts.Values[i]) {
+			v := ts.Values[i]
+			if math.IsNaN(v) {
 				continue
 			}
-			sum += ts.Values[i]
+			sum += v
 			count++
 		}
 		if count == 0 {
@@ -449,7 +453,7 @@ func aggrFuncZScore(afa *aggrFuncArg) ([]*timeseries, error) {
 	if err != nil {
 		return nil, err
 	}
-	afe := func(tss []*timeseries) []*timeseries {
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		for i := range tss[0].Values {
 			// Calculate avg and stddev for tss points at position i.
 			// See `Rapid calculation methods` at https://en.wikipedia.org/wiki/Standard_deviation
@@ -494,6 +498,8 @@ func aggrFuncZScore(afa *aggrFuncArg) ([]*timeseries, error) {
 //
 // It is expected that a doesn't contain NaNs.
 //
+// The function modifies contents for a, so the caller must prepare it accordingly.
+//
 // See https://en.wikipedia.org/wiki/Mode_(statistics)
 func modeNoNaNs(prevValue float64, a []float64) float64 {
 	if len(a) == 0 {
@@ -548,7 +554,7 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 		// Do nothing
 	}
 
-	afe := func(tss []*timeseries) []*timeseries {
+	afe := func(tss []*timeseries, modififer *metricsql.ModifierExpr) []*timeseries {
 		m := make(map[float64]bool)
 		for _, ts := range tss {
 			for _, v := range ts.Values {
@@ -600,7 +606,7 @@ func newAggrFuncTopK(isReverse bool) aggrFunc {
 		if err != nil {
 			return nil, err
 		}
-		afe := func(tss []*timeseries) []*timeseries {
+		afe := func(tss []*timeseries, modififer *metricsql.ModifierExpr) []*timeseries {
 			for n := range tss[0].Values {
 				sort.Slice(tss, func(i, j int) bool {
 					a := tss[i].Values[n]
@@ -621,21 +627,32 @@ func newAggrFuncTopK(isReverse bool) aggrFunc {
 func newAggrFuncRangeTopK(f func(values []float64) float64, isReverse bool) aggrFunc {
 	return func(afa *aggrFuncArg) ([]*timeseries, error) {
 		args := afa.args
-		if err := expectTransformArgsNum(args, 2); err != nil {
-			return nil, err
+		if len(args) < 2 {
+			return nil, fmt.Errorf(`unexpected number of args; got %d; want at least %d`, len(args), 2)
+		}
+		if len(args) > 3 {
+			return nil, fmt.Errorf(`unexpected number of args; got %d; want no more than %d`, len(args), 3)
 		}
 		ks, err := getScalar(args[0], 0)
 		if err != nil {
 			return nil, err
 		}
-		afe := func(tss []*timeseries) []*timeseries {
-			return getRangeTopKTimeseries(tss, ks, f, isReverse)
+		remainingSumTagName := ""
+		if len(args) == 3 {
+			remainingSumTagName, err = getString(args[2], 2)
+			if err != nil {
+				return nil, err
+			}
+		}
+		afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
+			return getRangeTopKTimeseries(tss, modifier, ks, remainingSumTagName, f, isReverse)
 		}
 		return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true)
 	}
 }
 
-func getRangeTopKTimeseries(tss []*timeseries, ks []float64, f func(values []float64) float64, isReverse bool) []*timeseries {
+func getRangeTopKTimeseries(tss []*timeseries, modifier *metricsql.ModifierExpr, ks []float64, remainingSumTagName string,
+	f func(values []float64) float64, isReverse bool) []*timeseries {
 	type tsWithValue struct {
 		ts    *timeseries
 		value float64
@@ -659,35 +676,74 @@ func getRangeTopKTimeseries(tss []*timeseries, ks []float64, f func(values []flo
 	for i := range maxs {
 		tss[i] = maxs[i].ts
 	}
+	remainingSumTS := getRemainingSumTimeseries(tss, modifier, ks, remainingSumTagName)
 	for i, k := range ks {
 		fillNaNsAtIdx(i, k, tss)
 	}
+	if remainingSumTS != nil {
+		tss = append(tss, remainingSumTS)
+	}
 	return removeNaNs(tss)
 }
 
+func getRemainingSumTimeseries(tss []*timeseries, modifier *metricsql.ModifierExpr, ks []float64, remainingSumTagName string) *timeseries {
+	if len(remainingSumTagName) == 0 || len(tss) == 0 {
+		return nil
+	}
+	var dst timeseries
+	dst.CopyFromShallowTimestamps(tss[0])
+	removeGroupTags(&dst.MetricName, modifier)
+	dst.MetricName.RemoveTag(remainingSumTagName)
+	dst.MetricName.AddTag(remainingSumTagName, remainingSumTagName)
+	for i, k := range ks {
+		kn := getIntK(k, len(tss))
+		var sum float64
+		count := 0
+		for _, ts := range tss[:len(tss)-kn] {
+			v := ts.Values[i]
+			if math.IsNaN(v) {
+				continue
+			}
+			sum += v
+			count++
+		}
+		if count == 0 {
+			sum = nan
+		}
+		dst.Values[i] = sum
+	}
+	return &dst
+}
+
 func fillNaNsAtIdx(idx int, k float64, tss []*timeseries) {
-	if math.IsNaN(k) {
-		k = 0
-	}
-	kn := int(k)
-	if kn < 0 {
-		kn = 0
-	}
-	if kn > len(tss) {
-		kn = len(tss)
-	}
+	kn := getIntK(k, len(tss))
 	for _, ts := range tss[:len(tss)-kn] {
 		ts.Values[idx] = nan
 	}
 }
 
-func minValue(values []float64) float64 {
-	if len(values) == 0 {
-		return nan
+func getIntK(k float64, kMax int) int {
+	if math.IsNaN(k) {
+		return 0
 	}
-	min := values[0]
-	for _, v := range values[1:] {
-		if v < min {
+	kn := int(k)
+	if kn < 0 {
+		return 0
+	}
+	if kn > kMax {
+		return kMax
+	}
+	return kn
+}
+
+func minValue(values []float64) float64 {
+	min := nan
+	for len(values) > 0 && math.IsNaN(min) {
+		min = values[0]
+		values = values[1:]
+	}
+	for _, v := range values {
+		if !math.IsNaN(v) && v < min {
 			min = v
 		}
 	}
@@ -695,12 +751,13 @@ func minValue(values []float64) float64 {
 }
 
 func maxValue(values []float64) float64 {
-	if len(values) == 0 {
-		return nan
+	max := nan
+	for len(values) > 0 && math.IsNaN(max) {
+		max = values[0]
+		values = values[1:]
 	}
-	max := values[0]
-	for _, v := range values[1:] {
-		if v > max {
+	for _, v := range values {
+		if !math.IsNaN(v) && v > max {
 			max = v
 		}
 	}
@@ -744,7 +801,7 @@ func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
 	if err != nil {
 		return nil, err
 	}
-	afe := func(tss []*timeseries) []*timeseries {
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		// Calculate medians for each point across tss.
 		medians := make([]float64, len(ks))
 		h := histogram.GetFast()
@@ -769,7 +826,7 @@ func aggrFuncOutliersK(afa *aggrFuncArg) ([]*timeseries, error) {
 			}
 			return sum2
 		}
-		return getRangeTopKTimeseries(tss, ks, f, false)
+		return getRangeTopKTimeseries(tss, &afa.ae.Modifier, ks, "", f, false)
 	}
 	return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true)
 }
@@ -790,7 +847,7 @@ func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 			maxK = k
 		}
 	}
-	afe := func(tss []*timeseries) []*timeseries {
+	afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		if len(tss) > maxK {
 			tss = tss[:maxK]
 		}
@@ -831,8 +888,8 @@ func aggrFuncMedian(afa *aggrFuncArg) ([]*timeseries, error) {
 	return aggrFuncExt(afe, tss, &afa.ae.Modifier, afa.ae.Limit, false)
 }
 
-func newAggrQuantileFunc(phis []float64) func(tss []*timeseries) []*timeseries {
-	return func(tss []*timeseries) []*timeseries {
+func newAggrQuantileFunc(phis []float64) func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
+	return func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
 		dst := tss[0]
 		h := histogram.GetFast()
 		defer histogram.PutFast(h)

app/vmselect/promql/eval.go

@@ -653,11 +653,7 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
 	} else {
 		minTimestamp -= ec.Step
 	}
-	sq := &storage.SearchQuery{
-		MinTimestamp: minTimestamp,
-		MaxTimestamp: ec.End,
-		TagFilterss:  [][]storage.TagFilter{tfs},
-	}
+	sq := storage.NewSearchQuery(minTimestamp, ec.End, [][]storage.TagFilter{tfs})
 	rss, err := netstorage.ProcessSearchQuery(sq, true, ec.Deadline)
 	if err != nil {
 		return nil, err
@@ -741,7 +737,7 @@ func getRollupMemoryLimiter() *memoryLimiter {
 
 func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
-	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
 		preFunc(rs.Values, rs.Timestamps)
 		ts := getTimeseries()
 		defer putTimeseries(ts)
@@ -761,6 +757,7 @@ func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncCo
 			ts.Timestamps = nil
 			ts.denyReuse = false
 		}
+		return nil
 	})
 	if err != nil {
 		return nil, err
@@ -773,7 +770,7 @@ func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
 	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
 	var tssLock sync.Mutex
-	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {
 			if tsm := newTimeseriesMap(name, sharedTimestamps, &rs.MetricName); tsm != nil {
@@ -789,6 +786,7 @@ func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs
 			tss = append(tss, &ts)
 			tssLock.Unlock()
 		}
+		return nil
 	})
 	if err != nil {
 		return nil, err

app/vmselect/promql/exec.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"sort"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -15,7 +16,13 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
-var logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging")
+var (
+	logSlowQueryDuration   = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging")
+	treatDotsAsIsInRegexps = flag.Bool("search.treatDotsAsIsInRegexps", false, "Whether to treat dots as is in regexp label filters used in queries. "+
+		`For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped `+
+		`in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. `+
+		`Such escaping can be useful when querying Graphite data`)
+)
 
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
 
@@ -26,8 +33,8 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
 		defer func() {
 			d := time.Since(startTime)
 			if d >= *logSlowQueryDuration {
-				logger.Warnf("slow query according to -search.logSlowQueryDuration=%s: duration=%.3f seconds, start=%d, end=%d, step=%d, query=%q",
-					*logSlowQueryDuration, d.Seconds(), ec.Start/1000, ec.End/1000, ec.Step/1000, q)
+				logger.Warnf("slow query according to -search.logSlowQueryDuration=%s: remoteAddr=%s, duration=%.3f seconds, start=%d, end=%d, step=%d, query=%q",
+					*logSlowQueryDuration, ec.QuotedRemoteAddr, d.Seconds(), ec.Start/1000, ec.End/1000, ec.Step/1000, q)
 				slowQueries.Inc()
 			}
 		}()
@@ -175,7 +182,11 @@ func parsePromQLWithCache(q string) (metricsql.Expr, error) {
 	if pcv == nil {
 		e, err := metricsql.Parse(q)
 		if err == nil {
+			e = metricsql.Optimize(e)
 			e = adjustCmpOps(e)
+			if *treatDotsAsIsInRegexps {
+				e = escapeDotsInRegexpLabelFilters(e)
+			}
 		}
 		pcv = &parseCacheValue{
 			e:   e,
@@ -189,6 +200,41 @@ func parsePromQLWithCache(q string) (metricsql.Expr, error) {
 	return pcv.e, nil
 }
 
+func escapeDotsInRegexpLabelFilters(e metricsql.Expr) metricsql.Expr {
+	metricsql.VisitAll(e, func(expr metricsql.Expr) {
+		me, ok := expr.(*metricsql.MetricExpr)
+		if !ok {
+			return
+		}
+		for i := range me.LabelFilters {
+			f := &me.LabelFilters[i]
+			if f.IsRegexp {
+				f.Value = escapeDots(f.Value)
+			}
+		}
+	})
+	return e
+}
+
+func escapeDots(s string) string {
+	dotsCount := strings.Count(s, ".")
+	if dotsCount <= 0 {
+		return s
+	}
+	result := make([]byte, 0, len(s)+2*dotsCount)
+	for i := 0; i < len(s); i++ {
+		if s[i] == '.' && (i == 0 || s[i-1] != '\\') && (i+1 == len(s) || i+1 < len(s) && s[i+1] != '*' && s[i+1] != '+' && s[i+1] != '{') {
+			// Escape a dot if the following conditions are met:
+			// - if it isn't escaped already, i.e. if there is no `\` char before the dot.
+			// - if there is no regexp modifiers such as '+', '*' or '{' after the dot.
+			result = append(result, '\\', '.')
+		} else {
+			result = append(result, s[i])
+		}
+	}
+	return string(result)
+}
+
 var parseCacheV = func() *parseCache {
 	pc := &parseCache{
 		m: make(map[string]*parseCacheValue),

app/vmselect/promql/exec_test.go

@@ -7,8 +7,46 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
+func TestEscapeDots(t *testing.T) {
+	f := func(s, resultExpected string) {
+		t.Helper()
+		result := escapeDots(s)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for escapeDots(%q); got\n%s\nwant\n%s", s, result, resultExpected)
+		}
+	}
+	f("", "")
+	f("a", "a")
+	f("foobar", "foobar")
+	f(".", `\.`)
+	f(".*", `.*`)
+	f(".+", `.+`)
+	f("..", `\.\.`)
+	f("foo.b.{2}ar..+baz.*", `foo\.b.{2}ar\..+baz.*`)
+}
+
+func TestEscapeDotsInRegexpLabelFilters(t *testing.T) {
+	f := func(s, resultExpected string) {
+		t.Helper()
+		e, err := metricsql.Parse(s)
+		if err != nil {
+			t.Fatalf("unexpected error in metricsql.Parse(%q): %s", s, err)
+		}
+		e = escapeDotsInRegexpLabelFilters(e)
+		result := e.AppendString(nil)
+		if string(result) != resultExpected {
+			t.Fatalf("unexpected result for escapeDotsInRegexpLabelFilters(%q);\ngot\n%s\nwant\n%s", s, result, resultExpected)
+		}
+	}
+	f("2", "2")
+	f(`foo.bar + 123`, `foo.bar + 123`)
+	f(`foo{bar=~"baz.xx.yyy"}`, `foo{bar=~"baz\\.xx\\.yyy"}`)
+	f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e", x=~"a\\.b.+[\\.a]", y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`)
+}
+
 func TestExecSuccess(t *testing.T) {
 	start := int64(1000e3)
 	end := int64(2000e3)
@@ -582,10 +620,6 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1, 1, 1, 1, 1, 1},
 			Timestamps: timestampsExpected,
 		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("yy"),
-			Value: []byte("foo"),
-		}}
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -636,6 +670,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1000, 1200, 1400, 1400, 1400, 1400},
 			Timestamps: timestampsExpected,
 		}
+		r.MetricName.MetricGroup = []byte("foobar")
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -647,6 +682,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{1000, 1200, 1400, 1400, 1400, 1400},
 			Timestamps: timestampsExpected,
 		}
+		r.MetricName.MetricGroup = []byte("foobar")
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -3634,6 +3670,7 @@ func TestExecSuccess(t *testing.T) {
 			Values:     []float64{7.8, 9.9, 11.9, 13.9, 15.9, 17.9},
 			Timestamps: timestampsExpected,
 		}
+		r.MetricName.MetricGroup = []byte("foobar")
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
@@ -4194,7 +4231,7 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`topk_max(1)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort(topk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
+		q := `topk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
@@ -4207,6 +4244,84 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1}
 		f(q, resultExpected)
 	})
+	t.Run(`topk_max(1, remaining_sum)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_desc(topk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"), "remaining_sum"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("remaining_sum"),
+				Value: []byte("remaining_sum"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`topk_max(2, remaining_sum)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_desc(topk_max(2, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"), "remaining_sum"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`topk_max(3, remaining_sum)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_desc(topk_max(3, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"), "remaining_sum"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 10.666666666666666, 12, 13.333333333333334},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`bottomk_max(1)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(bottomk_max(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
@@ -4450,7 +4565,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `distinct_over_time((time() < 1700)[500s])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{3, 3, 3, 3, 2, 1},
+			Values:     []float64{3, 3, 3, 3, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -4461,7 +4576,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `distinct_over_time((time() < 1700)[2.5i])`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{3, 3, 3, 3, 2, 1},
+			Values:     []float64{3, 3, 3, 3, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}

app/vmselect/promql/rollup.go

@@ -15,7 +15,7 @@ import (
 	"github.com/valyala/histogram"
 )
 
-var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The mimimum interval for staleness calculations. "+
+var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The minimum interval for staleness calculations. "+
 	"This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. "+
 	"See also '-search.maxStalenessInterval'")
 
@@ -169,12 +169,20 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
 }
 
 var rollupFuncsKeepMetricGroup = map[string]bool{
+	"holt_winters":          true,
+	"predict_linear":        true,
 	"default_rollup":        true,
+	"avg_over_time":         true,
+	"min_over_time":         true,
+	"max_over_time":         true,
+	"quantile_over_time":    true,
 	"rollup":                true,
+	"geomean_over_time":     true,
 	"hoeffding_bound_lower": true,
 	"hoeffding_bound_upper": true,
 	"first_over_time":       true,
 	"last_over_time":        true,
+	"mode_over_time":        true,
 }
 
 func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
@@ -250,15 +258,16 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en
 	}
 	newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig {
 		return &rollupConfig{
-			TagValue:        tagValue,
-			Func:            rf,
-			Start:           start,
-			End:             end,
-			Step:            step,
-			Window:          window,
-			MayAdjustWindow: !rollupFuncsCannotAdjustWindow[name],
-			LookbackDelta:   lookbackDelta,
-			Timestamps:      sharedTimestamps,
+			TagValue:          tagValue,
+			Func:              rf,
+			Start:             start,
+			End:               end,
+			Step:              step,
+			Window:            window,
+			MayAdjustWindow:   !rollupFuncsCannotAdjustWindow[name],
+			CanDropLastSample: name == "default_rollup",
+			LookbackDelta:     lookbackDelta,
+			Timestamps:        sharedTimestamps,
 		}
 	}
 	appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig {
@@ -317,14 +326,32 @@ func getRollupFunc(funcName string) newRollupFunc {
 }
 
 type rollupFuncArg struct {
-	prevValue     float64
-	prevTimestamp int64
-	values        []float64
-	timestamps    []int64
+	// The value preceeding values if it fits staleness interval.
+	prevValue float64
 
+	// The timestamp for prevValue.
+	prevTimestamp int64
+
+	// Values that fit window ending at currTimestamp.
+	values []float64
+
+	// Timestamps for values.
+	timestamps []int64
+
+	// Real value preceeding values without restrictions on staleness interval.
+	realPrevValue float64
+
+	// Real value which goes after values.
+	realNextValue float64
+
+	// Current timestamp for rollup evaluation.
 	currTimestamp int64
-	idx           int
-	window        int64
+
+	// Index for the currently evaluated point relative to time range for query evaluation.
+	idx int
+
+	// Time window for rollup calculations.
+	window int64
 
 	tsm *timeseriesMap
 }
@@ -362,6 +389,11 @@ type rollupConfig struct {
 	// when using window smaller than 2 x scrape_interval.
 	MayAdjustWindow bool
 
+	// Whether the last sample can be dropped during rollup calculations.
+	// The last sample can be dropped for `default_rollup()` function only.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748 .
+	CanDropLastSample bool
+
 	Timestamps []int64
 
 	// LoookbackDelta is the analog to `-query.lookback-delta` from Prometheus world.
@@ -474,6 +506,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	window := rc.Window
 	if window <= 0 {
 		window = rc.Step
+		if rc.LookbackDelta > 0 && window > rc.LookbackDelta {
+			// Implicitly set window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
+			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
+			window = rc.LookbackDelta
+		}
 	}
 	if rc.MayAdjustWindow && window < maxPrevInterval {
 		window = maxPrevInterval
@@ -487,6 +524,10 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	j := 0
 	ni := 0
 	nj := 0
+	stalenessInterval := int64(float64(scrapeInterval) * 0.9)
+	// Do not drop trailing data points for queries, which return 2 or 1 point (aka instant queries).
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
+	canDropLastSample := rc.CanDropLastSample && len(rc.Timestamps) > 2
 	for _, tEnd := range rc.Timestamps {
 		tStart := tEnd - window
 		ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni)
@@ -503,9 +544,28 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 			rfa.prevValue = values[i-1]
 			rfa.prevTimestamp = timestamps[i-1]
 		}
-
 		rfa.values = values[i:j]
 		rfa.timestamps = timestamps[i:j]
+		if canDropLastSample && j == len(timestamps) && j > 0 && (tEnd-timestamps[j-1] > stalenessInterval || i == j && len(timestamps) == 1) {
+			// Drop trailing data points in the following cases:
+			// - if the distance between the last raw sample and tEnd exceeds stalenessInterval
+			// - if time series contains only a single raw sample
+			// This should prevent from double counting when a label changes in time series (for instance,
+			// during new deployment in K8S). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
+			rfa.prevValue = nan
+			rfa.values = nil
+			rfa.timestamps = nil
+		}
+		if i > 0 {
+			rfa.realPrevValue = values[i-1]
+		} else {
+			rfa.realPrevValue = nan
+		}
+		if j < len(values) {
+			rfa.realNextValue = values[j]
+		} else {
+			rfa.realNextValue = nan
+		}
 		rfa.currTimestamp = tEnd
 		value := rc.Func(rfa)
 		rfa.idx++
@@ -1212,6 +1272,12 @@ func rollupDelta(rfa *rollupFuncArg) float64 {
 		if len(values) == 0 {
 			return nan
 		}
+		if !math.IsNaN(rfa.realPrevValue) {
+			// Assume that the value didn't change during the current gap.
+			// This should fix high delta() and increase() values at the end of gaps.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/894
+			return values[len(values)-1] - rfa.realPrevValue
+		}
 		// Assume that the previous non-existing value was 0
 		// only if the first value doesn't exceed too much the delta with the next value.
 		//
@@ -1224,6 +1290,8 @@ func rollupDelta(rfa *rollupFuncArg) float64 {
 		d := float64(10)
 		if len(values) > 1 {
 			d = values[1] - values[0]
+		} else if !math.IsNaN(rfa.realNextValue) {
+			d = rfa.realNextValue - values[0]
 		}
 		if math.Abs(values[0]) < 10*(math.Abs(d)+1) {
 			prevValue = 0
@@ -1574,7 +1642,23 @@ func rollupTimestamp(rfa *rollupFuncArg) float64 {
 func rollupModeOverTime(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
-	return modeNoNaNs(rfa.prevValue, rfa.values)
+
+	// Copy rfa.values to a.A, since modeNoNaNs modifies a.A contents.
+	a := float64sPool.Get().(*float64s)
+	a.A = append(a.A[:0], rfa.values...)
+	result := modeNoNaNs(rfa.prevValue, a.A)
+	float64sPool.Put(a)
+	return result
+}
+
+var float64sPool = &sync.Pool{
+	New: func() interface{} {
+		return &float64s{}
+	},
+}
+
+type float64s struct {
+	A []float64
 }
 
 func rollupAscentOverTime(rfa *rollupFuncArg) float64 {

app/vmselect/promql/rollup_test.go

@@ -583,7 +583,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, 123, 34, 32}
+		valuesExpected := []float64{nan, nan, 123, 34, nan}
 		timestampsExpected := []int64{-50, 0, 50, 100, 150}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -645,7 +645,7 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
+		valuesExpected := []float64{12, nan, nan, nan, 34, 34, nan}
 		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -690,7 +690,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 54, 44, 34}
+		valuesExpected := []float64{nan, 123, 54, 44, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -704,7 +704,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, 1}
+		valuesExpected := []float64{nan, 4, 4, 3, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -718,7 +718,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 21, 12, 32, 34}
+		valuesExpected := []float64{nan, 21, 12, 32, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -732,7 +732,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 99, 44, 34}
+		valuesExpected := []float64{nan, 123, 99, 44, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -746,7 +746,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 222, 199, 110, 34}
+		valuesExpected := []float64{nan, 222, 199, 110, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -760,7 +760,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, -9, 22, 0}
+		valuesExpected := []float64{nan, nan, -9, 22, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -788,7 +788,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.004, 0, 0, 0.03}
+		valuesExpected := []float64{nan, 0.004, 0, 0, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -802,7 +802,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01}
+		valuesExpected := []float64{nan, 0.031, 0.044, 0.04, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -816,7 +816,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125}
+		valuesExpected := []float64{nan, 0.031, 0.075, 0.115, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -830,7 +830,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01}
+		valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -844,7 +844,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125}
+		valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -858,7 +858,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, 0}
+		valuesExpected := []float64{nan, 4, 4, 3, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -886,7 +886,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 2, 2, 1, 0}
+		valuesExpected := []float64{nan, 2, 2, 1, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -900,7 +900,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34}
+		valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -914,7 +914,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, 0}
+		valuesExpected := []float64{0, -2879.310344827587, 558.0608793686595, 422.84569138276544, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -942,7 +942,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0}
+		valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -956,7 +956,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 5.830951894845301}
+		valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -970,7 +970,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36}
+		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -984,7 +984,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 4, 3, 1}
+		valuesExpected := []float64{nan, 4, 4, 3, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -998,7 +998,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 4, 7, 6, 3}
+		valuesExpected := []float64{nan, 4, 7, 6, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1012,7 +1012,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, 34, 44, 44}
+		valuesExpected := []float64{nan, 21, 34, 34, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1026,7 +1026,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 1262.5, 3187.5, 4059.523809523809, 6200}
+		valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1040,7 +1040,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 0.9397878236968458, 1.1969836716333457, 2.3112921116373175, nan}
+		valuesExpected := []float64{nan, -0.86650328627136, -1.1200838283548589, -0.40035755084856683, nan}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -1062,7 +1062,7 @@ func TestRollupBigNumberOfValues(t *testing.T) {
 		srcTimestamps[i] = int64(i / 2)
 	}
 	values := rc.Do(nil, srcValues, srcTimestamps)
-	valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan}
+	valuesExpected := []float64{1, 4001, 8001, nan, nan, nan}
 	timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000}
 	testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 }
@@ -1103,11 +1103,13 @@ func testRowsEqual(t *testing.T, values []float64, timestamps []int64, valuesExp
 }
 
 func TestRollupDelta(t *testing.T) {
-	f := func(prevValue float64, values []float64, resultExpected float64) {
+	f := func(prevValue, realPrevValue, realNextValue float64, values []float64, resultExpected float64) {
 		t.Helper()
 		rfa := &rollupFuncArg{
-			prevValue: prevValue,
-			values:    values,
+			prevValue:     prevValue,
+			values:        values,
+			realPrevValue: realPrevValue,
+			realNextValue: realNextValue,
 		}
 		result := rollupDelta(rfa)
 		if math.IsNaN(result) {
@@ -1120,22 +1122,36 @@ func TestRollupDelta(t *testing.T) {
 			t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
 		}
 	}
-	f(nan, nil, nan)
+	f(nan, nan, nan, nil, nan)
 
 	// Small initial value
-	f(nan, []float64{1}, 1)
-	f(nan, []float64{10}, 10)
-	f(nan, []float64{100}, 100)
-	f(nan, []float64{1, 2, 3}, 3)
-	f(1, []float64{1, 2, 3}, 2)
-	f(nan, []float64{5, 6, 8}, 8)
-	f(2, []float64{5, 6, 8}, 6)
+	f(nan, nan, nan, []float64{1}, 1)
+	f(nan, nan, nan, []float64{10}, 10)
+	f(nan, nan, nan, []float64{100}, 100)
+	f(nan, nan, nan, []float64{1, 2, 3}, 3)
+	f(1, nan, nan, []float64{1, 2, 3}, 2)
+	f(nan, nan, nan, []float64{5, 6, 8}, 8)
+	f(2, nan, nan, []float64{5, 6, 8}, 6)
 
 	// Too big initial value must be skipped.
-	f(nan, []float64{1000}, 0)
-	f(nan, []float64{1000, 1001, 1002}, 2)
+	f(nan, nan, nan, []float64{1000}, 0)
+	f(nan, nan, nan, []float64{1000, 1001, 1002}, 2)
+
+	// Non-nan realPrevValue
+	f(nan, 900, nan, []float64{1000}, 100)
+	f(nan, 1000, nan, []float64{1000}, 0)
+	f(nan, 1100, nan, []float64{1000}, -100)
+	f(nan, 900, nan, []float64{1000, 1001, 1002}, 102)
+
+	// Small delta between realNextValue and values
+	f(nan, nan, 990, []float64{1000}, 0)
+	f(nan, nan, 1005, []float64{1000}, 0)
+
+	// Big delta between relaNextValue and values
+	f(nan, nan, 800, []float64{1000}, 1000)
+	f(nan, nan, 1300, []float64{1000}, 1000)
 
 	// Empty values
-	f(1, nil, 0)
-	f(100, nil, 0)
+	f(1, nan, nan, nil, 0)
+	f(100, nan, nan, nil, 0)
 }

app/vmselect/promql/transform.go

@@ -17,6 +17,27 @@ import (
 	"github.com/valyala/histogram"
 )
 
+var transformFuncsKeepMetricGroup = map[string]bool{
+	"ceil":               true,
+	"clamp_max":          true,
+	"clamp_min":          true,
+	"floor":              true,
+	"round":              true,
+	"keep_last_value":    true,
+	"keep_next_value":    true,
+	"interpolate":        true,
+	"running_min":        true,
+	"running_max":        true,
+	"running_avg":        true,
+	"range_min":          true,
+	"range_max":          true,
+	"range_avg":          true,
+	"range_first":        true,
+	"range_last":         true,
+	"range_quantile":     true,
+	"smooth_exponential": true,
+}
+
 var transformFuncs = map[string]transformFunc{
 	// Standard promql funcs
 	// See funcs accepting instant-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
@@ -125,8 +146,12 @@ func newTransformFuncOneArg(tf func(v float64) float64) transformFunc {
 }
 
 func doTransformValues(arg []*timeseries, tf func(values []float64), fe *metricsql.FuncExpr) ([]*timeseries, error) {
+	name := strings.ToLower(fe.Name)
+	keepMetricGroup := transformFuncsKeepMetricGroup[name]
 	for _, ts := range arg {
-		ts.MetricName.ResetMetricGroup()
+		if !keepMetricGroup {
+			ts.MetricName.ResetMetricGroup()
+		}
 		tf(ts.Values)
 	}
 	return arg, nil
@@ -142,23 +167,24 @@ func transformAbsent(tfa *transformFuncArg) ([]*timeseries, error) {
 	if err := expectTransformArgsNum(args, 1); err != nil {
 		return nil, err
 	}
-	arg := args[0]
-	if len(arg) == 0 {
-		rvs := getAbsentTimeseries(tfa.ec, tfa.fe.Args[0])
+	tss := args[0]
+	rvs := getAbsentTimeseries(tfa.ec, tfa.fe.Args[0])
+	if len(tss) == 0 {
 		return rvs, nil
 	}
-	for _, ts := range arg {
-		ts.MetricName.ResetMetricGroup()
-		for i, v := range ts.Values {
-			if !math.IsNaN(v) {
-				v = nan
-			} else {
-				v = 1
+	for i := range tss[0].Values {
+		isAbsent := true
+		for _, ts := range tss {
+			if !math.IsNaN(ts.Values[i]) {
+				isAbsent = false
+				break
 			}
-			ts.Values[i] = v
+		}
+		if !isAbsent {
+			rvs[0].Values[i] = nan
 		}
 	}
-	return arg, nil
+	return rvs, nil
 }
 
 func getAbsentTimeseries(ec *EvalConfig, arg metricsql.Expr) []*timeseries {
@@ -912,10 +938,9 @@ func newTransformFuncRunning(rf func(a, b float64, idx int) float64) transformFu
 			prevValue := values[0]
 			values = values[1:]
 			for i, v := range values {
-				if math.IsNaN(v) {
-					continue
+				if !math.IsNaN(v) {
+					prevValue = rf(prevValue, v, i+1)
 				}
-				prevValue = rf(prevValue, v, i+1)
 				values[i] = prevValue
 			}
 		}

app/vmselect/searchutils/searchutils.go

@@ -15,14 +15,22 @@ import (
 
 var (
 	maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
-	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for search query execution")
+	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
 )
 
+func roundToSeconds(ms int64) int64 {
+	return ms - ms%1000
+}
+
 // GetTime returns time from the given argKey query arg.
-func GetTime(r *http.Request, argKey string, defaultValue int64) (int64, error) {
+//
+// If argKey is missing in r, then defaultMs rounded to seconds is returned.
+// The rounding is needed in order to align query results in Grafana
+// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720
+func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
 	argValue := r.FormValue(argKey)
 	if len(argValue) == 0 {
-		return defaultValue, nil
+		return roundToSeconds(defaultMs), nil
 	}
 	secs, err := strconv.ParseFloat(argValue, 64)
 	if err != nil {
@@ -96,6 +104,19 @@ func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, err
 
 const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000
 
+// GetMaxQueryDuration returns the maximum duration for query from r.
+func GetMaxQueryDuration(r *http.Request) time.Duration {
+	dms, err := GetDuration(r, "timeout", 0)
+	if err != nil {
+		dms = 0
+	}
+	d := time.Duration(dms) * time.Millisecond
+	if d <= 0 || d > *maxQueryDuration {
+		d = *maxQueryDuration
+	}
+	return d
+}
+
 // GetDeadlineForQuery returns deadline for the given query r.
 func GetDeadlineForQuery(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxQueryDuration.Milliseconds()

app/vmselect/searchutils/searchutils_test.go

@@ -17,12 +17,12 @@ func TestGetTimeSuccess(t *testing.T) {
 		}
 
 		// Verify defaultValue
-		ts, err := GetTime(r, "foo", 123)
+		ts, err := GetTime(r, "foo", 123456)
 		if err != nil {
 			t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
 		}
-		if ts != 123 {
-			t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123)
+		if ts != 123000 {
+			t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
 		}
 
 		// Verify timestampExpected
@@ -55,12 +55,12 @@ func TestGetTimeError(t *testing.T) {
 		}
 
 		// Verify defaultValue
-		ts, err := GetTime(r, "foo", 123)
+		ts, err := GetTime(r, "foo", 123456)
 		if err != nil {
 			t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
 		}
-		if ts != 123 {
-			t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123)
+		if ts != 123000 {
+			t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
 		}
 
 		// Verify timestampExpected

app/vmstorage/main.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -19,14 +20,19 @@ import (
 )
 
 var (
-	retentionPeriod = flag.Int("retentionPeriod", 1, "Retention period in months")
-	snapshotAuthKey = flag.String("snapshotAuthKey", "", "authKey, which must be passed in query string to /snapshot* pages")
+	retentionPeriod   = flagutil.NewDuration("retentionPeriod", 1, "Data with timestamps outside the retentionPeriod is automatically deleted")
+	snapshotAuthKey   = flag.String("snapshotAuthKey", "", "authKey, which must be passed in query string to /snapshot* pages")
+	forceMergeAuthKey = flag.String("forceMergeAuthKey", "", "authKey, which must be passed in query string to /internal/force_merge pages")
+	forceFlushAuthKey = flag.String("forceFlushAuthKey", "", "authKey, which must be passed in query string to /internal/force_flush pages")
 
 	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss")
 
 	// DataPath is a path to storage data.
 	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 
+	finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
+		"Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+
+		"disk IO usage and CPU usage")
 	bigMergeConcurrency   = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
 	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
 
@@ -40,12 +46,12 @@ func CheckTimeRange(tr storage.TimeRange) error {
 	if !*denyQueriesOutsideRetention {
 		return nil
 	}
-	minAllowedTimestamp := (int64(fasttime.UnixTimestamp()) - int64(*retentionPeriod)*3600*24*30) * 1000
+	minAllowedTimestamp := int64(fasttime.UnixTimestamp()*1000) - retentionPeriod.Msecs
 	if tr.MinTimestamp > minAllowedTimestamp {
 		return nil
 	}
 	return &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("the given time range %s is outside the allowed retention of %d months according to -denyQueriesOutsideRetention", &tr, *retentionPeriod),
+		Err:        fmt.Errorf("the given time range %s is outside the allowed -retentionPeriod=%s according to -denyQueriesOutsideRetention", &tr, retentionPeriod),
 		StatusCode: http.StatusServiceUnavailable,
 	}
 }
@@ -64,15 +70,16 @@ func InitWithoutMetrics() {
 		logger.Fatalf("invalid `-precisionBits`: %s", err)
 	}
 
+	storage.SetFinalMergeDelay(*finalMergeDelay)
 	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
 	storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
 
-	logger.Infof("opening storage at %q with retention period %d months", *DataPath, *retentionPeriod)
+	logger.Infof("opening storage at %q with -retentionPeriod=%s", *DataPath, retentionPeriod)
 	startTime := time.Now()
 	WG = syncwg.WaitGroup{}
-	strg, err := storage.OpenStorage(*DataPath, *retentionPeriod)
+	strg, err := storage.OpenStorage(*DataPath, retentionPeriod.Msecs)
 	if err != nil {
-		logger.Fatalf("cannot open a storage at %s with retention period %d months: %s", *DataPath, *retentionPeriod, err)
+		logger.Fatalf("cannot open a storage at %s with -retentionPeriod=%s: %s", *DataPath, retentionPeriod, err)
 	}
 	Storage = strg
 
@@ -106,6 +113,14 @@ func AddRows(mrs []storage.MetricRow) error {
 	return err
 }
 
+// RegisterMetricNames registers all the metrics from mrs in the storage.
+func RegisterMetricNames(mrs []storage.MetricRow) error {
+	WG.Add(1)
+	err := Storage.RegisterMetricNames(mrs)
+	WG.Done()
+	return err
+}
+
 // DeleteMetrics deletes metrics matching tfss.
 //
 // Returns the number of deleted metrics.
@@ -116,6 +131,22 @@ func DeleteMetrics(tfss []*storage.TagFilters) (int, error) {
 	return n, err
 }
 
+// SearchMetricNames returns metric names for the given tfss on the given tr.
+func SearchMetricNames(tfss []*storage.TagFilters, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]storage.MetricName, error) {
+	WG.Add(1)
+	mns, err := Storage.SearchMetricNames(tfss, tr, maxMetrics, deadline)
+	WG.Done()
+	return mns, err
+}
+
+// SearchTagKeysOnTimeRange searches for tag keys on tr.
+func SearchTagKeysOnTimeRange(tr storage.TimeRange, maxTagKeys int, deadline uint64) ([]string, error) {
+	WG.Add(1)
+	keys, err := Storage.SearchTagKeysOnTimeRange(tr, maxTagKeys, deadline)
+	WG.Done()
+	return keys, err
+}
+
 // SearchTagKeys searches for tag keys
 func SearchTagKeys(maxTagKeys int, deadline uint64) ([]string, error) {
 	WG.Add(1)
@@ -124,6 +155,14 @@ func SearchTagKeys(maxTagKeys int, deadline uint64) ([]string, error) {
 	return keys, err
 }
 
+// SearchTagValuesOnTimeRange searches for tag values for the given tagKey on tr.
+func SearchTagValuesOnTimeRange(tagKey []byte, tr storage.TimeRange, maxTagValues int, deadline uint64) ([]string, error) {
+	WG.Add(1)
+	values, err := Storage.SearchTagValuesOnTimeRange(tagKey, tr, maxTagValues, deadline)
+	WG.Done()
+	return values, err
+}
+
 // SearchTagValues searches for tag values for the given tagKey
 func SearchTagValues(tagKey []byte, maxTagValues int, deadline uint64) ([]string, error) {
 	WG.Add(1)
@@ -180,6 +219,36 @@ func Stop() {
 // RequestHandler is a storage request handler.
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
+	if path == "/internal/force_merge" {
+		authKey := r.FormValue("authKey")
+		if authKey != *forceMergeAuthKey {
+			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -forceMergeAuthKey command line flag", authKey)
+			return true
+		}
+		// Run force merge in background
+		partitionNamePrefix := r.FormValue("partition_prefix")
+		go func() {
+			activeForceMerges.Inc()
+			defer activeForceMerges.Dec()
+			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
+			startTime := time.Now()
+			if err := Storage.ForceMergePartitions(partitionNamePrefix); err != nil {
+				logger.Errorf("error in forced merge for partition_prefix=%q: %s", partitionNamePrefix, err)
+			}
+			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
+		}()
+		return true
+	}
+	if path == "/internal/force_flush" {
+		authKey := r.FormValue("authKey")
+		if authKey != *forceFlushAuthKey {
+			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -forceFlushAuthKey command line flag", authKey)
+			return true
+		}
+		logger.Infof("flushing storage to make pending data available for reading")
+		Storage.DebugFlush()
+		return true
+	}
 	prometheusCompatibleResponse := false
 	if path == "/api/v1/admin/tsdb/snapshot" {
 		// Handle Prometheus API - https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot .
@@ -198,7 +267,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	switch path {
 	case "/create":
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		snapshotPath, err := Storage.CreateSnapshot()
 		if err != nil {
 			err = fmt.Errorf("cannot create snapshot: %w", err)
@@ -212,7 +281,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/list":
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		snapshots, err := Storage.ListSnapshots()
 		if err != nil {
 			err = fmt.Errorf("cannot list snapshots: %w", err)
@@ -229,7 +298,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `]}`)
 		return true
 	case "/delete":
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		snapshotName := r.FormValue("snapshot")
 		if err := Storage.DeleteSnapshot(snapshotName); err != nil {
 			err = fmt.Errorf("cannot delete snapshot %q: %w", snapshotName, err)
@@ -239,7 +308,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/delete_all":
-		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		snapshots, err := Storage.ListSnapshots()
 		if err != nil {
 			err = fmt.Errorf("cannot list snapshots: %w", err)
@@ -260,6 +329,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
+var activeForceMerges = metrics.NewCounter("vm_active_force_merges")
+
 func registerStorageMetrics() {
 	mCache := &storage.Metrics{}
 	var mCacheLock sync.Mutex
@@ -349,12 +420,6 @@ func registerStorageMetrics() {
 	metrics.NewGauge(`vm_missing_tsids_for_metric_id_total`, func() float64 {
 		return float64(idbm().MissingTSIDsForMetricID)
 	})
-	metrics.NewGauge(`vm_recent_hour_metric_ids_search_calls_total`, func() float64 {
-		return float64(idbm().RecentHourMetricIDsSearchCalls)
-	})
-	metrics.NewGauge(`vm_recent_hour_metric_ids_search_hits_total`, func() float64 {
-		return float64(idbm().RecentHourMetricIDsSearchHits)
-	})
 	metrics.NewGauge(`vm_date_metric_ids_search_calls_total`, func() float64 {
 		return float64(idbm().DateMetricIDsSearchCalls)
 	})
@@ -375,6 +440,14 @@ func registerStorageMetrics() {
 		return float64(idbm().AssistedMerges)
 	})
 
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/686
+	metrics.NewGauge(`vm_merge_need_free_disk_space{type="storage/small"}`, func() float64 {
+		return float64(tm().SmallMergeNeedFreeDiskSpace)
+	})
+	metrics.NewGauge(`vm_merge_need_free_disk_space{type="storage/big"}`, func() float64 {
+		return float64(tm().BigMergeNeedFreeDiskSpace)
+	})
+
 	metrics.NewGauge(`vm_pending_rows{type="storage"}`, func() float64 {
 		return float64(tm().PendingRows)
 	})
@@ -412,6 +485,9 @@ func registerStorageMetrics() {
 		return float64(idbm().SizeBytes)
 	})
 
+	metrics.NewGauge(`vm_rows_added_to_storage_total`, func() float64 {
+		return float64(m().RowsAddedTotal)
+	})
 	metrics.NewGauge(`vm_deduplicated_samples_total{type="merge"}`, func() float64 {
 		return float64(m().DedupsDuringMerge)
 	})

dashboards/victoriametrics.json

@@ -56,7 +56,7 @@
   "gnetId": 10229,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1599034965731,
+  "iteration": 1603307754894,
   "links": [
     {
       "icon": "doc",
@@ -925,7 +925,7 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$ds",
-      "description": "Shows how many ongoing insertions are taking place.\n* `max` - equal to number of CPU * 2\n* `current` - current number of goroutines busy with inserting rows into storage\n\nWhen `current` hits `max` constantly, it means storage is overloaded and require more CPU.",
+      "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. However, these two actions are separated and controlled via different concurrency limiters. The `max` on this panel can't be changed and always equal to number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.\n\n",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -979,6 +979,7 @@
         {
           "expr": "sum(vm_concurrent_addrows_capacity{job=\"$job\", instance=\"$instance\"})",
           "format": "time_series",
+          "interval": "",
           "intervalFactor": 1,
           "legendFormat": "max",
           "refId": "A"
@@ -995,7 +996,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Concurrent inserts ($instance)",
+      "title": "Concurrent flushes on disk ($instance)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -1164,7 +1165,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 36
+            "y": 3
           },
           "hiddenSeries": false,
           "id": 10,
@@ -1250,7 +1251,7 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
-          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second.",
+          "description": "Shows the time needed to reach the 100% of disk capacity based on the following params:\n* free disk space;\n* rows ingestion rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.\n\n",
           "fieldConfig": {
             "defaults": {
               "custom": {},
@@ -1264,63 +1265,53 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 36
+            "y": 3
           },
           "hiddenSeries": false,
-          "id": 34,
+          "id": 73,
           "legend": {
-            "avg": false,
-            "current": false,
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "hideZero": true,
             "max": false,
             "min": false,
             "show": false,
             "total": false,
-            "values": false
+            "values": true
           },
           "lines": true,
           "linewidth": 1,
           "links": [],
-          "nullPointMode": "null",
+          "nullPointMode": "null as zero",
           "percentage": false,
           "pluginVersion": "7.1.1",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [
-            {
-              "alias": "pending index entries",
-              "yaxis": 2
-            }
-          ],
+          "seriesOverrides": [],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "vm_pending_rows{job=\"$job\", instance=~\"$instance\", type=\"storage\"}",
+              "expr": "vm_free_disk_space_bytes{job=\"$job\", instance=\"$instance\"} / (sum(rate(vm_rows_added_to_storage_total{job=\"$job\", instance=\"$instance\"}[1d])) * (sum(vm_data_size_bytes{job=\"$job\", instance=\"$instance\", type!=\"indexdb\"}) / sum(vm_rows{job=\"$job\", instance=\"$instance\", type!=\"indexdb\"})))",
               "format": "time_series",
               "hide": false,
+              "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "pending datapoints",
+              "legendFormat": "",
               "refId": "A"
-            },
-            {
-              "expr": "vm_pending_rows{job=\"$job\", instance=~\"$instance\", type=\"indexdb\"}",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "pending index entries",
-              "refId": "B"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Pending datapoints ($instance)",
+          "title": "Storage full ETA ($instance)",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -1333,7 +1324,8 @@
           },
           "yaxes": [
             {
-              "format": "short",
+              "decimals": null,
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1341,8 +1333,7 @@
               "show": true
             },
             {
-              "decimals": 3,
-              "format": "none",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1375,7 +1366,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 44
+            "y": 11
           },
           "hiddenSeries": false,
           "id": 30,
@@ -1472,7 +1463,7 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$ds",
-          "description": "Data parts of LSM tree.\nHigh number of parts could be an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage(hot data)\n* `storage/big` -  small parts gradually merged into big parts (cold data)",
+          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second.",
           "fieldConfig": {
             "defaults": {
               "custom": {},
@@ -1486,16 +1477,16 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 44
+            "y": 11
           },
           "hiddenSeries": false,
-          "id": 36,
+          "id": 34,
           "legend": {
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
@@ -1508,27 +1499,41 @@
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "pending index entries",
+              "yaxis": 2
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(vm_parts{job=\"$job\", instance=\"$instance\"}) by (type)",
+              "expr": "vm_pending_rows{job=\"$job\", instance=~\"$instance\", type=\"storage\"}",
               "format": "time_series",
+              "hide": false,
               "intervalFactor": 1,
-              "legendFormat": "{{type}}",
+              "legendFormat": "pending datapoints",
               "refId": "A"
+            },
+            {
+              "expr": "vm_pending_rows{job=\"$job\", instance=~\"$instance\", type=\"indexdb\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "pending index entries",
+              "refId": "B"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "LSM parts ($instance)",
+          "title": "Pending datapoints ($instance)",
           "tooltip": {
             "shared": true,
-            "sort": 2,
+            "sort": 0,
             "value_type": "individual"
           },
           "type": "graph",
@@ -1549,7 +1554,8 @@
               "show": true
             },
             {
-              "format": "short",
+              "decimals": 3,
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1582,7 +1588,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 19
           },
           "hiddenSeries": false,
           "id": 53,
@@ -1669,6 +1675,196 @@
             "alignLevel": null
           }
         },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "description": "Data parts of LSM tree.\nHigh number of parts could be an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage(hot data)\n* `storage/big` -  small parts gradually merged into big parts (cold data)",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 19
+          },
+          "hiddenSeries": false,
+          "id": 36,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pluginVersion": "7.1.1",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(vm_parts{job=\"$job\", instance=\"$instance\"}) by (type)",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{type}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "LSM parts ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "description": "The number of on-going merges in storage nodes.  It is expected to have high numbers for `storage/small` metric.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "hiddenSeries": false,
+          "id": 62,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "percentage": false,
+          "pluginVersion": "7.1.1",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(vm_active_merges{job=\"$job\", instance=\"$instance\"}) by(type)",
+              "legendFormat": "{{type}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Active merges ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": 0,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -1689,7 +1885,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 27
           },
           "hiddenSeries": false,
           "id": 55,
@@ -1764,194 +1960,6 @@
             "alignLevel": null
           }
         },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": "$ds",
-          "description": "The number of on-going merges in storage nodes.  It is expected to have high numbers for `storage/small` metric.",
-          "fieldConfig": {
-            "defaults": {
-              "custom": {},
-              "links": []
-            },
-            "overrides": []
-          },
-          "fill": 1,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 0,
-            "y": 60
-          },
-          "hiddenSeries": false,
-          "id": 62,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "nullPointMode": "null",
-          "percentage": false,
-          "pluginVersion": "7.1.1",
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "sum(vm_active_merges{job=\"$job\", instance=\"$instance\"}) by(type)",
-              "legendFormat": "{{type}}",
-              "refId": "A"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Active merges ($instance)",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": 0,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": "$ds",
-          "description": "The number of rows merged per second by storage nodes.",
-          "fieldConfig": {
-            "defaults": {
-              "custom": {},
-              "links": []
-            },
-            "overrides": []
-          },
-          "fill": 1,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 12,
-            "y": 60
-          },
-          "hiddenSeries": false,
-          "id": 64,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "nullPointMode": "null",
-          "percentage": false,
-          "pluginVersion": "7.1.1",
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "sum(rate(vm_rows_merged_total{job=\"$job\", instance=\"$instance\"}[5m])) by(type)",
-              "legendFormat": "{{type}}",
-              "refId": "A"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Merge speed ($instance)",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": 0,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
-        },
         {
           "aliasColors": {},
           "bars": false,
@@ -1972,7 +1980,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 68
+            "y": 35
           },
           "hiddenSeries": false,
           "id": 58,
@@ -2050,6 +2058,100 @@
             "alignLevel": null
           }
         },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$ds",
+          "description": "The number of rows merged per second by storage nodes.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 35
+          },
+          "hiddenSeries": false,
+          "id": 64,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "percentage": false,
+          "pluginVersion": "7.1.1",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(vm_rows_merged_total{job=\"$job\", instance=\"$instance\"}[5m])) by(type)",
+              "legendFormat": "{{type}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Merge speed ($instance)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": 0,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -2070,7 +2172,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 68
+            "y": 43
           },
           "hiddenSeries": false,
           "id": 67,
@@ -3293,4 +3395,4 @@
   "title": "VictoriaMetrics",
   "uid": "wNf0q_kZk",
   "version": 1
-}
+}

deployment/docker/Makefile

@@ -2,9 +2,9 @@
 
 DOCKER_NAMESPACE := victoriametrics
 
-ROOT_IMAGE ?= alpine:3.12
-CERTS_IMAGE := alpine:3.12
-GO_BUILDER_IMAGE := golang:1.15.2
+ROOT_IMAGE ?= alpine:3.12.1
+CERTS_IMAGE := alpine:3.12.1
+GO_BUILDER_IMAGE := golang:1.15.5
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr : _)
 BASE_IMAGE := local/base:1.1.1-$(shell echo $(ROOT_IMAGE) | tr : _)-$(shell echo $(CERTS_IMAGE) | tr : _)
 

deployment/docker/alerts.yml

@@ -0,0 +1,23 @@
+groups:
+  - name: groupGorSingleAlert
+    rules:
+      - alert: VMRows
+        for: 10s
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+  - name: TestGroup
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        for: 5s
+        annotations:
+          summary: "Too high connection number for {{$labels.instance}}"
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job)
+          (up == 1)

deployment/docker/docker-compose.yml

@@ -21,7 +21,10 @@ services:
     image: victoriametrics/victoria-metrics
     ports:
       - 8428:8428
+      - 8089:8089
+      - 8089:8089/udp
       - 2003:2003
+      - 2003:2003/udp
       - 4242:4242
     volumes:
       - vmdata:/storage
@@ -30,6 +33,7 @@ services:
       - '--graphiteListenAddr=:2003'
       - '--opentsdbListenAddr=:4242'
       - '--httpListenAddr=:8428'
+      - '--influxListenAddr=:8089'
     networks:
       - vm_net
     restart: always
@@ -48,6 +52,36 @@ services:
     networks:
       - vm_net
     restart: always
+  vmalert:
+    container_name: vmalert
+    image: victoriametrics/vmalert
+    depends_on:
+      - "victoriametrics"
+      - "alertmanager"
+    ports:
+      - 8880:8880
+    volumes:
+      - ./alerts.yml:/etc/alerts/alerts.yml
+    command:
+      - '--datasource.url=http://victoriametrics:8428/'
+      - '--remoteRead.url=http://victoriametrics:8428/'
+      - '--remoteWrite.url=http://victoriametrics:8428/'
+      - '--notifier.url=http://alertmanager:9093/'
+      - '--rule=/etc/alerts/*.yml'
+      # display source of alerts in grafana
+      - '-external.url=http://127.0.0.1:3000' #grafana outside container
+      - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":"{{$$expr|quotesEscape|pathEscape}}"},{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' ## when copypaste the line be aware of '$$' for escaping in '$expr'
+    networks:
+      - vm_net
+    restart: always
+  alertmanager:
+    container_name: alertmanager
+    image:  prom/alertmanager
+    ports:
+      - 9093:9093
+    networks:
+      - vm_net
+    restart: always
 volumes:
   vmagentdata: {}
   vmdata: {}

docs/Articles.md

@@ -1,5 +1,26 @@
 # Articles
 
+## Third-party articles and slides about VictoriaMetrics
+
+* [Better Prometheus rate() function with VictoriaMetrics](https://www.percona.com/blog/2020/02/28/better-prometheus-rate-function-with-victoriametrics/)
+* [Making peace with Prometheus rate()](https://blog.doit-intl.com/making-peace-with-prometheus-rate-43a3ea75c4cf)
+* [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
+* [Sismology: Iguana Solutions’ Monitoring System](https://medium.com/@IG1.com/sismology-iguana-solutions-monitoring-system-f46e4170447f)
+* [Prometheus High Availability and Fault Tolerance strategy, long term storage with VictoriaMetrics](https://medium.com/miro-engineering/prometheus-high-availability-and-fault-tolerance-strategy-long-term-storage-with-victoriametrics-82f6f3f0409e)
+* [How we improved our Kubernetes monitoring at Smarkets, and how you could too](https://smarketshq.com/monitoring-kubernetes-clusters-41a4b24c19e3)
+* [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit)
+* [CMS monitoring R&D: Real-time monitoring and alerts](https://indico.cern.ch/event/877333/contributions/3696707/attachments/1972189/3281133/CMS_mon_RD_for_opInt.pdf)
+* [The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf)
+* [Disk usage: VictoriaMetrics vs Prometheus](https://stas.starikevich.com/posts/disk-usage-for-vm-versus-prometheus/)
+* [Benchmarking time series workloads on Apache Kudu using TSBS](https://blog.cloudera.com/benchmarking-time-series-workloads-on-apache-kudu-using-tsbs/)
+* [What are Open Source Time Series Databases?](https://www.iunera.com/kraken/fabric/time-series-database/)
+* [Evaluating performance and correctness](https://www.robustperception.io/evaluating-performance-and-correctness)
+* [Running VictoriaMetrics on Raspberry PI](https://stas.starikevich.com/posts/raspberry-pi-4-prometheus/)
+* [Calculating the Error of Quantile Estimation with Histograms](https://linuxczar.net/blog/2020/08/13/histogram-error/)
+* [Monitoring private clouds with VictoriaMetrics at LeroyMerlin](https://www.youtube.com/watch?v=74swsWqf0Uc)
+* [Monitoring Kubernetes with VictoriaMetrics+Prometheus](https://speakerdeck.com/bo0km4n/victoriametrics-plus-prometheusdegou-zhu-surufu-shu-kubernetesfalsejian-shi-ji-pan)
+* [High-performance Graphite storage solution on top of VictoriaMetrics](https://golangexample.com/a-high-performance-graphite-storage-solution/)
+
 
 ## Our articles
 
@@ -26,22 +47,6 @@
 * [Billy: how VictoriaMetrics deals with more than 500 billion rows](https://medium.com/@valyala/billy-how-victoriametrics-deals-with-more-than-500-billion-rows-e82ff8f725da)
 * [How to migrate data from Prometheus to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
 * [Filtering and modifying time series during import to VictoriaMetrics](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
-
-
-## Third-party articles and slides
-
-* [Better Prometheus rate() function with VictoriaMetrics](https://www.percona.com/blog/2020/02/28/better-prometheus-rate-function-with-victoriametrics/)
-* [Making peace with Prometheus rate()](https://blog.doit-intl.com/making-peace-with-prometheus-rate-43a3ea75c4cf)
-* [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
-* [Sismology: Iguana Solutions’ Monitoring System](https://medium.com/@IG1.com/sismology-iguana-solutions-monitoring-system-f46e4170447f)
-* [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit)
-* [CMS monitoring R&D: Real-time monitoring and alerts](https://indico.cern.ch/event/877333/contributions/3696707/attachments/1972189/3281133/CMS_mon_RD_for_opInt.pdf)
-* [The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf)
-* [Disk usage: VictoriaMetrics vs Prometheus](https://stas.starikevich.com/posts/disk-usage-for-vm-versus-prometheus/)
-* [Benchmarking time series workloads on Apache Kudu using TSBS](https://blog.cloudera.com/benchmarking-time-series-workloads-on-apache-kudu-using-tsbs/)
-* [What are Open Source Time Series Databases?](https://www.iunera.com/kraken/fabric/time-series-database/)
-* [Evaluating performance and correctness](https://www.robustperception.io/evaluating-performance-and-correctness)
-* [Running VictoriaMetrics on Raspberry PI](https://stas.starikevich.com/posts/raspberry-pi-4-prometheus/)
-* [Calculating the Error of Quantile Estimation with Histograms](https://linuxczar.net/blog/2020/08/13/histogram-error/)
-* [Monitoring private clouds with VictoriaMetrics at LeroyMerlin](https://www.youtube.com/watch?v=74swsWqf0Uc)
-* [Monitoring Kubernetes with VictoriaMetrics+Prometheus](https://speakerdeck.com/bo0km4n/victoriametrics-plus-prometheusdegou-zhu-surufu-shu-kubernetesfalsejian-shi-ji-pan)
+* [Anomaly Detection in VictoriaMetrics](https://medium.com/@VictoriaMetrics/anomaly-detection-in-victoriametrics-9528538786a7)
+* [How to use relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
+* [First look at performance comparison between InfluxDB IOx and VictoriaMetrics](https://medium.com/@VictoriaMetrics/first-look-at-perfomance-comparassion-between-influxdb-iox-and-victoriametrics-e590f847935b)

docs/CHANGELOG.md

@@ -0,0 +1,225 @@
+# CHANGELOG
+
+# tip
+
+
+# [v1.48.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.48.0)
+
+* FEATURE: added [Snap package for single-node VictoriaMetrics](https://snapcraft.io/victoriametrics). This simplifies installation under Ubuntu to a single command:
+  ```bash
+  snap install victoriametrics
+  ```
+* FEATURE: vmselect: add `-replicationFactor` command-line flag for reducing query duration when replication is enabled and a part of vmstorage nodes
+  are temporarily slow and/or temporarily unavailable. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711
+* FEATURE: vminsert: export `vm_rpc_vmstorage_is_reachable` metric, which can be used for monitoring reachability of vmstorage nodes from vminsert nodes.
+* FEATURE: vmagent: add [Netflix Eureka](https://github.com/Netflix/eureka) service discovery (aka [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/851
+* FEATURE: add `filters` option to `dockerswarm_sd_config` like Prometheus did in v2.23.0 - see https://github.com/prometheus/prometheus/pull/8074
+* FEATURE: expose `__meta_ec2_ipv6_addresses` label for `ec2_sd_config` like Prometheus will do in the next release.
+* FEATURE: add `-loggerWarnsPerSecondLimit` command-line flag for rate limiting of WARN messages in logs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905
+* FEATURE: apply `loggerErrorsPerSecondLimit` and `-loggerWarnsPerSecondLimit` rate limit per caller. I.e. log messages are suppressed if the same caller logs the same message
+  at the rate exceeding the given limit. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905#issuecomment-729395855
+* FEATURE: add remoteAddr to slow query log in order to simplify identifying the client that sends slow queries to VictoriaMetrics.
+  Slow query logging is controlled with `-search.logSlowQueryDuration` command-line flag.
+* FEATURE: add `/tags/delSeries` handler from Graphite Tags API. See https://victoriametrics.github.io/#graphite-tags-api-usage
+* FEATURE: log metric name plus all its labels when the metric timestamp is out of the configured retention. This should simplify detecting the source of metrics with unexpected timestamps.
+* FEATURE: add `-dryRun` command-line flag to single-node VictoriaMetrics in order to check config file pointed by `-promscrape.config`.
+
+* BUGFIX: properly parse Prometheus metrics with [exemplars](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#exemplars-1) such as `foo 123 # {bar="baz"} 1`.
+* BUGFIX: properly parse "infinity" values in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#abnf).
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/924
+
+
+# [v1.47.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.47.0)
+
+* FEATURE: vmselect: return the original error from `vmstorage` node in query response if `-search.denyPartialResponse` is set.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/891
+* FEATURE: vmselect: add `"isPartial":{true|false}` field in JSON output for `/api/v1/*` functions
+  from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/). `"isPartial":true` is set if the response contains partial data
+  because of a part of `vmstorage` nodes were unavailable during query processing.
+* FEATURE: improve performance for `/api/v1/series`, `/api/v1/labels` and `/api/v1/label/<labelName>/values` on time ranges exceeding one day.
+* FEATURE: vmagent: reduce memory usage when service discovery detects big number of scrape targets and the set of discovered targets changes over time.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825
+* FEATURE: vmagent: add `-promscrape.dropOriginalLabels` command-line option, which can be used for reducing memory usage when scraping big number of targets.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825#issuecomment-724308361 for details.
+* FEATURE: vmalert: explicitly set extra labels to alert entities. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/870
+* FEATURE: add `-search.treatDotsAsIsInRegexps` command-line flag, which can be used for automatic escaping of dots in regexp label filters used in queries.
+  For example, if `-search.treatDotsAsIsInRegexps` is set, then the query `foo{bar=~"aaa.bb.cc|dd.eee"}` is automatically converted to `foo{bar=~"aaa\\.bb\\.cc|dd\\.eee"}`.
+  This may be useful for querying Graphite data.
+* FEATURE: consistently return text-based HTTP responses such as `plain/text` and `application/json` with `charset=utf-8`.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/897
+* FEATURE: update Go builder from v1.15.4 to v1.15.5. This should fix [these issues in Go](https://github.com/golang/go/issues?q=milestone%3AGo1.15.5+label%3ACherryPickApproved).
+* FEATURE: added `/internal/force_flush` http handler for flushing recently ingested data from in-memory buffers to persistent storage.
+  See [troubleshooting docs](https://victoriametrics.github.io/#troubleshooting) for more details.
+* FEATURE: added [Graphite Tags API](https://graphite.readthedocs.io/en/stable/tags.html) support.
+  See [these docs](https://victoriametrics.github.io/#graphite-tags-api-usage) for details.
+
+* BUGFIX: do not return data points in the end of the selected time range for time series ending in the middle of the selected time range.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/887 and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
+* BUGFIX: remove spikes at the end of time series gaps for `increase()` or `delta()` functions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/894
+* BUGFIX: vminsert: properly return HTTP 503 status code when all the vmstorage nodes are unavailable. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/896
+
+
+# [v1.46.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.46.0)
+
+* FEATURE: optimize requests to `/api/v1/labels` and `/api/v1/label/<name>/values` when `start` and `end` args are set.
+* FEATURE: reduce memory usage when query touches big number of time series.
+* FEATURE: vmagent: reduce memory usage when `kubernetes_sd_config` discovers big number of scrape targets (e.g. hundreds of thouthands) and the majority of these targets (99%)
+  are dropped during relabeling. Previously labels for all the dropped targets were displayed at `/api/v1/targets` page. Now only up to `-promscrape.maxDroppedTargets` such
+  targets are displayed. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/878 for details.
+* FEATURE: vmagent: reduce memory usage when scraping big number of targets with big number of temporary labels starting with `__`.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825
+* FEATURE: vmagent: add `/ready` HTTP endpoint, which returns 200 OK status code when all the service discovery has been initialized.
+  This may be useful during rolling upgrades. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/875
+
+* BUGFIX: vmagent: eliminate data race when `-promscrape.streamParse` command-line is set. Previously this mode could result in scraped metrics with garbage labels.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825#issuecomment-723198247 for details.
+* BUGFIX: properly calculate `topk_*` and `bottomk_*` functions from [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) for time series with gaps.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/883
+
+
+# [v1.45.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.45.0)
+
+* FEATURE: allow setting `-retentionPeriod` smaller than one month. I.e. `-retentionPeriod=3d`, `-retentionPeriod=2w`, etc. is supported now.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/173
+* FEATURE: optimize more cases according to https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization . Now the following cases are optimized too:
+  * `rollup_func(foo{filters}[d]) op bar` -> `rollup_func(foo{filters}[d]) op bar{filters}`
+  * `transform_func(foo{filters}) op bar` -> `transform_func(foo{filters}) op bar{filters}`
+  * `num_or_scalar op foo{filters} op bar` -> `num_or_scalar op foo{filters} op bar{filters}`
+* FEATURE: improve time series search for queries with multiple label filters. I.e. `foo{label1="value", label2=~"regexp"}`.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/781
+* FEATURE: vmagent: add `stream parse` mode. This mode allows reducing memory usage when individual scrape targets expose tens of millions of metrics.
+  For example, during scraping Prometheus in [federation](https://prometheus.io/docs/prometheus/latest/federation/) mode.
+  See `-promscrape.streamParse` command-line option and `stream_parse: true` config option for `scrape_config` section in `-promscrape.config`.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825 and [troubleshooting docs for vmagent](https://victoriametrics.github.io/vmagent.html#troubleshooting).
+* FEATURE: vmalert: add `-dryRun` command-line option for validating the provided config files without the need to start `vmalert` service.
+* FEATURE: accept optional third argument of string type at `topk_*` and `bottomk_*` functions. This is label name for additional time series to return with the sum of time series outside top/bottom K. See [MetricsQL docs](https://victoriametrics.github.io/MetricsQL.html) for more details.
+* FEATURE: vmagent: expose `/api/v1/targets` page according to [the corresponding Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/#targets).
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/643
+
+* BUGFIX: vmagent: properly handle OpenStack endpoint ending with `v3.0` such as `https://ostack.example.com:5000/v3.0`
+  in the same way as Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/728#issuecomment-709914803
+* BUGFIX: drop trailing data points for time series with a single raw sample. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
+* BUGFIX: do not drop trailing data points for instant queries to `/api/v1/query`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845
+* BUGFIX: vmbackup: fix panic when `-origin` isn't specified. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/856
+* BUGFIX: vmalert: skip automatically added labels on alerts restore. Label `alertgroup` was introduced in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/611)
+  and automatically added to generated time series. By mistake, this new label wasn't correctly purged on restore event and affected alert's ID uniqueness.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/870
+* BUGFIX: vmagent: fix panic at scrape error body formating. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/864
+* BUGFIX: vmagent: add leading missing slash to metrics path like Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/835
+* BUGFIX: vmagent: drop packet if remote storage returns 4xx status code. This make the behaviour consistent with Prometheus.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
+* BUGFIX: vmagent: properly handle 301 redirects. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/869
+
+
+# [v1.44.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.44.0)
+
+* FEATURE: automatically add missing label filters to binary operands as described at https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization .
+  This should improve performance for queries with missing label filters in binary operands. For example, the following query should work faster now, because it shouldn't
+  fetch and discard time series for `node_filesystem_files_free` metric without matching labels for the left side of the expression:
+  ```
+     node_filesystem_files{ host="$host", mountpoint="/" } - node_filesystem_files_free
+  ```
+* FEATURE: vmagent: add Docker Swarm service discovery (aka [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)).
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/656
+* FEATURE: add ability to export data in CSV format. See [these docs](https://victoriametrics.github.io/#how-to-export-csv-data) for details.
+* FEATURE: vmagent: add `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag for suppressing `duplicate scrape target` errors.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651 and https://victoriametrics.github.io/vmagent.html#troubleshooting .
+* FEATURE: vmagent: show original labels before relabeling is applied on `duplicate scrape target` errors. This should simplify debugging for incorrect relabeling.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651
+* FEATURE: vmagent: `/targets` page now accepts optional `show_original_labels=1` query arg for displaying original labels for each target before relabeling is applied.
+  This should simplify debugging for target relabeling configs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/651
+* FEATURE: add `-finalMergeDelay` command-line flag for configuring the delay before final merge for per-month partitions.
+  The final merge is started after no new data is ingested into per-month partition during `-finalMergeDelay`.
+* FEATURE: add `vm_rows_added_to_storage_total` metric, which shows the total number of rows added to storage since app start.
+  The `sum(rate(vm_rows_added_to_storage_total))` can be smaller than `sum(rate(vm_rows_inserted_total))` if certain metrics are dropped
+  due to [relabeling](https://victoriametrics.github.io/#relabeling). The `sum(rate(vm_rows_added_to_storage_total))` can be bigger
+  than `sum(rate(vm_rows_inserted_total))` if [replication](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#replication-and-data-safety) is enabled.
+* FEATURE: keep metric name after applying [MetricsQL](https://victoriametrics.github.io/MetricsQL.html) functions, which don't change time series meaning.
+  The list of such functions:
+   * `keep_last_value`
+   * `keep_next_value`
+   * `interpolate`
+   * `running_min`
+   * `running_max`
+   * `running_avg`
+   * `range_min`
+   * `range_max`
+   * `range_avg`
+   * `range_first`
+   * `range_last`
+   * `range_quantile`
+   * `smooth_exponential`
+   * `ceil`
+   * `floor`
+   * `round`
+   * `clamp_min`
+   * `clamp_max`
+   * `max_over_time`
+   * `min_over_time`
+   * `avg_over_time`
+   * `quantile_over_time`
+   * `mode_over_time`
+   * `geomean_over_time`
+   * `holt_winters`
+   * `predict_linear`
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/674
+
+* BUGFIX: properly handle stale time series after K8S deployment. Previously such time series could be double-counted.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/748
+* BUGFIX: return a single time series at max from `absent()` function like Prometheus does.
+* BUGFIX: vmalert: accept days, weeks and years in `for: ` part of config like Prometheus does. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/817
+* BUGFIX: fix `mode_over_time(m[d])` calculations. Previously the function could return incorrect results.
+
+
+# [v1.43.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.43.0)
+
+* FEATURE: reduce CPU usage for repeated queries over sliding time window when no new time series are added to the database.
+  Typical use cases: repeated evaluation of alerting rules in [vmalert](https://victoriametrics.github.io/vmalert.html) or dashboard auto-refresh in Grafana.
+* FEATURE: vmagent: add OpenStack service discovery aka [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config).
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/728 .
+* FEATURE: vmalert: make `-maxIdleConnections` configurable for datasource HTTP client. This option can be used for minimizing connection churn.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/795 .
+* FEATURE: add `-influx.maxLineSize` command-line flag for configuring the maximum size for a single Influx line during parsing.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/807
+
+* BUGFIX: properly handle `inf` values during [background merge of LSM parts](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+  Previously `Inf` values could result in `NaN` values for adjancent samples in time series. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/805 .
+* BUGFIX: fill gaps on graphs for `range_*` and `running_*` functions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/806 .
+* BUGFIX: make a copy of label with new name during relabeling with `action: labelmap` in the same way as Prometheus does.
+  Previously the original label name has been replaced. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/812 .
+* BUGFIX: support parsing floating-point timestamp like Graphite Carbon does. Such timestmaps are truncated to seconds.
+
+
+# [v1.42.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.42.0)
+
+* FEATURE: use all the available CPU cores when accepting data via a single TCP connection
+  for [all the supported protocols](https://victoriametrics.github.io/#how-to-import-time-series-data).
+  Previously data ingested via a single TCP connection could use only a single CPU core. This could limit data ingestion performance.
+  The main benefit of this feature is that data can be imported at max speed via a single connection - there is no need to open multiple concurrent
+  connections to VictoriaMetrics or [vmagent](https://victoriametrics.github.io/vmagent.html) in order to achieve the maximum data ingestion speed.
+* FEATURE: cluster: improve performance for data ingestion path from `vminsert` to `vmstorage` nodes. The maximum data ingestion performance
+  for a single connection between `vminsert` and `vmstorage` node scales with the number of available CPU cores on `vmstorage` side.
+  This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/791 .
+* FEATURE: add ability to export / import data in native format via `/api/v1/export/native` and `/api/v1/import/native`.
+  This is the most optimized approach for data migration between VictoriaMetrics instances. Both single-node and cluster instances are supported.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/787#issuecomment-700632551 .
+* FEATURE: add `reduce_mem_usage` query option to `/api/v1/export` in order to reduce memory usage during data export / import.
+  See [these docs](https://victoriametrics.github.io/#how-to-export-data-in-json-line-format) for details.
+* FEATURE: improve performance for `/api/v1/series` handler when it returns big number of time series.
+* FEATURE: add `vm_merge_need_free_disk_space` metric, which can be used for estimating the number of deferred background data merges due to the lack of free disk space.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/686 .
+* FEATURE: add OpenBSD support. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/785 .
+
+* BUGFIX: properly apply `-search.maxStalenessInterval` command-line flag value. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784 .
+* BUGFIX: fix displaying data in Grafana tables. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 .
+* BUGFIX: do not adjust the number of detected CPU cores found at `/sys/devices/system/cpu/online`.
+  The adjustement was increasing the resulting GOMAXPROC by 1, which looked confusing to users.
+  See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-698595309 .
+* BUGFIX: vmagent: do not show `-remoteWrite.url` in initial logs if `-remoteWrite.showURL` isn't set. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/773 .
+* BUGFIX: properly handle case when [/metrics/find](https://victoriametrics.github.io/#graphite-metrics-api-usage) finds both a leaf and a node for the given `query=prefix.*`.
+  In this case only the node must be returned with stripped dot in the end of id as carbonapi does.
+
+
+# Previous releases
+
+See [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).

docs/CaseStudies.md

@@ -3,10 +3,24 @@
 Below are approved public case studies and talks from VictoriaMetrics users. Join our [community Slack channel](http://slack.victoriametrics.com/)
 and feel free asking for references, reviews and additional case studies from real VictoriaMetrics users there.
 
-See also [articles about VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Articles).
+See also [articles about VictoriaMetrics from our users](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Articles#third-party-articles-and-slides).
 
 
-## Adidas
+* [adidas](#adidas)
+* [CERN](#cern)
+* [COLOPL](#colopl)
+* [Zerodha](#zerodha)
+* [Wix.com](#wixcom)
+* [Wedos.com](#wedoscom)
+* [Synthesio](#synthesio)
+* [Dreamteam](#dreamteam)
+* [Brandwatch](#brandwatch)
+* [Adsterra](#adsterra)
+* [ARNES](#arnes)
+* [Idealo.de](#idealode)
+
+
+## adidas
 
 See [slides](https://promcon.io/2019-munich/slides/remote-write-storage-wars.pdf) and [video](https://youtu.be/OsH6gPdxR4s)
 from [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk at [PromCon 2019](https://promcon.io/2019-munich/).
@@ -73,13 +87,14 @@ See [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d
 
 Numbers:
 
-* The number of active time series per VictoriaMetrics instance is 20M.
-* The total number of time series per VictoriaMetrics instance is 400M+.
-* Ingestion rate per VictoriaMetrics instance is 800K data points per second.
-* The average time series churn rate is ~3M per day.
-* The average query rate is ~1K per minute (mostly alert queries).
-* Query duration: median is ~70ms, 99th percentile is ~2sec.
-* Retention: 6 months.
+* The number of active time series per VictoriaMetrics instance is 40M.
+* The total number of time series per VictoriaMetrics instance is 5000M+.
+* Ingestion rate per VictoriaMetrics instance is 1M data points per second.
+* The total number of datapoints per VictoriaMetrics instance is 8.5 trillions.
+* The average time series churn rate is ~80M per day.
+* The average query rate is ~100 per second (mostly alert queries).
+* Query duration: median is ~20ms, 99th percentile is ~1.5sec.
+* Retention: 3 months.
 
 > Alternatives that we’ve played with before choosing VictoriaMetrics are: federated Prometheus, Cortex, IronDB and Thanos.
 > Points that were critical to us when we were choosing a central tsdb, in order of importance:
@@ -303,3 +318,21 @@ Grafana has a LB infront, so if one DC has problems, we can still view all metri
 
 We are still in the process of migration, but we are really happy with the whole stack. It has proven as an essential piece
 for insight into our services during COVID-19 and has enabled us to provide better service and spot problems faster.
+
+
+## Idealo.de
+
+[idealo.de](https://www.idealo.de/) is the leading price comparison website in Germany. We use Prometheus for metrics on our container platform.
+When we introduced Prometheus at idealo we started with m3db as a longterm storage. In our setup m3db was quite unstable and consumed a lot of resources.
+
+VictoriaMetrics runs very stable for us and uses only a fraction of the resources. Although we also increased our retention time from 1 month to 13 months.
+
+Numbers:
+
+- The number of active time series per VictoriaMetrics instance is 21M.
+- Total ingestion rate 120k metrics per second.
+- The total number of datapoints 3.1 trillion.
+- The average time series churn rate is ~9M per day.
+- The average query rate is ~20 per second. Response time for 99th quantile is 120ms.
+- Retention: 13 months.
+- Size of all datapoints: 3.5 TB

docs/Cluster-VictoriaMetrics.md

@@ -177,10 +177,11 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
        This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag.
        See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#sending-opentsdb-data-via-http-apiput-requests) for details.
      - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` on `vmselect` (see below).
+     - `prometheus/api/v1/import/native` - for importing data obtained via `api/v1/export/native` on `vmselect` (see below).
      - `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data) for details.
      - `prometheus/api/v1/import/prometheus` - for importing data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
 
-* URLs for [Prmetheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
+* URLs for [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
   - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant)
   - `<suffix>` may have the following values:
     - `api/v1/query` - performs [PromQL instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries).
@@ -189,8 +190,12 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
     - `api/v1/labels` - returns a [list of label names](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names).
     - `api/v1/label/<label_name>/values` - returns values for the given `<label_name>` according [to API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values).
     - `federate` - returns [federated metrics](https://prometheus.io/docs/prometheus/latest/federation/).
-    - `api/v1/export` - exports raw data. See [this article](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639) for details.
+    - `api/v1/export` - exports raw data in JSON line format. See [this article](https://medium.com/@valyala/analyzing-prometheus-data-with-external-tools-5f3e5e147639) for details.
+    - `api/v1/export/native` - exports raw data in native binary format. It may be imported into another VictoriaMetrics via `api/v1/import/native` (see above).
+    - `api/v1/export/csv` - exports data in CSV. It may be imported into another VictoriaMetrics via `api/v1/import/csv` (see above).
     - `api/v1/status/tsdb` - for time series stats. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats) for details.
+      VictoriaMetrics accepts optional `topN=N` and `date=YYYY-MM-DD` query args for this handler, where `N` is the number of top entries to return in the response
+      and `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day.
     - `api/v1/status/active_queries` - for currently executed active queries. Note that every `vmselect` maintains an independent list of active queries,
       which is returned in the response.
 
@@ -200,12 +205,21 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr
       - `metrics/find` - searches Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find).
       - `metrics/expand` - expands Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand).
       - `metrics/index.json` - returns all the metric names. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json).
+      - `tags/tagSeries` - registers time series. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb).
+      - `tags/tagMultiSeries` - register multiple time series. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb).
+      - `tags` - returns tag names. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags).
+      - `tags/<tag_name>` - returns tag values for the given `<tag_name>`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags).
+      - `tags/findSeries` - returns series matching the given `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags).
+      - `tags/autoComplete/tags` - returns tags matching the given `tagPrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support).
+      - `tags/autoComplete/values` - returns tag values matching the given `valuePrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support).
+      - `tags/delSeries` - deletes series matching the given `path`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb).
 
 * URL for time series deletion: `http://<vmselect>:8481/delete/<accountID>/prometheus/api/v1/admin/tsdb/delete_series?match[]=<timeseries_selector_for_delete>`.
   Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't
   be used on a regular basis, since it carries non-zero overhead.
 
 * `vmstorage` nodes provide the following HTTP endpoints on `8482` port:
+  - `/internal/force_merge` - initiate [forced compactions](https://victoriametrics.github.io/#forced-merge) on the given `vmstorage` node.
   - `/snapshot/create` - create [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282),
     which can be used for backups in background. Snapshots are created in `<storageDataPath>/snapshots` folder, where `<storageDataPath>` is the corresponding
     command-line flag value.
@@ -274,8 +288,14 @@ Each instance type - `vminsert`, `vmselect` and `vmstorage` - can run on the mos
 #### vmstorage
 
 * The recommended total number of vCPU cores for all the `vmstorage` instances can be calculated from the ingestion rate: `vCPUs = ingestion_rate / 150K`.
-* The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = active_time_series * 1KB`.
+* The recommended total amount of RAM for all the `vmstorage` instances can be calculated from the number of active time series: `RAM = 2 * active_time_series * 1KB`.
   Time series is active if it received at least a single data point during the last hour or if it has been queried during the last hour.
+  The required RAM per each `vmstorage` should be multiplied by `-replicationFactor` if [replication](#replication-and-data-safety) is enabled.
+  Additional RAM can be required for query processing.
+  Calculated RAM requrements may differ from actual RAM requirements due to various factors:
+  * The average number of labels per time series. More labels require more RAM.
+  * The average length of label names and label values. Longer labels require more RAM.
+  * The type of queries. Heavy queries that scan big number of time series over long time ranges require more RAM.
 * The recommended total amount of storage space for all the `vmstorage` instances can be calculated
   from the ingestion rate and retention: `storage_space = ingestion_rate * retention_seconds`.
 
@@ -318,8 +338,9 @@ In order to enable application-level replication, `-replicationFactor=N` command
 This guarantees that all the data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable.
 For example, when `-replicationFactor=3` is passed to `vminsert`, then it replicates all the ingested data to 3 distinct `vmstorage` nodes.
 
-When the replication is enabled, `-dedup.minScrapeInterval=1ms` command-line flag must be passed to `vmselect`
-in order to de-duplicate replicated data during queries. It is OK if `-dedup.minScrapeInterval` exceeds 1ms
+When the replication is enabled, `-replicationFactor=N` and `-dedup.minScrapeInterval=1ms` command-line flag must be passed to `vmselect` nodes.
+The `-replicationFactor=N` improves query performance when a part of vmstorage nodes respond slowly and/or temporarily unavailable.
+The `-dedup.minScrapeInterval=1ms` de-duplicates replicated data during queries. It is OK if `-dedup.minScrapeInterval` exceeds 1ms
 when [deduplication](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#deduplication) is used additionally to replication.
 
 Note that [replication doesn't save from disaster](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883),

docs/FAQ.md

@@ -90,8 +90,8 @@ The main differences between Cortex and VictoriaMetrics:
 - Cortex may lose up to 12 hours of recent data on Ingestor failure - see [the corresponding docs](https://github.com/cortexproject/cortex/blob/fe56f1420099aa1bf1ce09316c186e05bddee879/docs/architecture.md#ingesters-failure-and-data-loss).
   VictoriaMetrics may lose only a few seconds of recent data, which isn't synced to persistent storage yet.
   See [this article for details](https://medium.com/@valyala/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
-- Cortex is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) and [other case studies](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies).
-- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV.
+- Cortex is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) and [other case studies](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies).
+- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV, JSON, native binary.
   See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
 
 
@@ -111,8 +111,8 @@ The main differences between Cortex and VictoriaMetrics:
   which is much easier to setup and operate than Thanos components.
 - Thanos may be harder to setup and operate comparing to VictoriaMetrics, since it has more moving parts, which can be connected with less reliable networks.
   See [this article for details](https://medium.com/faun/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
-- Thanos is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
-- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV.
+- Thanos is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
+- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to Prometheus remote_write protocol - InfluxDB, OpenTSDB, Graphite, CSV, JSON, native binary.
   See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
 
 
@@ -120,7 +120,7 @@ The main differences between Cortex and VictoriaMetrics:
 
 - VictoriaMetrics requires [10x less RAM](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) and it [works faster](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
 - VictoriaMetrics provides [better query language](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085) than InfluxQL or Flux.
-- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to InfluxDB - Prometheus remote_write, OpenTSDB, Graphite, CSV.
+- VictoriaMetrics accepts data in multiple popular data ingestion protocols additionally to InfluxDB - Prometheus remote_write, OpenTSDB, Graphite, CSV, JSON, native binary.
   See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data) for details.
 
 

docs/MetricsQL.md

@@ -15,6 +15,8 @@ The following functionality is implemented differently in MetricsQL comparing to
 * MetricsQL removes all the `NaN` values from the output, so some queries like `(-1)^0.5` return empty results in VictoriaMetrics, while returning
   a series of `NaN` values in Prometheus. Note that Grafana doesn't draw any lines or dots for `NaN` values, so usually the end result looks the same for both
   VictoriaMetrics and Prometheus.
+* MetricsQL keeps metric names after applying functions, which don't change the meaining of the original time series. For example, `min_over_time(foo)` or `round(foo)`
+  leave `foo` metric name in the result. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/674) for details.
 
 Other PromQL functionality should work the same in MetricsQL. [File an issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues)
 if you notice discrepancies between PromQL and MetricsQL results other than mentioned above.
@@ -26,7 +28,7 @@ Feel free [filing a feature request](https://github.com/VictoriaMetrics/Victoria
 
 This functionality can be tried at [an editable Grafana dashboard](http://play-grafana.victoriametrics.com:3000/d/4ome8yJmz/node-exporter-on-victoriametrics-demo).
 
-- [`WITH` templates](https://play.victoriametrics.com/promql/expand-with-exprs). This feature simplifies writing and managing complex queries. Go to [`WITH` templates playground](https://victoriametrics.com/promql/expand-with-exprs) and try it.
+- [`WITH` templates](https://play.victoriametrics.com/promql/expand-with-exprs). This feature simplifies writing and managing complex queries. Go to [`WITH` templates playground](https://play.victoriametrics.com/promql/expand-with-exprs) and try it.
 - Range duration in functions such as [rate](https://prometheus.io/docs/prometheus/latest/querying/functions/#rate()) may be omitted. VictoriaMetrics automatically selects range duration depending on the current step used for building the graph. For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
 - All the aggregate functions support optional `limit N` suffix in order to limit the number of output series. For example, `sum(x) by (y) limit 10` limits
   the number of output time series after the aggregation to 10. All the other time series are dropped.
@@ -66,15 +68,15 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `step()` function for returning the step in seconds used in the query.
 - `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query.
 - `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`.
-- `ideriv(m)` - for calculating `instant` derivative for `m`.
+- `ideriv(m[d])` - for calculating `instant` derivative for the metric `m` over the duration `d`.
 - `deriv_fast(m[d])` - for calculating `fast` derivative for `m` based on the first and the last points from duration `d`.
 - `running_` functions - `running_sum`, `running_min`, `running_max`, `running_avg` - for calculating [running values](https://en.wikipedia.org/wiki/Running_total) on the selected time range.
-- `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range.
+- `range_` functions - `range_sum`, `range_min`, `range_max`, `range_avg`, `range_first`, `range_last`, `range_median`, `range_quantile` - for calculating global value over the selected time range. Note that global value is based on calculated datapoints for the inner query. The calculated datapoints can differ from raw datapoints stored in the database. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) for details.
 - `smooth_exponential(q, sf)` - smooths `q` using [exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) with the given smooth factor `sf`.
 - `remove_resets(q)` - removes counter resets from `q`.
-- `lag(q[d])` - returns lag between the current timestamp and the timestamp from the previous data point in `q` over `d`.
-- `lifetime(q[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `q`.
-- `scrape_interval(q[d])` - returns the average interval in seconds between data points of `q` over `d` aka `scrape interval`.
+- `lag(m[d])` - returns lag between the current timestamp and the timestamp from the previous data point in `m` over `d`.
+- `lifetime(m[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `m`.
+- `scrape_interval(m[d])` - returns the average interval in seconds between data points of `m` over `d` aka `scrape interval`.
 - Trigonometric functions - `sin(q)`, `cos(q)`, `asin(q)`, `acos(q)` and `pi()`.
 - `range_over_time(m[d])` - returns value range for `m` over `d` time window, i.e. `max_over_time(m[d])-min_over_time(m[d])`.
 - `median_over_time(m[d])` - calculates median values for `m` over `d` time window. Shorthand to `quantile_over_time(0.5, m[d])`.
@@ -99,8 +101,8 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `histogram_over_time(m[d])` - calculates [VictoriaMetrics histogram](https://godoc.org/github.com/VictoriaMetrics/metrics#Histogram) for `m` over `d`.
   For example, the following query calculates median temperature by country over the last 24 hours:
   `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmbucket, country))`.
-- `histogram_share(le, buckets)` - returns share (in the range 0..1) for `buckets`. Useful for calculating SLI and SLO.
-  For instance, the following query returns the share of requests which are performed under 1.5 seconds: `histogram_share(1.5, sum(request_duration_seconds_bucket) by (le))`.
+- `histogram_share(le, buckets)` - returns share (in the range 0..1) for `buckets` that fall below `le`. Useful for calculating SLI and SLO.
+  For instance, the following query returns the share of requests which are performed under 1.5 seconds during the last 5 minutes: `histogram_share(1.5, sum(rate(request_duration_seconds_bucket[5m])) by (le))`.
 - `topk_*` and `bottomk_*` aggregate functions, which return up to K time series. Note that the standard `topk` function may return more than K time series -
    see [this article](https://www.robustperception.io/graph-top-n-time-series-in-grafana) for details.
    - `topk_min(k, q)` - returns top K time series with the max minimums on the given time range
@@ -110,7 +112,9 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
    - `bottomk_min(k, q)` - returns bottom K time series with the min minimums on the given time range
    - `bottomk_max(k, q)` - returns bottom K time series with the min maximums on the given time range
    - `bottomk_avg(k, q)` - returns bottom K time series with the min averages on the given time range
-   - `bottomk_median(k, q)` - returns bottom K time series with the min medians on the given time range
+   - `bottomk_median(k, q)` - returns bottom K time series with the min medians on the given time range.
+
+  All the `topk_*` and `bottomk_*` functions accept optional third argument - label name for the sum of the remaining time series outside top K or bottom K time series. For example, `topk_max(3, process_resident_memory_bytes, "remaining_sum")` would return up to 3 time series with the maximum value for `process_resident_memory_bytes` plus fourth time series with the sum of the remaining time series if any. The fourth time series will contain `remaining_sum="remaining_sum"` additional label.
 - `share_le_over_time(m[d], le)` - returns share (in the range 0..1) of values in `m` over `d`, which are smaller or equal to `le`. Useful for calculating SLI and SLO.
   Example: `share_le_over_time(memory_usage_bytes[24h], 100*1024*1024)` returns the share of time series values for the last 24 hours when memory usage was below or equal to 100MB.
 - `share_gt_over_time(m[d], gt)` - returns share (in the range 0..1) of values in `m` over `d`, which are bigger than `gt`. Useful for calculating SLI and SLO.

docs/Quick-Start.md

@@ -1,14 +1,16 @@
 # Quick Start
 
-1. Download the latest VictoriaMetrics release from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
+1. If you run Ubuntu, then just run `snap install victoriametrics` command in order to install and start VictoriaMetrics, then read [these docs](https://snapcraft.io/victoriametrics).
+   Otherwise download the latest VictoriaMetrics release from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
    from [Docker hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/)
    or [build it from sources](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Single-server-VictoriaMetrics#how-to-build-from-sources).
 
-2. Run the binary or Docker image with the desired command-line flags. Pass `-help` in order to see description for all the available flags
+2. This step isn't needed if you run VictoriaMetrics via `snap install victoriametrics` as described above.
+   Otherwise run the binary or Docker image with the desired command-line flags. Pass `-help` in order to see description for all the available flags
    and their default values. Default flag values should fit the majoirty of cases. The minimum required flags to configure are:
 
    * `-storageDataPath` - path to directory where VictoriaMetrics stores all the data.
-   * `-retentionPeriod` - data retention in months.
+   * `-retentionPeriod` - data retention.
 
    For instance:
 

docs/Release-Guide.md

@@ -2,6 +2,7 @@ Release process guidance
 
 ## Release version and Docker images
 
+0. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md).
 1. Create release tag with `git tag v1.xx.y`.
 2. Run `make release` for creating `*.tar.gz` release archive with the corresponding `_checksums.txt` inside `bin` directory.
 3. Run `make publish` for creating and publishing Docker images.

docs/vmagent.md

@@ -25,7 +25,8 @@ to `vmagent` (like the ability to push metrics instead of pulling them). We did
   * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
   * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
   * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
-  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-json-line-format).
+  * Native data import protocol via `http://<vmagent>:8429/api/v1/import/native`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-native-format).
   * Data in Prometheus exposition format. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-data-in-prometheus-exposition-format) for details.
   * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
 * Can replicate collected metrics simultaneously to multiple remote storage systems.
@@ -41,7 +42,7 @@ Just download `vmutils-*` archive from [releases page](https://github.com/Victor
 and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
 
 * `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
-* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary amount of remote storage systems.
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary number of remote storage systems.
 
 Example command line:
 
@@ -62,6 +63,22 @@ Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://gi
 Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
 
 
+### Configuration update
+
+`vmagent` should be restarted in order to update config options set via command-line args.
+
+`vmagent` supports multiple approaches for reloading configs from updated config files such as `-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
+
+* Sending `SUGHUP` signal to `vmagent` process:
+  ```bash
+  kill -SIGHUP `pidof vmagent`
+  ```
+
+* Sending HTTP request to `http://vmagent:8429/-/reload` endpoint.
+
+There is also `-promscrape.configCheckInterval` command-line option, which can be used for automatic reloading configs from updated `-promscrape.config` file.
+
+
 ### Use cases
 
 
@@ -135,7 +152,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `ec2_sd_configs` - for scraping targets in Amazon EC2.
   See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
-  `vmagent` doesn't support `role_arn` config param yet.
+  `vmagent` doesn't support `profile` config param and aws credentials file yet.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
   `vmagent` provides the following additional functionality for `gce_sd_config`:
@@ -147,6 +164,13 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
 * `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
   See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
+* `openstack_sd_configs` - for scraping OpenStack targets.
+  See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
+  [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
+* `dockerswarm_sd_configs` - for scraping Docker Swarm targets.
+  See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
+* `eureka_sd_configs` - for scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka).
+  See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
 
 File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
@@ -191,6 +215,7 @@ The relabeling can be defined in the following places:
 
 Read more about relabeling in the following articles:
 
+* [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
 * [Life of a label](https://www.robustperception.io/life-of-a-label)
 * [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
 * [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
@@ -205,7 +230,16 @@ either via `vmagent` itself or via Prometheus, so the exported metrics could be
 Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
 If you have suggestions, improvements or found a bug - feel free to open an issue on github or add review to the dashboard.
 
-`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+`vmagent` also exports target statuses at the following handlers:
+
+* `http://vmagent-host:8429/targets`. This handler returns human-readable plaintext status for every active target.
+This page is convenient to query from command line with `wget`, `curl` or similar tools.
+It accepts optional `show_original_labels=1` query arg, which shows the original labels per each target before applying relabeling.
+This information may be useful for debugging target relabeling.
+* `http://vmagent-host:8429/api/v1/targets`. This handler returns data compatible with [the corresponding page from Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/#targets).
+
+* `http://vmagent-host:8429/ready`. This handler returns http 200 status code when `vmagent` finishes initialization for all service_discovery configs.
+It may be useful for performing `vmagent` rolling update without scrape loss.
 
 
 ### Troubleshooting
@@ -216,7 +250,32 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
   since `vmagent` establishes at least a single TCP connection per each target.
 
 * When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
-  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
+  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`
+  and `http://vmagent-host:8429/api/v1/targets`.
+
+* The `/api/v1/targets` page could be useful for debugging relabeling process for scrape targets.
+  This page contains original labels for targets dropped during relabeling (see "droppedTargets" section in the page output). By default up to `-promscrape.maxDroppedTargets` targets are shown here. If your setup drops more targets during relabeling, then increase `-promscrape.maxDroppedTargets` command-line flag value in order to see all the dropped targets. Note that tracking each dropped target requires up to 10Kb of RAM, so big values for `-promscrape.maxDroppedTargets` may result in increased memory usage if big number of scrape targets are dropped during relabeling.
+
+* If `vmagent` scrapes big number of targets, then `-promscrape.dropOriginalLabels` command-line option may be passed to `vmagent` in order to reduce memory usage.
+  This option drops `"discoveredLabels"` and `"droppedTargets"` lists at `/api/v1/targets` page, which may result in reduced debuggability for improperly configured per-target relabeling.
+
+* If `vmagent` scrapes targets with millions of metrics per each target (for instance, when scraping [federation endpoints](https://prometheus.io/docs/prometheus/latest/federation/)),
+  then it is recommended enabling `stream parsing mode` in order to reduce memory usage during scraping. This mode may be enabled either globally for all the scrape targets
+  by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example:
+
+  ```yml
+  scrape_configs:
+  - job_name: 'big-federate'
+    stream_parse: true
+    static_configs:
+    - targets:
+      - big-prometeus1
+      - big-prometeus2
+    honor_labels: true
+    metrics_path: /federate
+    params:
+      'match[]': ['{__name__!=""}']
+  ```
 
 * It is recommended to increase `-remoteWrite.queues` if `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
 
@@ -228,8 +287,13 @@ If you have suggestions, improvements or found a bug - feel free to open an issu
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
   If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
 
+* By default `vmagent` masks `-remoteWrite.url` with `secret-url` values in logs and at `/metrics` page because
+  the url may contain sensitive information such as auth tokens or passwords.
+  Pass `-remoteWrite.showURL` command-line flag when starting `vmagent` in order to see all the valid urls.
+
 * If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports
-  or they use init container.
+  or they use init container. These errors can be either fixed or suppressed with `-promscrape.suppressDuplicateScrapeTargetErrors` command-line flag.
+  See available options below if you prefer fixing the root cause of the error:
 
   The following `relabel_configs` section may help determining `__meta_*` labels resulting in duplicate targets:
   ```yml
@@ -274,7 +338,7 @@ Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

docs/vmalert.md

@@ -11,6 +11,7 @@ rules against configured address.
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
  support;
 * Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Keeps the alerts [state on restarts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmalert#alerts-state-on-restarts);
 * Lightweight without extra dependencies.
 
 ### Limitations:
@@ -121,14 +122,6 @@ annotations:
   [ <labelname>: <tmpl_string> ]
 ``` 
 
-`vmalert` has no local storage and alerts state is stored in process memory. Hence, after reloading of `vmalert` process
-alerts state will be lost. To avoid this situation, `vmalert` may be configured via following flags:
-* `-remoteWrite.url` - URL to Victoria Metrics or VMInsert. `vmalert` will persist alerts state into the configured
-address in form of timeseries with name `ALERTS` via remote-write protocol.
-* `-remoteRead.url` - URL to Victoria Metrics or VMSelect. `vmalert` will try to restore alerts state from configured
-address by querying `ALERTS` timeseries.
-
-
 ##### Recording rules
 
 The syntax for recording rules is following:
@@ -147,6 +140,22 @@ labels:
 For recording rules to work `-remoteWrite.url` must specified.
 
 
+#### Alerts state on restarts
+
+`vmalert` has no local storage, so alerts state is stored in the process memory. Hence, after reloading of `vmalert` 
+the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
+* `-remoteWrite.url` - URL to VictoriaMetrics (Single) or VMInsert (Cluster). `vmalert` will persist alerts state 
+into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol. 
+These are regular time series and may be queried from VM just as any other time series. 
+The state stored to the configured address on every rule evaluation.
+* `-remoteRead.url` - URL to VictoriaMetrics (Single) or VMSelect (Cluster). `vmalert` will try to restore alerts state 
+from configured address by querying time series with name `ALERTS_FOR_STATE`.
+
+Both flags are required for the proper state restoring. Restore process may fail if time series are missing
+in configured `-remoteRead.url`, weren't updated in the last `1h` or received state doesn't match current `vmalert` 
+rules configuration.
+
+
 #### WEB
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
@@ -166,6 +175,10 @@ The shortlist of configuration flags is the following:
     	Optional basic auth password for -datasource.url
   -datasource.basicAuth.username string
     	Optional basic auth username for -datasource.url
+  -datasource.lookback duration
+        Lookback defines how far to look into past when evaluating queries. For example, if datasource.lookback=5m then param "time" with value now()-5m will be added to every query.
+  -datasource.maxIdleConnections int
+        Defines the number of idle (keep-alive connections) to configured datasource.Consider to set this value equal to the value: groups_total * group.concurrency. Too low value may result into high number of sockets in TIME_WAIT state. (default 100)
   -datasource.tlsCAFile string
     	Optional path to TLS CA file to use for verifying connections to -datasource.url. By default system CA is used
   -datasource.tlsCertFile string

docs/vmauth.md

@@ -113,7 +113,7 @@ Run `make package-vmauth`. It builds `victoriametrics/vmauth:<PKG_TAG>` docker i
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmauth`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

docs/vmbackup.md

@@ -6,12 +6,12 @@ Supported storage systems for backups:
 
 * [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
 * [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
-* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`
 
-Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+`vmbackup` supports incremental and full backups. Incremental backups created automatically if the destination path already contains data from the previous backup.
 Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
-data between the existing backup and new backup. This saves time and costs on data transfer.
+data between the existing backup and new backup. It saves time and costs on data transfer.
 
 Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
 
@@ -35,8 +35,8 @@ vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-
 
 * `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
   There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
-* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
 
 
@@ -49,13 +49,13 @@ with the following command:
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
 ```
 
-This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
 
 
 #### Incremental backups
 
-Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
-This saves time and network bandwidth costs when working with big backups:
+Incremental backups performed if `-dst` points to an already existing backup. In this case only new data uploaded to remote storage.
+It saves time and network bandwidth costs when working with big backups:
 
 ```
 vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
@@ -100,16 +100,16 @@ The backup algorithm is the following:
 2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
 3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
 4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
-   This are usually the biggest and the oldest files, which are shared between backups.
-5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+   These are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
 
-The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+The algorithm splits source files into 100 MB chunks in the backup. Each chunk stored as a separate file in the backup.
 Such splitting minimizes the amounts of data to re-transfer after temporary errors.
 
 `vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
 
 - All the files in the snapshot are immutable.
-- Old files are periodically merged into new files.
+- Old files periodically merged into new files.
 - Smaller files have higher probability to be merged.
 - Consecutive snapshots share many identical files.
 
@@ -129,7 +129,45 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 ### Advanced usage
 
-Run `vmbackup -help` in order to see all the available options:
+
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with the following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
+  You have to add a custom url endpoint via flag:
+```
+  # for minio
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+* Run `vmbackup -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -207,7 +245,7 @@ Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` dock
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

docs/vmrestore.md

@@ -3,7 +3,7 @@
 `vmrestore` restores data from backups created by [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
 VictoriaMetrics `v1.29.0` and newer versions must be used for working with the restored data.
 
-Restore process can be interrupted at any time. It is automatically resumed from the inerruption point
+Restore process can be interrupted at any time. It is automatically resumed from the interruption point
 when restarting `vmrestore` with the same args.
 
 
@@ -21,7 +21,7 @@ vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/r
 * `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
   to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
 
-The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup,
+The original `-storageDataPath` directory may contain old files. They will be substituted by the files from backup,
 i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/questions/476041/how-do-i-make-rsync-delete-files-that-have-been-deleted-from-the-source-folder).
 
 
@@ -33,7 +33,44 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 
 ### Advanced usage
 
-Run `vmrestore -help` in order to see all the available options:
+* Obtaining credentials from a file.
+
+  Add flag `-credsFilePath=/etc/credentials` with following content:
+
+    for s3 (aws, minio or other s3 compatible storages):
+     ```bash
+     [default]
+     aws_access_key_id=theaccesskey
+     aws_secret_access_key=thesecretaccesskeyvalue
+    ```
+
+    for gce cloud storage:
+    ```json
+    {
+           "type": "service_account",
+           "project_id": "project-id",
+           "private_key_id": "key-id",
+           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+           "client_email": "service-account-email",
+           "client_id": "client-id",
+           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+           "token_uri": "https://accounts.google.com/o/oauth2/token",
+           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+    }
+    ```
+
+* Usage with s3 custom url endpoint.  It is possible to use `vmrestore` with s3 api compatible storages, like  minio, cloudian and other.
+  You have to add custom url endpoint with a flag:
+```
+  # for minio:
+  -customS3Endpoint=http://localhost:9000
+
+  # for aws gov region
+  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+```
+
+*  Run `vmrestore -help` in order to see all the available options:
 
 ```
   -concurrency int
@@ -104,7 +141,7 @@ Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` do
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.
 
-By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
 by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
 
 ```bash

go.mod

@@ -1,31 +1,32 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
 require (
-	cloud.google.com/go/storage v1.11.0
+	cloud.google.com/go v0.72.0 // indirect
+	cloud.google.com/go/storage v1.12.0
 	github.com/VictoriaMetrics/fastcache v1.5.7
 
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
-	github.com/VictoriaMetrics/fasthttp v1.0.5
+	github.com/VictoriaMetrics/fasthttp v1.0.7
 	github.com/VictoriaMetrics/metrics v1.12.3
-	github.com/VictoriaMetrics/metricsql v0.6.0
-	github.com/aws/aws-sdk-go v1.34.20
+	github.com/VictoriaMetrics/metricsql v0.7.2
+	github.com/aws/aws-sdk-go v1.35.31
 	github.com/cespare/xxhash/v2 v2.1.1
-	github.com/golang/snappy v0.0.1
-	github.com/klauspost/compress v1.11.0
-	github.com/valyala/fastjson v1.5.4
+	github.com/golang/snappy v0.0.2
+	github.com/klauspost/compress v1.11.3
+	github.com/stretchr/testify v1.5.1 // indirect
+	github.com/valyala/fastjson v1.6.3
 	github.com/valyala/fastrand v1.0.0
 	github.com/valyala/fasttemplate v1.2.1
 	github.com/valyala/gozstd v1.8.3
 	github.com/valyala/histogram v1.1.2
-	github.com/valyala/quicktemplate v1.6.2
-	golang.org/x/net v0.0.0-20200904194848-62affa334b73 // indirect
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
-	golang.org/x/sys v0.0.0-20200909081042-eff7692f9009
-	golang.org/x/tools v0.0.0-20200909210914-44a2922940c2 // indirect
-	google.golang.org/api v0.31.0
-	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
-	google.golang.org/grpc v1.32.0 // indirect
+	github.com/valyala/quicktemplate v1.6.3
+	golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58
+	golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
+	golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0 // indirect
+	google.golang.org/api v0.35.0
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 )
 

go.sum

@@ -9,62 +9,49 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4=
 cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms=
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0 h1:RmDygqvj27Zf3fCQjQRtLyC7KwFcHkeJitcO0OoGOcA=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.64.0/go.mod h1:xfORb36jGvE+6EexW71nMEtL025s3x6xvuYUKM4JLv4=
-cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko=
+cloud.google.com/go v0.72.0 h1:eWRCuwubtDrCJG0oSUMgnsbD4CmPFQF2ei4OFbXvwww=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0 h1:a/O/bK/vWrYGOTFtH8di4rBxMZnmkjy+Y5LxpDwo+dA=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU=
 cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0 h1:86K1Gel7BQ9/WmNWn7dTKMvTLFzwtBe5FNqYbi9X35g=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.11.0 h1:bSLyzhbGjLMYxCratCDRSSH7+xRGpNApTBmowDUFGLk=
-cloud.google.com/go/storage v1.11.0/go.mod h1:/PAbprKS+5msVYogBmczjWalDXnQ9mr64yEq9YnyPeo=
+cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4=
+cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
 github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
-github.com/VictoriaMetrics/fasthttp v1.0.5 h1:DvNn5bStvgb7cZbWUkvd7psaPY9ul+7b2gA6uoiCmHc=
-github.com/VictoriaMetrics/fasthttp v1.0.5/go.mod h1:88NVQ+fmidWtXEFT153t7lAEZNu6Q3DgK1g0l1ZyQtw=
-github.com/VictoriaMetrics/metrics v1.12.2 h1:SG8iAmqavDNuh7GIdHPoGHUhDL23KeKfvSZSozucNeA=
+github.com/VictoriaMetrics/fasthttp v1.0.7 h1:9RntF8jE9z+ooyXy0tb3Pl76jan4DLaCQoCQFp/cIUE=
+github.com/VictoriaMetrics/fasthttp v1.0.7/go.mod h1:eaGv8oDDOWE8JQa6GgBgpCwjJhRxavmprtf2oWw8b3o=
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I=
 github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.6.0 h1:JnHUmifuA3fdy1GQrmkZJFO+CwFrhLxKwzMv89wNgJ4=
-github.com/VictoriaMetrics/metricsql v0.6.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.7.2 h1:ZdFPiA9Etrf3dow43IcPvLjPi5BYWIYj194wPKIhKfs=
+github.com/VictoriaMetrics/metricsql v0.7.2/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/aws/aws-sdk-go v1.34.20 h1:D9otznteZZyN5pRyFETqveYia/85Xzk7+RaPGB1I9fE=
-github.com/aws/aws-sdk-go v1.34.20/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.35.31 h1:6tlaYq4Q311qfhft/fIaND33XI27aW3zIdictcHxifE=
+github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -83,7 +70,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -108,10 +94,12 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -120,14 +108,14 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -135,26 +123,29 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
-github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=
-github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg=
 github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
+github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -172,9 +163,9 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
-github.com/valyala/fastjson v1.5.4 h1:r8gpiVwdzDU09NrlN38OyL5dUFpdwGQR5RQEBqY+hLg=
-github.com/valyala/fastjson v1.5.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
+github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
+github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
+github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
@@ -183,8 +174,8 @@ github.com/valyala/gozstd v1.8.3 h1:nHlS+sCFoNLsZpRPKDviXkhHybaRSUjH2w0P/myYo0I=
 github.com/valyala/gozstd v1.8.3/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/valyala/histogram v1.1.2 h1:vOk5VrGjMBIoPR5k6wA8vBaC8toeJ8XO0yfRjFEc1h8=
 github.com/valyala/histogram v1.1.2/go.mod h1:CZAr6gK9dbD7hYx2s8WSPh0p5x5wETjC+2b3PJVtEdg=
-github.com/valyala/quicktemplate v1.6.2 h1:k0vgK7zlmFzqAoIBIOrhrfmZ6JoTGJlLRPLbkPGr2/M=
-github.com/valyala/quicktemplate v1.6.2/go.mod h1:mtEJpQtUiBV0SHhMX6RtiJtqxncgrfmjcUy5T68X8TM=
+github.com/valyala/quicktemplate v1.6.3 h1:O7EuMwuH7Q94U2CXD6sOX8AYHqQqWtmIk690IhmpkKA=
+github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -194,8 +185,9 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -258,22 +250,21 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
-golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
 golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
 golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 h1:Mj83v+wSRNEar42a/MQgxk9X42TdEmrOl9i+y8WbxLo=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -282,6 +273,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -309,14 +301,17 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -359,12 +354,14 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200817023811-d00afeaade8f/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200827163409-021d7c6f1ec3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200909210914-44a2922940c2 h1:daAzF/Ytp6YSqJDu1hZJthJIhOrsAa7UbIkziU1t0K4=
-golang.org/x/tools v0.0.0-20200909210914-44a2922940c2/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0 h1:26vapYZ9m+DJd68m3DCFP/swNNErXAU7tOMFG7NyUuM=
+golang.org/x/tools v0.0.0-20201119132711-4783bc9bebf0/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -385,17 +382,19 @@ google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
 google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0 h1:yfrXXP61wVuLb0vBcG6qaOoIoqYEzOQS8jum51jkv2w=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.31.0 h1:1w5Sz/puhxFo9lTtip2n47k7toB/U2nCqOKNHd3Yrbo=
 google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo=
+google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.35.0 h1:TBCmTTxUrRDA1iTctnK/fIeitxIZ+TQuaf0j29fmCGo=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -424,12 +423,14 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200815001618-f69a88009b70/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr11OJ+54YeCMCGYIygTA7R/YZxH5M=
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 h1:Rt0FRalMgdSlXAVJvX4pr65KfqaxHXSLkSJRD9pw6g0=
+google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -441,12 +442,11 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs=
 google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
 google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -463,6 +463,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

lib/backup/actions/backup.go

@@ -71,45 +71,47 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
 
 	logger.Infof("starting backup from %s to %s using origin %s", src, dst, origin)
 
-	logger.Infof("obtaining list of parts at %s", src)
 	srcParts, err := src.ListParts()
 	if err != nil {
 		return fmt.Errorf("cannot list src parts: %w", err)
 	}
-	logger.Infof("obtaining list of parts at %s", dst)
+	logger.Infof("obtained %d parts from src %s", len(srcParts), src)
+
 	dstParts, err := dst.ListParts()
 	if err != nil {
 		return fmt.Errorf("cannot list dst parts: %w", err)
 	}
-	logger.Infof("obtaining list of parts at %s", origin)
+	logger.Infof("obtained %d parts from dst %s", len(dstParts), dst)
+
 	originParts, err := origin.ListParts()
 	if err != nil {
 		return fmt.Errorf("cannot list origin parts: %w", err)
 	}
+	logger.Infof("obtained %d parts from origin %s", len(originParts), origin)
 
 	backupSize := getPartsSize(srcParts)
 
 	partsToDelete := common.PartsDifference(dstParts, srcParts)
 	deleteSize := getPartsSize(partsToDelete)
 	if len(partsToDelete) > 0 {
-		logger.Infof("deleting %d parts from %s", len(partsToDelete), dst)
+		logger.Infof("deleting %d parts from dst %s", len(partsToDelete), dst)
 		deletedParts := uint64(0)
 		err = runParallel(concurrency, partsToDelete, func(p common.Part) error {
-			logger.Infof("deleting %s from %s", &p, dst)
+			logger.Infof("deleting %s from dst %s", &p, dst)
 			if err := dst.DeletePart(p); err != nil {
-				return fmt.Errorf("cannot delete %s from %s: %w", &p, dst, err)
+				return fmt.Errorf("cannot delete %s from dst %s: %w", &p, dst, err)
 			}
 			atomic.AddUint64(&deletedParts, 1)
 			return nil
 		}, func(elapsed time.Duration) {
 			n := atomic.LoadUint64(&deletedParts)
-			logger.Infof("deleted %d out of %d parts from %s in %s", n, len(partsToDelete), dst, elapsed)
+			logger.Infof("deleted %d out of %d parts from dst %s in %s", n, len(partsToDelete), dst, elapsed)
 		})
 		if err != nil {
 			return err
 		}
 		if err := dst.RemoveEmptyDirs(); err != nil {
-			return fmt.Errorf("cannot remove empty directories at %s: %w", dst, err)
+			return fmt.Errorf("cannot remove empty directories at dst %s: %w", dst, err)
 		}
 	}
 
@@ -117,18 +119,18 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
 	originCopyParts := common.PartsIntersect(originParts, partsToCopy)
 	copySize := getPartsSize(originCopyParts)
 	if len(originCopyParts) > 0 {
-		logger.Infof("server-side copying %d parts from %s to %s", len(originCopyParts), origin, dst)
+		logger.Infof("server-side copying %d parts from origin %s to dst %s", len(originCopyParts), origin, dst)
 		copiedParts := uint64(0)
 		err = runParallel(concurrency, originCopyParts, func(p common.Part) error {
-			logger.Infof("server-side copying %s from %s to %s", &p, origin, dst)
+			logger.Infof("server-side copying %s from origin %s to dst %s", &p, origin, dst)
 			if err := dst.CopyPart(origin, p); err != nil {
-				return fmt.Errorf("cannot copy %s from %s to %s: %w", &p, origin, dst, err)
+				return fmt.Errorf("cannot copy %s from origin %s to dst %s: %w", &p, origin, dst, err)
 			}
 			atomic.AddUint64(&copiedParts, 1)
 			return nil
 		}, func(elapsed time.Duration) {
 			n := atomic.LoadUint64(&copiedParts)
-			logger.Infof("server-side copied %d out of %d parts from %s to %s in %s", n, len(originCopyParts), origin, dst, elapsed)
+			logger.Infof("server-side copied %d out of %d parts from origin %s to dst %s in %s", n, len(originCopyParts), origin, dst, elapsed)
 		})
 		if err != nil {
 			return err
@@ -138,36 +140,36 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
 	srcCopyParts := common.PartsDifference(partsToCopy, originParts)
 	uploadSize := getPartsSize(srcCopyParts)
 	if len(srcCopyParts) > 0 {
-		logger.Infof("uploading %d parts from %s to %s", len(srcCopyParts), src, dst)
+		logger.Infof("uploading %d parts from src %s to dst %s", len(srcCopyParts), src, dst)
 		bytesUploaded := uint64(0)
 		err = runParallel(concurrency, srcCopyParts, func(p common.Part) error {
-			logger.Infof("uploading %s from %s to %s", &p, src, dst)
+			logger.Infof("uploading %s from src %s to dst %s", &p, src, dst)
 			rc, err := src.NewReadCloser(p)
 			if err != nil {
-				return fmt.Errorf("cannot create reader for %s from %s: %w", &p, src, err)
+				return fmt.Errorf("cannot create reader for %s from src %s: %w", &p, src, err)
 			}
 			sr := &statReader{
 				r:         rc,
 				bytesRead: &bytesUploaded,
 			}
 			if err := dst.UploadPart(p, sr); err != nil {
-				return fmt.Errorf("cannot upload %s to %s: %w", &p, dst, err)
+				return fmt.Errorf("cannot upload %s to dst %s: %w", &p, dst, err)
 			}
 			if err = rc.Close(); err != nil {
-				return fmt.Errorf("cannot close reader for %s from %s: %w", &p, src, err)
+				return fmt.Errorf("cannot close reader for %s from src %s: %w", &p, src, err)
 			}
 			return nil
 		}, func(elapsed time.Duration) {
 			n := atomic.LoadUint64(&bytesUploaded)
-			logger.Infof("uploaded %d out of %d bytes from %s to %s in %s", n, uploadSize, src, dst, elapsed)
+			logger.Infof("uploaded %d out of %d bytes from src %s to dst %s in %s", n, uploadSize, src, dst, elapsed)
 		})
 		if err != nil {
 			return err
 		}
 	}
 
-	logger.Infof("backed up %d bytes in %.3f seconds; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
-		backupSize, time.Since(startTime).Seconds(), deleteSize, copySize, uploadSize)
+	logger.Infof("backup from src %s to dst %s with origin %s is complete; backed up %d bytes in %.3f seconds; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
+		src, dst, origin, backupSize, time.Since(startTime).Seconds(), deleteSize, copySize, uploadSize)
 
 	return nil
 }