lib/protoparser: add parser for Prometheus exposition text format

This parser will be used by vmagent

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019 VictoriaMetrics, Inc.
+   Copyright 2019-2020 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -6,7 +6,7 @@
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/workflows/main/badge.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions)
 [![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 
-<img alt="Victoria Metrics" src="logo.png" height="200px">
+<img alt="Victoria Metrics" src="logo.png">
 
 ## VictoriaMetrics
 
@@ -18,8 +18,10 @@ in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just downl
 Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 
-## Case studies
+## Case studies and talks
 
+* [Adidas](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#adidas)
+* [COLOPL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#colopl)
 * [Wix.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wixcom)
 * [Wedos.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wedoscom)
 * [Dreamteam](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#dreamteam)
@@ -38,9 +40,11 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
 * High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
   may be crammed into limited storage comparing to TimescaleDB.
 * Optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
-* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, Uber M3, Cortex, InfluxDB or TimescaleDB.
-  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
-  and [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
+* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB.
+  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae),
+  [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
+  and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk
+  from [PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
 * Easy operation:
   * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
@@ -501,6 +505,7 @@ Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/iss
 ### Third-party contributions
 
 * [Unofficial yum repository](https://copr.fedorainfracloud.org/coprs/antonpatsev/VictoriaMetrics/) ([source code](https://github.com/patsevanton/victoriametrics-rpm))
+* [Prometheus -> VictoriaMetrics exporter](https://github.com/ryotarai/prometheus-tsdb-dump)
 
 
 ### How to work with snapshots?
@@ -542,6 +547,19 @@ the deleted time series isn't freed instantly - it is freed during subsequent me
 It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
 before actually deleting the metrics.
 
+The delete API is intended mainly for the following cases:
+
+- One-off deleting of accidentally written invalid (or undesired) time series.
+- One-off deleting of user data due to [GDPR](https://en.wikipedia.org/wiki/General_Data_Protection_Regulation).
+
+It isn't recommended using delete API for the following cases, since it brings non-zero overhead:
+
+- Regular cleanups for unneded data. Just prevent writing unneeded data into VictoriaMetrics.
+- Reducing disk space usage by deleting unneded time series. This doesn't work as expected, since the deleted
+  time series occupy disk space until the next merge operation, which can never occur.
+
+It is better using `-retentionPeriod` command-line flag for efficient pruning of old data.
+
 
 ### How to export time series?
 
@@ -625,7 +643,7 @@ A rough estimation of the required resources for ingestion path:
 * RAM size: less than 1KB per active time series. So, ~1GB of RAM is required for 1M active time series.
   Time series is considered active if new data points have been added to it recently or if it has been recently queried.
   The number of active time series may be obtained from `vm_cache_entries{type="storage/hour_metric_ids"}` metric
-  exproted on the `/metrics` page.
+  exported on the `/metrics` page.
   VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited by `-memory.allowedPercent` flag.
 
 * CPU cores: a CPU core per 300K inserted data points per second. So, ~4 CPU cores are required for processing
@@ -706,7 +724,7 @@ There is no downsampling support at the moment, but:
 - VictoriaMetrics has good compression for on-disk data. See [this article](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
   for details.
 
-These properties reduce the need in downsampling. We plan to implement downsampling in the future.
+These properties reduce the need of downsampling. We plan to implement downsampling in the future.
 See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/36) for details.
 
 
@@ -728,8 +746,10 @@ horizontally scalable long-term remote storage for really large Prometheus deplo
 
 ### Alerting
 
-VictoriaMetrics doesn't support rule evaluation and alerting yet, so these actions must be performed either
-on [Prometheus side](https://prometheus.io/docs/alerting/overview/) or on [Grafana side](https://grafana.com/docs/alerting/rules/).
+VictoriaMetrics doesn't support rule evaluation and alerting yet, so these actions can be performed at the following places:
+* At Prometheus - see [the corresponding docs](https://prometheus.io/docs/alerting/overview/).
+* At Promxy - see [the corresponding docs](https://github.com/jacksontj/promxy/blob/master/README.md#how-do-i-use-alertingrecording-rules-in-promxy).
+* At Grafana - see [the corresponding docs](https://grafana.com/docs/alerting/rules/).
 
 
 ### Security
@@ -749,14 +769,14 @@ For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<i
 
 ### Tuning
 
-* There is no need in VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
+* There is no need for VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
   which are automatically adjusted for the available CPU and RAM resources.
-* There is no need in Operating System tuning since VictoriaMetrics is optimized for default OS settings.
+* There is no need for Operating System tuning since VictoriaMetrics is optimized for default OS settings.
   The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a),
   so Prometheus instances could establish more connections to VictoriaMetrics.
 * The recommended filesystem is `ext4`, the recommended persistent storage is [persistent HDD-based disk on GCP](https://cloud.google.com/compute/docs/disks/#pdspecs),
   since it is protected from hardware failures via internal replication and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).
-  If you plan storing more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
+  If you plan to store more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
   then the following options are recommended to pass to `mkfs.ext4`:
 
 ```
@@ -787,7 +807,7 @@ The most interesting metrics are:
 ### Troubleshooting
 
 * It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
-  in tweaking these flag values arises.
+  of tweaking these flag values arises.
 
 * If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
   then it is likely you have too many active time series for the current amount of RAM.
@@ -801,13 +821,14 @@ The most interesting metrics are:
   has at least 20% of free space comparing to disk size.
 
 * If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
-  then just remove directoreis with broken parts. This will recover VictoriaMetrics at the cost
+  then just remove directories with broken parts. This will recover VictoriaMetrics at the cost
   of data loss stored in the broken parts. In the future, `vmrecover` tool will be created
   for automatic recovering from such errors.
 
 
 ### Backfilling
 
+VictoriaMetrics accepts historical data in arbitrary order of time.
 Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.
 
 It is recommended disabling query cache with `-search.disableCache` command-line flag when writing
@@ -852,7 +873,7 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
 - [ ] CLI tool for data migration, re-balancing and adding/removing nodes [#103](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/103)
 
 
-The discussion happens [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/129). Feel free to comment any item or add own one.
+The discussion happens [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/129). Feel free to comment on any item or add you own one.
 
 
 ## Contacts
@@ -865,6 +886,7 @@ Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics
 Feel free asking any questions regarding VictoriaMetrics:
 
 - [slack](http://slack.victoriametrics.com/)
+- [reddit](https://www.reddit.com/r/VictoriaMetrics/)
 - [telegram-en](https://t.me/VictoriaMetrics_en)
 - [telegram-ru](https://t.me/VictoriaMetrics_ru1)
 - [google groups](https://groups.google.com/forum/#!forum/victorametrics-users)
@@ -895,7 +917,7 @@ Report bugs and propose new features [here](https://github.com/VictoriaMetrics/V
 
 ## Victoria Metrics Logo
 
-[Zip](VM_logo.zip) contains three folders with different image orientation (main color and inverted version).
+[Zip](VM_logo.zip) contains three folders with different image orientations (main color and inverted version).
 
 Files included in each folder:
 

app/victoria-metrics/main.go

@@ -28,7 +28,7 @@ func main() {
 	vminsert.Init()
 
 	go httpserver.Serve(*httpListenAddr, requestHandler)
-	logger.Infof("started VictoriaMetrics in %s", time.Since(startTime))
+	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
@@ -39,14 +39,14 @@ func main() {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
-	logger.Infof("successfully shut down the webservice in %s", time.Since(startTime))
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	vmstorage.Stop()
 	vmselect.Stop()
 
 	fs.MustStopDirRemover()
 
-	logger.Infof("the VictoriaMetrics has been stopped in %s", time.Since(startTime))
+	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -13,11 +13,8 @@
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_set"},"values":[
 	      ["{TIME_S-150s}","4"],
-	      ["{TIME_S-140s}","4"],
 	      ["{TIME_S-120s}","3"],
-	      ["{TIME_S-110s}","3"],
 	      ["{TIME_S-60s}","2"],
-	      ["{TIME_S-50s}","2"],
 	      ["{TIME_S-30s}","1"],
 	      ["{TIME_S-20s}","1"]
       ]}]}}

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -19,14 +19,12 @@
 	      ["{TIME_S-110s}","3"],
 	      ["{TIME_S-100s}","3"],
 	      ["{TIME_S-90s}","3"],
-	      ["{TIME_S-80s}","3"],
-	      ["{TIME_S-70s}","3"],
 	      ["{TIME_S-60s}","2"],
 	      ["{TIME_S-50s}","2"],
 	      ["{TIME_S-40s}","2"],
 	      ["{TIME_S-30s}","1"],
 	      ["{TIME_S-20s}","1"],
 	      ["{TIME_S-10s}","1"],
-	      ["{TIME_S}","1"]
+	      ["{TIME_S-0s}","1"]
       ]}]}}
 }

app/vminsert/common/gzip_reader.go

@@ -1,9 +1,10 @@
 package common
 
 import (
-	"compress/gzip"
 	"io"
 	"sync"
+
+	"github.com/klauspost/compress/gzip"
 )
 
 // GetGzipReader returns new gzip reader from the pool.

app/vminsert/concurrencylimiter/concurrencylimiter.go

@@ -12,17 +12,14 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-var maxConcurrentInserts = flag.Int("maxConcurrentInserts", runtime.GOMAXPROCS(-1)*4, "The maximum number of concurrent inserts")
-
 var (
-	// ch is the channel for limiting concurrent calls to Do.
-	ch chan struct{}
-
-	// waitDuration is the amount of time to wait until at least a single
-	// concurrent Do call out of cap(ch) inserts is complete.
-	waitDuration = time.Second * 30
+	maxConcurrentInserts = flag.Int("maxConcurrentInserts", runtime.GOMAXPROCS(-1)*4, "The maximum number of concurrent inserts; see also `-insert.maxQueueDuration`")
+	maxQueueDuration     = flag.Duration("insert.maxQueueDuration", time.Minute, "The maximum duration for waiting in the queue for insert requests due to `-maxConcurrentInserts`")
 )
 
+// ch is the channel for limiting concurrent calls to Do.
+var ch chan struct{}
+
 // Init initializes concurrencylimiter.
 //
 // Init must be called after flag.Parse call.
@@ -43,9 +40,9 @@ func Do(f func() error) error {
 	}
 
 	// All the workers are busy.
-	// Sleep for up to waitDuration.
+	// Sleep for up to *maxQueueDuration.
 	concurrencyLimitReached.Inc()
-	t := timerpool.Get(waitDuration)
+	t := timerpool.Get(*maxQueueDuration)
 	select {
 	case ch <- struct{}{}:
 		timerpool.Put(t)
@@ -56,7 +53,9 @@ func Do(f func() error) error {
 		timerpool.Put(t)
 		concurrencyLimitTimeout.Inc()
 		return &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase -maxConcurrentInserts or reduce the load", cap(ch)),
+			Err: fmt.Errorf("cannot handle more than %d concurrent inserts during %s; possible solutions: "+
+				"increase `-insert.maxQueueDuration`, increase `-maxConcurrentInserts`, "+
+				"decrease `-search.maxConcurrentRequests`, increase server capacity", *maxConcurrentInserts, *maxQueueDuration),
 			StatusCode: http.StatusServiceUnavailable,
 		}
 	}

app/vminsert/graphite/request_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -106,7 +107,7 @@ func (ctx *pushCtx) Read(r io.Reader) bool {
 }
 
 type pushCtx struct {
-	Rows   Rows
+	Rows   graphite.Rows
 	Common common.InsertCtx
 
 	reqBuf  []byte

app/vminsert/influx/request_handler.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -171,7 +172,7 @@ var (
 )
 
 type pushCtx struct {
-	Rows   Rows
+	Rows   influx.Rows
 	Common common.InsertCtx
 
 	reqBuf         []byte

app/vminsert/main.go

@@ -24,7 +24,6 @@ var (
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
-	maxInsertRequestSize   = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
 )
 
@@ -43,10 +42,10 @@ func Init() {
 		graphiteServer = graphite.MustStart(*graphiteListenAddr)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdbServer = opentsdb.MustStart(*opentsdbListenAddr, int64(*maxInsertRequestSize))
+		opentsdbServer = opentsdb.MustStart(*opentsdbListenAddr)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		opentsdbhttpServer = opentsdbhttp.MustStart(*opentsdbHTTPListenAddr, int64(*maxInsertRequestSize))
+		opentsdbhttpServer = opentsdbhttp.MustStart(*opentsdbHTTPListenAddr)
 	}
 }
 
@@ -69,7 +68,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := prometheus.InsertHandler(r, int64(*maxInsertRequestSize)); err != nil {
+		if err := prometheus.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true

app/vminsert/opentsdb/request_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -105,7 +106,7 @@ func (ctx *pushCtx) Read(r io.Reader) bool {
 }
 
 type pushCtx struct {
-	Rows   Rows
+	Rows   opentsdb.Rows
 	Common common.InsertCtx
 
 	reqBuf  []byte

app/vminsert/opentsdb/server.go

@@ -36,7 +36,7 @@ type Server struct {
 // MustStart starts OpenTSDB collector on the given addr.
 //
 // MustStop must be called on the returned server when it is no longer needed.
-func MustStart(addr string, maxRequestSize int64) *Server {
+func MustStart(addr string) *Server {
 	logger.Infof("starting TCP OpenTSDB collector at %q", addr)
 	lnTCP, err := netutil.NewTCPListener("opentsdb", addr)
 	if err != nil {
@@ -45,7 +45,7 @@ func MustStart(addr string, maxRequestSize int64) *Server {
 	ls := newListenerSwitch(lnTCP)
 	lnHTTP := ls.newHTTPListener()
 	lnTelnet := ls.newTelnetListener()
-	httpServer := opentsdbhttp.MustServe(lnHTTP, maxRequestSize)
+	httpServer := opentsdbhttp.MustServe(lnHTTP)
 
 	logger.Infof("starting UDP OpenTSDB collector at %q", addr)
 	lnUDP, err := net.ListenPacket("udp4", addr)

app/vminsert/opentsdbhttp/request_handler.go

@@ -1,6 +1,7 @@
 package opentsdbhttp
 
 import (
+	"flag"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,10 +12,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
 )
 
+var maxInsertRequestSize = flag.Int("opentsdbhttp.maxInsertRequestSize", 32*1024*1024, "The maximum size of OpenTSDB HTTP put request")
+
 var (
 	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb-http"}`)
 	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="opentsdb-http"}`)
@@ -26,13 +29,13 @@ var (
 
 // insertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
-func insertHandler(req *http.Request, maxSize int64) error {
+func insertHandler(req *http.Request) error {
 	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(req, maxSize)
+		return insertHandlerInternal(req)
 	})
 }
 
-func insertHandlerInternal(req *http.Request, maxSize int64) error {
+func insertHandlerInternal(req *http.Request) error {
 	readCalls.Inc()
 
 	r := req.Body
@@ -50,20 +53,20 @@ func insertHandlerInternal(req *http.Request, maxSize int64) error {
 	defer putPushCtx(ctx)
 
 	// Read the request in ctx.reqBuf
-	lr := io.LimitReader(r, maxSize+1)
+	lr := io.LimitReader(r, int64(*maxInsertRequestSize)+1)
 	reqLen, err := ctx.reqBuf.ReadFrom(lr)
 	if err != nil {
 		readErrors.Inc()
 		return fmt.Errorf("cannot read HTTP OpenTSDB request: %s", err)
 	}
-	if reqLen > maxSize {
+	if reqLen > int64(*maxInsertRequestSize) {
 		readErrors.Inc()
-		return fmt.Errorf("too big HTTP OpenTSDB request; mustn't exceed %d bytes", maxSize)
+		return fmt.Errorf("too big HTTP OpenTSDB request; mustn't exceed `-opentsdbhttp.maxInsertRequestSize=%d` bytes", *maxInsertRequestSize)
 	}
 
 	// Unmarshal the request to ctx.Rows
-	p := parserPool.Get()
-	defer parserPool.Put(p)
+	p := opentsdbhttp.GetParser()
+	defer opentsdbhttp.PutParser(p)
 	v, err := p.ParseBytes(ctx.reqBuf.B)
 	if err != nil {
 		unmarshalErrors.Inc()
@@ -110,10 +113,8 @@ func insertHandlerInternal(req *http.Request, maxSize int64) error {
 
 const secondMask int64 = 0x7FFFFFFF00000000
 
-var parserPool fastjson.ParserPool
-
 type pushCtx struct {
-	Rows   Rows
+	Rows   opentsdbhttp.Rows
 	Common common.InsertCtx
 
 	reqBuf bytesutil.ByteBuffer

app/vminsert/opentsdbhttp/server.go

@@ -28,20 +28,20 @@ type Server struct {
 // MustStart starts HTTP OpenTSDB server on the given addr.
 //
 // MustStop must be called on the returned server when it is no longer needed.
-func MustStart(addr string, maxRequestSize int64) *Server {
+func MustStart(addr string) *Server {
 	logger.Infof("starting HTTP OpenTSDB server at %q", addr)
 	lnTCP, err := netutil.NewTCPListener("opentsdbhttp", addr)
 	if err != nil {
 		logger.Fatalf("cannot start HTTP OpenTSDB collector at %q: %s", addr, err)
 	}
-	return MustServe(lnTCP, maxRequestSize)
+	return MustServe(lnTCP)
 }
 
-// MustServe serves OpenTSDB HTTP put requests from ln with up to maxRequestSize size.
+// MustServe serves OpenTSDB HTTP put requests from ln.
 //
 // MustStop must be called on the returned server when it is no longer needed.
-func MustServe(ln net.Listener, maxRequestSize int64) *Server {
-	h := newRequestHandler(maxRequestSize)
+func MustServe(ln net.Listener) *Server {
+	h := newRequestHandler()
 	hs := &http.Server{
 		Handler:      h,
 		ReadTimeout:  30 * time.Second,
@@ -82,12 +82,12 @@ func (s *Server) MustStop() {
 	logger.Infof("OpenTSDB HTTP server at %q has been stopped", s.ln.Addr())
 }
 
-func newRequestHandler(maxRequestSize int64) http.Handler {
+func newRequestHandler() http.Handler {
 	rh := func(w http.ResponseWriter, r *http.Request) {
 		switch r.URL.Path {
 		case "/api/put":
 			writeRequests.Inc()
-			if err := insertHandler(r, maxRequestSize); err != nil {
+			if err := insertHandler(r); err != nil {
 				writeErrors.Inc()
 				httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 				return

app/vminsert/prometheus/request_handler.go

@@ -18,16 +18,16 @@ var (
 )
 
 // InsertHandler processes remote write for prometheus.
-func InsertHandler(r *http.Request, maxSize int64) error {
+func InsertHandler(r *http.Request) error {
 	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r, maxSize)
+		return insertHandlerInternal(r)
 	})
 }
 
-func insertHandlerInternal(r *http.Request, maxSize int64) error {
+func insertHandlerInternal(r *http.Request) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
-	if err := ctx.Read(r, maxSize); err != nil {
+	if err := ctx.Read(r); err != nil {
 		return err
 	}
 	timeseries := ctx.req.Timeseries
@@ -65,11 +65,11 @@ func (ctx *pushCtx) reset() {
 	ctx.reqBuf = ctx.reqBuf[:0]
 }
 
-func (ctx *pushCtx) Read(r *http.Request, maxSize int64) error {
+func (ctx *pushCtx) Read(r *http.Request) error {
 	prometheusReadCalls.Inc()
 
 	var err error
-	ctx.reqBuf, err = prompb.ReadSnappy(ctx.reqBuf[:0], r.Body, maxSize)
+	ctx.reqBuf, err = prompb.ReadSnappy(ctx.reqBuf[:0], r.Body)
 	if err != nil {
 		prometheusReadErrors.Inc()
 		return fmt.Errorf("cannot read prompb.WriteRequest: %s", err)

app/vminsert/vmimport/parser_test.go

@@ -1,6 +1,7 @@
 package vmimport
 
 import (
+	"math"
 	"reflect"
 	"testing"
 )
@@ -101,6 +102,18 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		}},
 	})
 
+	// Inf and nan values
+	f(`{"metric":{"foo":"bar"},"values":[Inf, -Inf],"timestamps":[456, 789]}`, &Rows{
+		Rows: []Row{{
+			Tags: []Tag{{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			}},
+			Values:     []float64{math.Inf(1), math.Inf(-1)},
+			Timestamps: []int64{456, 789},
+		}},
+	})
+
 	// Line with multiple tags
 	f(`{"metric":{"foo":"bar","baz":"xx"},"values":[1.23, -3.21],"timestamps" : [456,789]}`, &Rows{
 		Rows: []Row{{

app/vmrestore/main.go

@@ -16,8 +16,9 @@ var (
 		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
 	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
 		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup")
-	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
-	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
+	concurrency             = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
+	maxBytesPerSecond       = flag.Int("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
+	skipBackupCompleteCheck = flag.Bool("skipBackupCompleteCheck", false, "Whether to skip checking for `backup complete` file in `-src`. This may be useful for restoring from old backups, which were created without `backup complete` file")
 )
 
 func main() {
@@ -34,9 +35,10 @@ func main() {
 		logger.Fatalf("%s", err)
 	}
 	a := &actions.Restore{
-		Concurrency: *concurrency,
-		Src:         srcFS,
-		Dst:         dstFS,
+		Concurrency:             *concurrency,
+		Src:                     srcFS,
+		Dst:                     dstFS,
+		SkipBackupCompleteCheck: *skipBackupCompleteCheck,
 	}
 	if err := a.Run(); err != nil {
 		logger.Fatalf("cannot restore from backup: %s", err)

app/vmselect/main.go

@@ -21,10 +21,25 @@ import (
 
 var (
 	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series")
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", runtime.GOMAXPROCS(-1)*2, "The maximum number of concurrent search requests. It shouldn't exceed 2*vCPUs for better performance. See also -search.maxQueueDuration")
-	maxQueueDuration      = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
+		"It shouldn't be high, since a single request can saturate all the CPU cores. See also `-search.maxQueueDuration`")
+	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when `-search.maxConcurrentRequests` limit is reached")
 )
 
+func getDefaultMaxConcurrentRequests() int {
+	n := runtime.GOMAXPROCS(-1)
+	if n <= 4 {
+		n *= 2
+	}
+	if n > 16 {
+		// A single request can saturate all the CPU cores, so there is no sense
+		// in allowing higher number of concurrent requests - they will just contend
+		// for unavailable CPU time.
+		n = 16
+	}
+	return n
+}
+
 // Init initializes vmselect
 func Init() {
 	tmpDirPath := *vmstorage.DataPath + "/tmp"
@@ -72,7 +87,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			timerpool.Put(t)
 			concurrencyLimitTimeout.Inc()
 			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("cannot handle more than %d concurrent requests", cap(concurrencyCh)),
+				Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
+					"increase `-search.maxQueueDuration`, increase `-search.maxConcurrentRequests`, increase server capacity",
+					*maxConcurrentRequests, *maxQueueDuration),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, "%s", err)
@@ -199,7 +216,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
-	logger.Errorf("error in %q: %s", r.URL.Path, err)
+	logger.Errorf("error in %q: %s", r.RequestURI, err)
 
 	w.Header().Set("Content-Type", "application/json")
 	statusCode := http.StatusUnprocessableEntity

app/vmselect/netstorage/netstorage.go

@@ -103,7 +103,7 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 			rowsProcessed := 0
 			for pts := range workCh {
 				if time.Until(rss.deadline.Deadline) < 0 {
-					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.Timeout)
+					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
 					break
 				}
 				if err = pts.Unpack(rss.tbf, rs, rss.tr, rss.fetchData, maxWorkersCount); err != nil {
@@ -499,7 +499,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 		}
 		if time.Until(deadline.Deadline) < 0 {
 			putTmpBlocksFile(tbf)
-			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.Timeout)
+			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
 		metricName := sr.MetricBlock.MetricName
 		m[string(metricName)] = append(m[string(metricName)], addr)
@@ -575,13 +575,24 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
 // Deadline contains deadline with the corresponding timeout for pretty error messages.
 type Deadline struct {
 	Deadline time.Time
-	Timeout  time.Duration
+
+	timeout  time.Duration
+	flagHint string
 }
 
 // NewDeadline returns deadline for the given timeout.
-func NewDeadline(timeout time.Duration) Deadline {
+//
+// flagHint must contain a hit for command-line flag, which could be used
+// in order to increase timeout.
+func NewDeadline(timeout time.Duration, flagHint string) Deadline {
 	return Deadline{
 		Deadline: time.Now().Add(timeout),
-		Timeout:  timeout,
+		timeout:  timeout,
+		flagHint: flagHint,
 	}
 }
+
+// String returns human-readable string representation for d.
+func (d *Deadline) String() string {
+	return fmt.Sprintf("%.3f seconds; the timeout can be adjusted with `%s` command-line flag", d.timeout.Seconds(), d.flagHint)
+}

app/vmselect/netstorage/tmp_blocks_file.go

@@ -36,6 +36,9 @@ func maxInmemoryTmpBlocksFile() int {
 	if maxLen < 64*1024 {
 		return 64 * 1024
 	}
+	if maxLen > 4*1024*1024 {
+		return 4 * 1024 * 1024
+	}
 	return maxLen
 }
 

app/vmselect/prometheus/prometheus.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/quicktemplate"
@@ -23,7 +24,7 @@ import (
 var (
 	latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the colection. "+
 		"Too small value can result in incomplete last points for query results")
-	maxExportDuration = flag.Duration("search.maxExportDuration", 10*time.Minute, "The maximum duration for `/api/v1/export` call")
+	maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for `/api/v1/export` call")
 	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for search query execution")
 	maxQueryLen       = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
 	maxLookback       = flag.Duration("search.maxLookback", 0, "Synonim to `-search.lookback-delta` from Prometheus. "+
@@ -135,7 +136,7 @@ func ExportHandler(w http.ResponseWriter, r *http.Request) error {
 		end = start + defaultStep
 	}
 	if err := exportHandler(w, matches, start, end, format, deadline); err != nil {
-		return err
+		return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %s", matches, start, end, err)
 	}
 	exportDuration.UpdateDuration(startTime)
 	return nil
@@ -559,7 +560,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	}
 
 	if len(query) > *maxQueryLen {
-		return fmt.Errorf(`too long query; got %d bytes; mustn't exceed %d bytes`, len(query), *maxQueryLen)
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), *maxQueryLen)
 	}
 	if !getBool(r, "nocache") && ct-start < queryOffset {
 		// Adjust start time only if `nocache` arg isn't set.
@@ -579,7 +580,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 		end := start
 		start = end - window
 		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", deadline); err != nil {
-			return err
+			return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %s", childQuery, start, end, err)
 		}
 		queryDuration.UpdateDuration(startTime)
 		return nil
@@ -604,7 +605,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 		end := start
 		start = end - window
 		if err := queryRangeHandler(w, childQuery, start, end, step, r, ct); err != nil {
-			return err
+			return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %s", childQuery, start, end, step, err)
 		}
 		queryDuration.UpdateDuration(startTime)
 		return nil
@@ -619,7 +620,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	}
 	result, err := promql.Exec(&ec, query, true)
 	if err != nil {
-		return fmt.Errorf("cannot execute %q: %s", query, err)
+		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %s", query, start, step, err)
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -634,14 +635,14 @@ func parseDuration(s string, step int64) (int64, error) {
 	if len(s) == 0 {
 		return 0, nil
 	}
-	return promql.DurationValue(s, step)
+	return metricsql.DurationValue(s, step)
 }
 
 func parsePositiveDuration(s string, step int64) (int64, error) {
 	if len(s) == 0 {
 		return 0, nil
 	}
-	return promql.PositiveDurationValue(s, step)
+	return metricsql.PositiveDurationValue(s, step)
 }
 
 // QueryRangeHandler processes /api/v1/query_range request.
@@ -668,7 +669,7 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 	if err := queryRangeHandler(w, query, start, end, step, r, ct); err != nil {
-		return err
+		return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %s", query, start, end, step, err)
 	}
 	queryRangeDuration.UpdateDuration(startTime)
 	return nil
@@ -684,7 +685,7 @@ func queryRangeHandler(w http.ResponseWriter, query string, start, end, step int
 
 	// Validate input args.
 	if len(query) > *maxQueryLen {
-		return fmt.Errorf(`too long query; got %d bytes; mustn't exceed %d bytes`, len(query), *maxQueryLen)
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), *maxQueryLen)
 	}
 	if start > end {
 		end = start + defaultStep
@@ -706,7 +707,7 @@ func queryRangeHandler(w http.ResponseWriter, query string, start, end, step int
 	}
 	result, err := promql.Exec(&ec, query, false)
 	if err != nil {
-		return fmt.Errorf("cannot execute %q: %s", query, err)
+		return fmt.Errorf("cannot execute query: %s", err)
 	}
 	queryOffset := getLatencyOffsetMilliseconds()
 	if ct-end < queryOffset {
@@ -869,15 +870,15 @@ func getMaxLookback(r *http.Request) (int64, error) {
 
 func getDeadlineForQuery(r *http.Request) netstorage.Deadline {
 	dMax := int64(maxQueryDuration.Seconds() * 1e3)
-	return getDeadlineWithMaxDuration(r, dMax)
+	return getDeadlineWithMaxDuration(r, dMax, "-search.maxQueryDuration")
 }
 
 func getDeadlineForExport(r *http.Request) netstorage.Deadline {
 	dMax := int64(maxExportDuration.Seconds() * 1e3)
-	return getDeadlineWithMaxDuration(r, dMax)
+	return getDeadlineWithMaxDuration(r, dMax, "-search.maxExportDuration")
 }
 
-func getDeadlineWithMaxDuration(r *http.Request, dMax int64) netstorage.Deadline {
+func getDeadlineWithMaxDuration(r *http.Request, dMax int64, flagHint string) netstorage.Deadline {
 	d, err := getDuration(r, "timeout", 0)
 	if err != nil {
 		d = 0
@@ -886,7 +887,7 @@ func getDeadlineWithMaxDuration(r *http.Request, dMax int64) netstorage.Deadline
 		d = dMax
 	}
 	timeout := time.Duration(d) * time.Millisecond
-	return netstorage.NewDeadline(timeout)
+	return netstorage.NewDeadline(timeout, flagHint)
 }
 
 func getBool(r *http.Request, argKey string) bool {

app/vmselect/promql/aggr.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/histogram"
@@ -48,7 +49,7 @@ type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error)
 
 type aggrFuncArg struct {
 	args [][]*timeseries
-	ae   *aggrFuncExpr
+	ae   *metricsql.AggrFuncExpr
 	ec   *EvalConfig
 }
 
@@ -57,20 +58,6 @@ func getAggrFunc(s string) aggrFunc {
 	return aggrFuncs[s]
 }
 
-func isAggrFunc(s string) bool {
-	return getAggrFunc(s) != nil
-}
-
-func isAggrFuncModifier(s string) bool {
-	s = strings.ToLower(s)
-	switch s {
-	case "by", "without":
-		return true
-	default:
-		return false
-	}
-}
-
 func newAggrFunc(afe func(tss []*timeseries) []*timeseries) aggrFunc {
 	return func(afa *aggrFuncArg) ([]*timeseries, error) {
 		args := afa.args
@@ -81,7 +68,7 @@ func newAggrFunc(afe func(tss []*timeseries) []*timeseries) aggrFunc {
 	}
 }
 
-func removeGroupTags(metricName *storage.MetricName, modifier *modifierExpr) {
+func removeGroupTags(metricName *storage.MetricName, modifier *metricsql.ModifierExpr) {
 	groupOp := strings.ToLower(modifier.Op)
 	switch groupOp {
 	case "", "by":
@@ -93,7 +80,7 @@ func removeGroupTags(metricName *storage.MetricName, modifier *modifierExpr) {
 	}
 }
 
-func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeseries, modifier *modifierExpr, keepOriginal bool) ([]*timeseries, error) {
+func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeseries, modifier *metricsql.ModifierExpr, keepOriginal bool) ([]*timeseries, error) {
 	arg := copyTimeseriesMetricNames(argOrig)
 
 	// Perform grouping.

app/vmselect/promql/aggr_incremental.go

@@ -4,10 +4,12 @@ import (
 	"math"
 	"strings"
 	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 )
 
 // callbacks for optimized incremental calculations for aggregate functions
-// over rollups over metricExpr.
+// over rollups over metricsql.MetricExpr.
 //
 // These calculations save RAM for aggregates over big number of time series.
 var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{
@@ -49,7 +51,7 @@ var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{
 }
 
 type incrementalAggrFuncContext struct {
-	ae *aggrFuncExpr
+	ae *metricsql.AggrFuncExpr
 
 	mLock sync.Mutex
 	m     map[uint]map[string]*incrementalAggrContext
@@ -57,7 +59,7 @@ type incrementalAggrFuncContext struct {
 	callbacks *incrementalAggrFuncCallbacks
 }
 
-func newIncrementalAggrFuncContext(ae *aggrFuncExpr, callbacks *incrementalAggrFuncCallbacks) *incrementalAggrFuncContext {
+func newIncrementalAggrFuncContext(ae *metricsql.AggrFuncExpr, callbacks *incrementalAggrFuncCallbacks) *incrementalAggrFuncContext {
 	return &incrementalAggrFuncContext{
 		ae:        ae,
 		m:         make(map[uint]map[string]*incrementalAggrContext),

app/vmselect/promql/aggr_incremental_test.go

@@ -7,6 +7,8 @@ import (
 	"runtime"
 	"sync"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 )
 
 func TestIncrementalAggr(t *testing.T) {
@@ -42,7 +44,7 @@ func TestIncrementalAggr(t *testing.T) {
 	f := func(name string, valuesExpected []float64) {
 		t.Helper()
 		callbacks := getIncrementalAggrFuncCallbacks(name)
-		ae := &aggrFuncExpr{
+		ae := &metricsql.AggrFuncExpr{
 			Name: name,
 		}
 		tssExpected := []*timeseries{{

app/vmselect/promql/binary_op.go

@@ -6,24 +6,26 @@ import (
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql/binaryop"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 var binaryOpFuncs = map[string]binaryOpFunc{
-	"+": newBinaryOpArithFunc(binaryOpPlus),
-	"-": newBinaryOpArithFunc(binaryOpMinus),
-	"*": newBinaryOpArithFunc(binaryOpMul),
-	"/": newBinaryOpArithFunc(binaryOpDiv),
-	"%": newBinaryOpArithFunc(binaryOpMod),
-	"^": newBinaryOpArithFunc(binaryOpPow),
+	"+": newBinaryOpArithFunc(binaryop.Plus),
+	"-": newBinaryOpArithFunc(binaryop.Minus),
+	"*": newBinaryOpArithFunc(binaryop.Mul),
+	"/": newBinaryOpArithFunc(binaryop.Div),
+	"%": newBinaryOpArithFunc(binaryop.Mod),
+	"^": newBinaryOpArithFunc(binaryop.Pow),
 
 	// cmp ops
-	"==": newBinaryOpCmpFunc(binaryOpEq),
-	"!=": newBinaryOpCmpFunc(binaryOpNeq),
-	">":  newBinaryOpCmpFunc(binaryOpGt),
-	"<":  newBinaryOpCmpFunc(binaryOpLt),
-	">=": newBinaryOpCmpFunc(binaryOpGte),
-	"<=": newBinaryOpCmpFunc(binaryOpLte),
+	"==": newBinaryOpCmpFunc(binaryop.Eq),
+	"!=": newBinaryOpCmpFunc(binaryop.Neq),
+	">":  newBinaryOpCmpFunc(binaryop.Gt),
+	"<":  newBinaryOpCmpFunc(binaryop.Lt),
+	">=": newBinaryOpCmpFunc(binaryop.Gte),
+	"<=": newBinaryOpCmpFunc(binaryop.Lte),
 
 	// logical set ops
 	"and":    binaryOpAnd,
@@ -31,38 +33,9 @@ var binaryOpFuncs = map[string]binaryOpFunc{
 	"unless": binaryOpUnless,
 
 	// New op
-	"if":      newBinaryOpArithFunc(binaryOpIf),
-	"ifnot":   newBinaryOpArithFunc(binaryOpIfnot),
-	"default": newBinaryOpArithFunc(binaryOpDefault),
-}
-
-var binaryOpPriorities = map[string]int{
-	"default": -1,
-
-	"if":    0,
-	"ifnot": 0,
-
-	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence
-	"or": 1,
-
-	"and":    2,
-	"unless": 2,
-
-	"==": 3,
-	"!=": 3,
-	"<":  3,
-	">":  3,
-	"<=": 3,
-	">=": 3,
-
-	"+": 4,
-	"-": 4,
-
-	"*": 5,
-	"/": 5,
-	"%": 5,
-
-	"^": 6,
+	"if":      newBinaryOpArithFunc(binaryop.If),
+	"ifnot":   newBinaryOpArithFunc(binaryop.Ifnot),
+	"default": newBinaryOpArithFunc(binaryop.Default),
 }
 
 func getBinaryOpFunc(op string) binaryOpFunc {
@@ -70,144 +43,8 @@ func getBinaryOpFunc(op string) binaryOpFunc {
 	return binaryOpFuncs[op]
 }
 
-func isBinaryOp(op string) bool {
-	return getBinaryOpFunc(op) != nil
-}
-
-func binaryOpPriority(op string) int {
-	op = strings.ToLower(op)
-	return binaryOpPriorities[op]
-}
-
-func scanBinaryOpPrefix(s string) int {
-	n := 0
-	for op := range binaryOpFuncs {
-		if len(s) < len(op) {
-			continue
-		}
-		ss := strings.ToLower(s[:len(op)])
-		if ss == op && len(op) > n {
-			n = len(op)
-		}
-	}
-	return n
-}
-
-func isRightAssociativeBinaryOp(op string) bool {
-	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence
-	return op == "^"
-}
-
-func isBinaryOpGroupModifier(s string) bool {
-	s = strings.ToLower(s)
-	switch s {
-	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
-	case "on", "ignoring":
-		return true
-	default:
-		return false
-	}
-}
-
-func isBinaryOpJoinModifier(s string) bool {
-	s = strings.ToLower(s)
-	switch s {
-	case "group_left", "group_right":
-		return true
-	default:
-		return false
-	}
-}
-
-func isBinaryOpBoolModifier(s string) bool {
-	s = strings.ToLower(s)
-	return s == "bool"
-}
-
-func isBinaryOpCmp(op string) bool {
-	switch op {
-	case "==", "!=", ">", "<", ">=", "<=":
-		return true
-	default:
-		return false
-	}
-}
-
-func isBinaryOpLogicalSet(op string) bool {
-	op = strings.ToLower(op)
-	switch op {
-	case "and", "or", "unless":
-		return true
-	default:
-		return false
-	}
-}
-
-func binaryOpConstants(op string, left, right float64, isBool bool) float64 {
-	if isBinaryOpCmp(op) {
-		evalCmp := func(cf func(left, right float64) bool) float64 {
-			if isBool {
-				if cf(left, right) {
-					return 1
-				}
-				return 0
-			}
-			if cf(left, right) {
-				return left
-			}
-			return nan
-		}
-		switch op {
-		case "==":
-			left = evalCmp(binaryOpEq)
-		case "!=":
-			left = evalCmp(binaryOpNeq)
-		case ">":
-			left = evalCmp(binaryOpGt)
-		case "<":
-			left = evalCmp(binaryOpLt)
-		case ">=":
-			left = evalCmp(binaryOpGte)
-		case "<=":
-			left = evalCmp(binaryOpLte)
-		default:
-			logger.Panicf("BUG: unexpected comparison binaryOp: %q", op)
-		}
-	} else {
-		switch op {
-		case "+":
-			left = binaryOpPlus(left, right)
-		case "-":
-			left = binaryOpMinus(left, right)
-		case "*":
-			left = binaryOpMul(left, right)
-		case "/":
-			left = binaryOpDiv(left, right)
-		case "%":
-			left = binaryOpMod(left, right)
-		case "^":
-			left = binaryOpPow(left, right)
-		case "and":
-			// Nothing to do
-		case "or":
-			// Nothing to do
-		case "unless":
-			left = nan
-		case "default":
-			left = binaryOpDefault(left, right)
-		case "if":
-			left = binaryOpIf(left, right)
-		case "ifnot":
-			left = binaryOpIfnot(left, right)
-		default:
-			logger.Panicf("BUG: unexpected non-comparison binaryOp: %q", op)
-		}
-	}
-	return left
-}
-
 type binaryOpFuncArg struct {
-	be    *binaryOpExpr
+	be    *metricsql.BinaryOpExpr
 	left  []*timeseries
 	right []*timeseries
 }
@@ -267,7 +104,7 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
-func adjustBinaryOpTags(be *binaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
+func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {
 			// Fast path: `scalar op vector`
@@ -348,7 +185,7 @@ func adjustBinaryOpTags(be *binaryOpExpr, left, right []*timeseries) ([]*timeser
 	return rvsLeft, rvsRight, dst, nil
 }
 
-func ensureSingleTimeseries(side string, be *binaryOpExpr, tss []*timeseries) error {
+func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
 	if len(tss) == 0 {
 		logger.Panicf("BUG: tss must contain at least one value")
 	}
@@ -362,7 +199,7 @@ func ensureSingleTimeseries(side string, be *binaryOpExpr, tss []*timeseries) er
 	return nil
 }
 
-func groupJoin(singleTimeseriesSide string, be *binaryOpExpr, rvsLeft, rvsRight, tssLeft, tssRight []*timeseries) ([]*timeseries, []*timeseries, error) {
+func groupJoin(singleTimeseriesSide string, be *metricsql.BinaryOpExpr, rvsLeft, rvsRight, tssLeft, tssRight []*timeseries) ([]*timeseries, []*timeseries, error) {
 	joinTags := be.JoinModifier.Args
 	var m map[string]*timeseries
 	for _, tsLeft := range tssLeft {
@@ -432,8 +269,8 @@ func mergeNonOverlappingTimeseries(dst, src *timeseries) bool {
 	return true
 }
 
-func resetMetricGroupIfRequired(be *binaryOpExpr, ts *timeseries) {
-	if isBinaryOpCmp(be.Op) && !be.Bool {
+func resetMetricGroupIfRequired(be *metricsql.BinaryOpExpr, ts *timeseries) {
+	if metricsql.IsBinaryOpCmp(be.Op) && !be.Bool {
 		// Do not reset MetricGroup for non-boolean `compare` binary ops like Prometheus does.
 		return
 	}
@@ -445,90 +282,6 @@ func resetMetricGroupIfRequired(be *binaryOpExpr, ts *timeseries) {
 	ts.MetricName.ResetMetricGroup()
 }
 
-func binaryOpPlus(left, right float64) float64 {
-	return left + right
-}
-
-func binaryOpMinus(left, right float64) float64 {
-	return left - right
-}
-
-func binaryOpMul(left, right float64) float64 {
-	return left * right
-}
-
-func binaryOpDiv(left, right float64) float64 {
-	return left / right
-}
-
-func binaryOpMod(left, right float64) float64 {
-	return math.Mod(left, right)
-}
-
-func binaryOpPow(left, right float64) float64 {
-	return math.Pow(left, right)
-}
-
-func binaryOpDefault(left, right float64) float64 {
-	if math.IsNaN(left) {
-		return right
-	}
-	return left
-}
-
-func binaryOpIf(left, right float64) float64 {
-	if math.IsNaN(right) {
-		return nan
-	}
-	return left
-}
-
-func binaryOpIfnot(left, right float64) float64 {
-	if math.IsNaN(right) {
-		return left
-	}
-	return nan
-}
-
-func binaryOpEq(left, right float64) bool {
-	// Special handling for nan == nan.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 .
-	if math.IsNaN(left) {
-		return math.IsNaN(right)
-	}
-
-	return left == right
-}
-
-func binaryOpNeq(left, right float64) bool {
-	// Special handling for comparison with nan.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 .
-	if math.IsNaN(left) {
-		return !math.IsNaN(right)
-	}
-	if math.IsNaN(right) {
-		return true
-	}
-
-	return left != right
-}
-
-func binaryOpGt(left, right float64) bool {
-	return left > right
-}
-
-func binaryOpLt(left, right float64) bool {
-	return left < right
-}
-
-func binaryOpGte(left, right float64) bool {
-	return left >= right
-}
-
-func binaryOpLte(left, right float64) bool {
-	return left <= right
-}
-
 func binaryOpAnd(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries
@@ -565,7 +318,7 @@ func binaryOpUnless(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	return rvs, nil
 }
 
-func createTimeseriesMapByTagSet(be *binaryOpExpr, left, right []*timeseries) (map[string][]*timeseries, map[string][]*timeseries) {
+func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*timeseries) (map[string][]*timeseries, map[string][]*timeseries) {
 	groupTags := be.GroupModifier.Args
 	groupOp := strings.ToLower(be.GroupModifier.Op)
 	if len(groupOp) == 0 {

app/vmselect/promql/eval.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -57,6 +58,14 @@ func AdjustStartEnd(start, end, step int64) (int64, int64) {
 	if adjust > 0 {
 		end += step - adjust
 	}
+
+	// Make sure that the new number of points is the same as the initial number of points.
+	newPoints := (end-start)/step + 1
+	for newPoints > points {
+		end -= step
+		newPoints--
+	}
+
 	return start, end
 }
 
@@ -144,25 +153,25 @@ func getTimestamps(start, end, step int64) []int64 {
 	return timestamps
 }
 
-func evalExpr(ec *EvalConfig, e expr) ([]*timeseries, error) {
-	if me, ok := e.(*metricExpr); ok {
-		re := &rollupExpr{
+func evalExpr(ec *EvalConfig, e metricsql.Expr) ([]*timeseries, error) {
+	if me, ok := e.(*metricsql.MetricExpr); ok {
+		re := &metricsql.RollupExpr{
 			Expr: me,
 		}
-		rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, re, nil)
+		rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, e, re, nil)
 		if err != nil {
 			return nil, fmt.Errorf(`cannot evaluate %q: %s`, me.AppendString(nil), err)
 		}
 		return rv, nil
 	}
-	if re, ok := e.(*rollupExpr); ok {
-		rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, re, nil)
+	if re, ok := e.(*metricsql.RollupExpr); ok {
+		rv, err := evalRollupFunc(ec, "default_rollup", rollupDefault, e, re, nil)
 		if err != nil {
 			return nil, fmt.Errorf(`cannot evaluate %q: %s`, re.AppendString(nil), err)
 		}
 		return rv, nil
 	}
-	if fe, ok := e.(*funcExpr); ok {
+	if fe, ok := e.(*metricsql.FuncExpr); ok {
 		nrf := getRollupFunc(fe.Name)
 		if nrf == nil {
 			args, err := evalExprs(ec, fe.Args)
@@ -192,17 +201,17 @@ func evalExpr(ec *EvalConfig, e expr) ([]*timeseries, error) {
 		if err != nil {
 			return nil, err
 		}
-		rv, err := evalRollupFunc(ec, fe.Name, rf, re, nil)
+		rv, err := evalRollupFunc(ec, fe.Name, rf, e, re, nil)
 		if err != nil {
 			return nil, fmt.Errorf(`cannot evaluate %q: %s`, fe.AppendString(nil), err)
 		}
 		return rv, nil
 	}
-	if ae, ok := e.(*aggrFuncExpr); ok {
+	if ae, ok := e.(*metricsql.AggrFuncExpr); ok {
 		if callbacks := getIncrementalAggrFuncCallbacks(ae.Name); callbacks != nil {
 			fe, nrf := tryGetArgRollupFuncWithMetricExpr(ae)
 			if fe != nil {
-				// There is an optimized path for calculating aggrFuncExpr over rollupFunc over metricExpr.
+				// There is an optimized path for calculating metricsql.AggrFuncExpr over rollupFunc over metricsql.MetricExpr.
 				// The optimized path saves RAM for aggregates over big number of time series.
 				args, re, err := evalRollupFuncArgs(ec, fe)
 				if err != nil {
@@ -213,7 +222,7 @@ func evalExpr(ec *EvalConfig, e expr) ([]*timeseries, error) {
 					return nil, err
 				}
 				iafc := newIncrementalAggrFuncContext(ae, callbacks)
-				return evalRollupFunc(ec, fe.Name, rf, re, iafc)
+				return evalRollupFunc(ec, fe.Name, rf, e, re, iafc)
 			}
 		}
 		args, err := evalExprs(ec, ae.Args)
@@ -235,7 +244,7 @@ func evalExpr(ec *EvalConfig, e expr) ([]*timeseries, error) {
 		}
 		return rv, nil
 	}
-	if be, ok := e.(*binaryOpExpr); ok {
+	if be, ok := e.(*metricsql.BinaryOpExpr); ok {
 		left, err := evalExpr(ec, be.Left)
 		if err != nil {
 			return nil, err
@@ -259,18 +268,18 @@ func evalExpr(ec *EvalConfig, e expr) ([]*timeseries, error) {
 		}
 		return rv, nil
 	}
-	if ne, ok := e.(*numberExpr); ok {
+	if ne, ok := e.(*metricsql.NumberExpr); ok {
 		rv := evalNumber(ec, ne.N)
 		return rv, nil
 	}
-	if se, ok := e.(*stringExpr); ok {
+	if se, ok := e.(*metricsql.StringExpr); ok {
 		rv := evalString(ec, se.S)
 		return rv, nil
 	}
 	return nil, fmt.Errorf("unexpected expression %q", e.AppendString(nil))
 }
 
-func tryGetArgRollupFuncWithMetricExpr(ae *aggrFuncExpr) (*funcExpr, newRollupFunc) {
+func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.FuncExpr, newRollupFunc) {
 	if len(ae.Args) != 1 {
 		return nil, nil
 	}
@@ -281,31 +290,31 @@ func tryGetArgRollupFuncWithMetricExpr(ae *aggrFuncExpr) (*funcExpr, newRollupFu
 	// - rollupFunc(metricExpr)
 	// - rollupFunc(metricExpr[d])
 
-	if me, ok := e.(*metricExpr); ok {
+	if me, ok := e.(*metricsql.MetricExpr); ok {
 		// e = metricExpr
 		if me.IsEmpty() {
 			return nil, nil
 		}
-		fe := &funcExpr{
+		fe := &metricsql.FuncExpr{
 			Name: "default_rollup",
-			Args: []expr{me},
+			Args: []metricsql.Expr{me},
 		}
 		nrf := getRollupFunc(fe.Name)
 		return fe, nrf
 	}
-	if re, ok := e.(*rollupExpr); ok {
-		if me, ok := re.Expr.(*metricExpr); !ok || me.IsEmpty() || re.ForSubquery() {
+	if re, ok := e.(*metricsql.RollupExpr); ok {
+		if me, ok := re.Expr.(*metricsql.MetricExpr); !ok || me.IsEmpty() || re.ForSubquery() {
 			return nil, nil
 		}
 		// e = metricExpr[d]
-		fe := &funcExpr{
+		fe := &metricsql.FuncExpr{
 			Name: "default_rollup",
-			Args: []expr{re},
+			Args: []metricsql.Expr{re},
 		}
 		nrf := getRollupFunc(fe.Name)
 		return fe, nrf
 	}
-	fe, ok := e.(*funcExpr)
+	fe, ok := e.(*metricsql.FuncExpr)
 	if !ok {
 		return nil, nil
 	}
@@ -314,19 +323,23 @@ func tryGetArgRollupFuncWithMetricExpr(ae *aggrFuncExpr) (*funcExpr, newRollupFu
 		return nil, nil
 	}
 	rollupArgIdx := getRollupArgIdx(fe.Name)
+	if rollupArgIdx >= len(fe.Args) {
+		// Incorrect number of args for rollup func.
+		return nil, nil
+	}
 	arg := fe.Args[rollupArgIdx]
-	if me, ok := arg.(*metricExpr); ok {
+	if me, ok := arg.(*metricsql.MetricExpr); ok {
 		if me.IsEmpty() {
 			return nil, nil
 		}
 		// e = rollupFunc(metricExpr)
-		return &funcExpr{
+		return &metricsql.FuncExpr{
 			Name: fe.Name,
-			Args: []expr{me},
+			Args: []metricsql.Expr{me},
 		}, nrf
 	}
-	if re, ok := arg.(*rollupExpr); ok {
-		if me, ok := re.Expr.(*metricExpr); !ok || me.IsEmpty() || re.ForSubquery() {
+	if re, ok := arg.(*metricsql.RollupExpr); ok {
+		if me, ok := re.Expr.(*metricsql.MetricExpr); !ok || me.IsEmpty() || re.ForSubquery() {
 			return nil, nil
 		}
 		// e = rollupFunc(metricExpr[d])
@@ -335,7 +348,7 @@ func tryGetArgRollupFuncWithMetricExpr(ae *aggrFuncExpr) (*funcExpr, newRollupFu
 	return nil, nil
 }
 
-func evalExprs(ec *EvalConfig, es []expr) ([][]*timeseries, error) {
+func evalExprs(ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
 	var rvs [][]*timeseries
 	for _, e := range es {
 		rv, err := evalExpr(ec, e)
@@ -347,9 +360,12 @@ func evalExprs(ec *EvalConfig, es []expr) ([][]*timeseries, error) {
 	return rvs, nil
 }
 
-func evalRollupFuncArgs(ec *EvalConfig, fe *funcExpr) ([]interface{}, *rollupExpr, error) {
-	var re *rollupExpr
+func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
+	var re *metricsql.RollupExpr
 	rollupArgIdx := getRollupArgIdx(fe.Name)
+	if len(fe.Args) <= rollupArgIdx {
+		return nil, nil, fmt.Errorf("expecting at least %d args to %q; got %d args; expr: %q", rollupArgIdx+1, fe.Name, len(fe.Args), fe.AppendString(nil))
+	}
 	args := make([]interface{}, len(fe.Args))
 	for i, arg := range fe.Args {
 		if i == rollupArgIdx {
@@ -366,11 +382,11 @@ func evalRollupFuncArgs(ec *EvalConfig, fe *funcExpr) ([]interface{}, *rollupExp
 	return args, re, nil
 }
 
-func getRollupExprArg(arg expr) *rollupExpr {
-	re, ok := arg.(*rollupExpr)
+func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
+	re, ok := arg.(*metricsql.RollupExpr)
 	if !ok {
-		// Wrap non-rollup arg into rollupExpr.
-		return &rollupExpr{
+		// Wrap non-rollup arg into metricsql.RollupExpr.
+		return &metricsql.RollupExpr{
 			Expr: arg,
 		}
 	}
@@ -378,45 +394,50 @@ func getRollupExprArg(arg expr) *rollupExpr {
 		// Return standard rollup if it doesn't contain subquery.
 		return re
 	}
-	me, ok := re.Expr.(*metricExpr)
+	me, ok := re.Expr.(*metricsql.MetricExpr)
 	if !ok {
 		// arg contains subquery.
 		return re
 	}
 	// Convert me[w:step] -> default_rollup(me)[w:step]
 	reNew := *re
-	reNew.Expr = &funcExpr{
+	reNew.Expr = &metricsql.FuncExpr{
 		Name: "default_rollup",
-		Args: []expr{
-			&rollupExpr{Expr: me},
+		Args: []metricsql.Expr{
+			&metricsql.RollupExpr{Expr: me},
 		},
 	}
 	return &reNew
 }
 
-func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, re *rollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
 	ecNew := ec
 	var offset int64
 	if len(re.Offset) > 0 {
 		var err error
-		offset, err = DurationValue(re.Offset, ec.Step)
+		offset, err = metricsql.DurationValue(re.Offset, ec.Step)
 		if err != nil {
 			return nil, err
 		}
 		ecNew = newEvalConfig(ec)
 		ecNew.Start -= offset
 		ecNew.End -= offset
-		ecNew.Start, ecNew.End = AdjustStartEnd(ecNew.Start, ecNew.End, ecNew.Step)
+		if ecNew.MayCache {
+			start, end := AdjustStartEnd(ecNew.Start, ecNew.End, ecNew.Step)
+			offset += ecNew.Start - start
+			ecNew.Start = start
+			ecNew.End = end
+		}
 	}
 	var rvs []*timeseries
 	var err error
-	if me, ok := re.Expr.(*metricExpr); ok {
-		rvs, err = evalRollupFuncWithMetricExpr(ecNew, name, rf, me, iafc, re.Window)
+	if me, ok := re.Expr.(*metricsql.MetricExpr); ok {
+		rvs, err = evalRollupFuncWithMetricExpr(ecNew, name, rf, expr, me, iafc, re.Window)
 	} else {
 		if iafc != nil {
 			logger.Panicf("BUG: iafc must be nil for rollup %q over subquery %q", name, re.AppendString(nil))
 		}
-		rvs, err = evalRollupFuncWithSubquery(ecNew, name, rf, re)
+		rvs, err = evalRollupFuncWithSubquery(ecNew, name, rf, expr, re)
 	}
 	if err != nil {
 		return nil, err
@@ -435,12 +456,12 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, re *rollupExpr,
 	return rvs, nil
 }
 
-func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *rollupExpr) ([]*timeseries, error) {
-	// Do not use rollupResultCacheV here, since it works only with metricExpr.
+func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr) ([]*timeseries, error) {
+	// TODO: determine whether to use rollupResultCacheV here.
 	var step int64
 	if len(re.Step) > 0 {
 		var err error
-		step, err = PositiveDurationValue(re.Step, ec.Step)
+		step, err = metricsql.PositiveDurationValue(re.Step, ec.Step)
 		if err != nil {
 			return nil, err
 		}
@@ -450,7 +471,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *
 	var window int64
 	if len(re.Window) > 0 {
 		var err error
-		window, err = PositiveDurationValue(re.Window, ec.Step)
+		window, err = metricsql.PositiveDurationValue(re.Window, ec.Step)
 		if err != nil {
 			return nil, err
 		}
@@ -467,9 +488,19 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *
 	if err != nil {
 		return nil, err
 	}
+	if len(tssSQ) == 0 {
+		if name == "absent_over_time" {
+			tss := evalNumber(ec, 1)
+			return tss, nil
+		}
+		return nil, nil
+	}
 
 	sharedTimestamps := getTimestamps(ec.Start, ec.End, ec.Step)
-	preFunc, rcs := getRollupConfigs(name, rf, ec.Start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
+	preFunc, rcs, err := getRollupConfigs(name, rf, expr, ec.Start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
+	if err != nil {
+		return nil, err
+	}
 	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
 	var tssLock sync.Mutex
 	removeMetricGroup := !rollupFuncsKeepMetricGroup[name]
@@ -477,6 +508,13 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, re *
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
 		preFunc(values, timestamps)
 		for _, rc := range rcs {
+			if tsm := newTimeseriesMap(name, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
+				rc.DoTimeseriesMap(tsm, values, timestamps)
+				tssLock.Lock()
+				tss = tsm.AppendTimeseriesTo(tss)
+				tssLock.Unlock()
+				continue
+			}
 			var ts timeseries
 			doRollupForTimeseries(rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps, removeMetricGroup)
 			tssLock.Lock()
@@ -544,21 +582,22 @@ var (
 	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
 )
 
-func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me *metricExpr, iafc *incrementalAggrFuncContext, windowStr string) ([]*timeseries, error) {
+func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowStr string) ([]*timeseries, error) {
 	if me.IsEmpty() {
 		return evalNumber(ec, nan), nil
 	}
 	var window int64
 	if len(windowStr) > 0 {
 		var err error
-		window, err = PositiveDurationValue(windowStr, ec.Step)
+		window, err = metricsql.PositiveDurationValue(windowStr, ec.Step)
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	// Search for partial results in cache.
-	tssCached, start := rollupResultCacheV.Get(name, ec, me, iafc, window)
+	tssCached, start := rollupResultCacheV.Get(ec, expr, window)
 	if start > ec.End {
 		// The result is fully cached.
 		rollupResultCacheFullHits.Inc()
@@ -570,11 +609,20 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 		rollupResultCacheMiss.Inc()
 	}
 
+	// Obtain rollup configs before fetching data from db,
+	// so type errors can be caught earlier.
+	sharedTimestamps := getTimestamps(start, ec.End, ec.Step)
+	preFunc, rcs, err := getRollupConfigs(name, rf, expr, start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
+	if err != nil {
+		return nil, err
+	}
+
 	// Fetch the remaining part of the result.
+	tfs := toTagFilters(me.LabelFilters)
 	sq := &storage.SearchQuery{
 		MinTimestamp: start - window - maxSilenceInterval,
 		MaxTimestamp: ec.End + ec.Step,
-		TagFilterss:  [][]storage.TagFilter{me.TagFilters},
+		TagFilterss:  [][]storage.TagFilter{tfs},
 	}
 	rss, err := netstorage.ProcessSearchQuery(sq, true, ec.Deadline)
 	if err != nil {
@@ -583,14 +631,16 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 	rssLen := rss.Len()
 	if rssLen == 0 {
 		rss.Cancel()
+		var tss []*timeseries
+		if name == "absent_over_time" {
+			tss = getAbsentTimeseries(ec, me)
+		}
 		// Add missing points until ec.End.
 		// Do not cache the result, since missing points
 		// may be backfilled in the future.
-		tss := mergeTimeseries(tssCached, nil, start, ec)
+		tss = mergeTimeseries(tssCached, tss, start, ec)
 		return tss, nil
 	}
-	sharedTimestamps := getTimestamps(start, ec.End, ec.Step)
-	preFunc, rcs := getRollupConfigs(name, rf, start, ec.End, ec.Step, window, ec.LookbackDelta, sharedTimestamps)
 
 	// Verify timeseries fit available memory after the rollup.
 	// Take into account points from tssCached.
@@ -602,8 +652,8 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 		if iafc.ae.Modifier.Op != "" {
 			// Increase the number of timeseries for non-empty group list: `aggr() by (something)`,
 			// since each group can have own set of time series in memory.
-			// Estimate the number of such groups is lower than 100 :)
-			timeseriesLen *= 100
+			// Estimate the number of such groups is lower than 1000 :)
+			timeseriesLen *= 1000
 		}
 	}
 	rollupPoints := mulNoOverflow(pointsPerTimeseries, int64(timeseriesLen*len(rcs)))
@@ -622,16 +672,15 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc, me
 	removeMetricGroup := !rollupFuncsKeepMetricGroup[name]
 	var tss []*timeseries
 	if iafc != nil {
-		tss, err = evalRollupWithIncrementalAggregate(iafc, rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
+		tss, err = evalRollupWithIncrementalAggregate(name, iafc, rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
 	} else {
-		tss, err = evalRollupNoIncrementalAggregate(rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
+		tss, err = evalRollupNoIncrementalAggregate(name, rss, rcs, preFunc, sharedTimestamps, removeMetricGroup)
 	}
 	if err != nil {
 		return nil, err
 	}
 	tss = mergeTimeseries(tssCached, tss, start, ec)
-	rollupResultCacheV.Put(name, ec, me, iafc, window, tss)
-
+	rollupResultCacheV.Put(ec, expr, window, tss)
 	return tss, nil
 }
 
@@ -647,13 +696,20 @@ func getRollupMemoryLimiter() *memoryLimiter {
 	return &rollupMemoryLimiter
 }
 
-func evalRollupWithIncrementalAggregate(iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
+func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
 	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
 		preFunc(rs.Values, rs.Timestamps)
 		ts := getTimeseries()
 		defer putTimeseries(ts)
 		for _, rc := range rcs {
+			if tsm := newTimeseriesMap(name, sharedTimestamps, &rs.MetricName); tsm != nil {
+				rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
+				for _, ts := range tsm.m {
+					iafc.updateTimeseries(ts, workerID)
+				}
+				continue
+			}
 			ts.Reset()
 			doRollupForTimeseries(rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps, removeMetricGroup)
 			iafc.updateTimeseries(ts, workerID)
@@ -670,13 +726,20 @@ func evalRollupWithIncrementalAggregate(iafc *incrementalAggrFuncContext, rss *n
 	return tss, nil
 }
 
-func evalRollupNoIncrementalAggregate(rss *netstorage.Results, rcs []*rollupConfig,
+func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) {
 	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
 	var tssLock sync.Mutex
 	err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {
+			if tsm := newTimeseriesMap(name, sharedTimestamps, &rs.MetricName); tsm != nil {
+				rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
+				tssLock.Lock()
+				tss = tsm.AppendTimeseriesTo(tss)
+				tssLock.Unlock()
+				continue
+			}
 			var ts timeseries
 			doRollupForTimeseries(rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps, removeMetricGroup)
 			tssLock.Lock()
@@ -704,62 +767,6 @@ func doRollupForTimeseries(rc *rollupConfig, tsDst *timeseries, mnSrc *storage.M
 	tsDst.denyReuse = true
 }
 
-func getRollupConfigs(name string, rf rollupFunc, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) (
-	func(values []float64, timestamps []int64), []*rollupConfig) {
-	preFunc := func(values []float64, timestamps []int64) {}
-	if rollupFuncsRemoveCounterResets[name] {
-		preFunc = func(values []float64, timestamps []int64) {
-			removeCounterResets(values)
-		}
-	}
-	newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig {
-		return &rollupConfig{
-			TagValue:        tagValue,
-			Func:            rf,
-			Start:           start,
-			End:             end,
-			Step:            step,
-			Window:          window,
-			MayAdjustWindow: rollupFuncsMayAdjustWindow[name],
-			LookbackDelta:   lookbackDelta,
-			Timestamps:      sharedTimestamps,
-		}
-	}
-	appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig {
-		dst = append(dst, newRollupConfig(rollupMin, "min"))
-		dst = append(dst, newRollupConfig(rollupMax, "max"))
-		dst = append(dst, newRollupConfig(rollupAvg, "avg"))
-		return dst
-	}
-	var rcs []*rollupConfig
-	switch name {
-	case "rollup":
-		rcs = appendRollupConfigs(rcs)
-	case "rollup_rate", "rollup_deriv":
-		preFuncPrev := preFunc
-		preFunc = func(values []float64, timestamps []int64) {
-			preFuncPrev(values, timestamps)
-			derivValues(values, timestamps)
-		}
-		rcs = appendRollupConfigs(rcs)
-	case "rollup_increase", "rollup_delta":
-		preFuncPrev := preFunc
-		preFunc = func(values []float64, timestamps []int64) {
-			preFuncPrev(values, timestamps)
-			deltaValues(values)
-		}
-		rcs = appendRollupConfigs(rcs)
-	case "rollup_candlestick":
-		rcs = append(rcs, newRollupConfig(rollupFirst, "open"))
-		rcs = append(rcs, newRollupConfig(rollupLast, "close"))
-		rcs = append(rcs, newRollupConfig(rollupMin, "low"))
-		rcs = append(rcs, newRollupConfig(rollupMax, "high"))
-	default:
-		rcs = append(rcs, newRollupConfig(rf, ""))
-	}
-	return preFunc, rcs
-}
-
 var bbPool bytesutil.ByteBufferPool
 
 func evalNumber(ec *EvalConfig, n float64) []*timeseries {
@@ -798,3 +805,23 @@ func mulNoOverflow(a, b int64) int64 {
 	}
 	return a * b
 }
+
+func toTagFilters(lfs []metricsql.LabelFilter) []storage.TagFilter {
+	tfs := make([]storage.TagFilter, len(lfs))
+	for i := range lfs {
+		toTagFilter(&tfs[i], &lfs[i])
+	}
+	return tfs
+}
+
+func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) {
+	if src.Label != "__name__" {
+		dst.Key = []byte(src.Label)
+	} else {
+		// This is required for storage.Search.
+		dst.Key = nil
+	}
+	dst.Value = []byte(src.Value)
+	dst.IsRegexp = src.IsRegexp
+	dst.IsNegative = src.IsNegative
+}

app/vmselect/promql/exec.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -18,17 +19,6 @@ var logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.S
 
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
 
-// ExpandWithExprs expands WITH expressions inside q and returns the resulting
-// PromQL without WITH expressions.
-func ExpandWithExprs(q string) (string, error) {
-	e, err := parsePromQLWithCache(q)
-	if err != nil {
-		return "", err
-	}
-	buf := e.AppendString(nil)
-	return string(buf), nil
-}
-
 // Exec executes q for the given ec.
 func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result, error) {
 	if *logSlowQueryDuration > 0 {
@@ -36,8 +26,8 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
 		defer func() {
 			d := time.Since(startTime)
 			if d >= *logSlowQueryDuration {
-				logger.Infof("slow query according to -search.logSlowQueryDuration=%s: duration=%s, start=%d, end=%d, step=%d, query=%q",
-					*logSlowQueryDuration, d, ec.Start/1000, ec.End/1000, ec.Step/1000, q)
+				logger.Infof("slow query according to -search.logSlowQueryDuration=%s: duration=%.3f seconds, start=%d, end=%d, step=%d, query=%q",
+					*logSlowQueryDuration, d.Seconds(), ec.Start/1000, ec.End/1000, ec.Step/1000, q)
 				slowQueries.Inc()
 			}
 		}()
@@ -85,12 +75,12 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
 	return result, err
 }
 
-func maySortResults(e expr, tss []*timeseries) bool {
+func maySortResults(e metricsql.Expr, tss []*timeseries) bool {
 	if len(tss) > 100 {
 		// There is no sense in sorting a lot of results
 		return false
 	}
-	fe, ok := e.(*funcExpr)
+	fe, ok := e.(*metricsql.FuncExpr)
 	if !ok {
 		return true
 	}
@@ -154,10 +144,10 @@ func removeNaNs(tss []*timeseries) []*timeseries {
 	return rvs
 }
 
-func parsePromQLWithCache(q string) (expr, error) {
+func parsePromQLWithCache(q string) (metricsql.Expr, error) {
 	pcv := parseCacheV.Get(q)
 	if pcv == nil {
-		e, err := parsePromQL(q)
+		e, err := metricsql.Parse(q)
 		pcv = &parseCacheValue{
 			e:   e,
 			err: err,
@@ -189,7 +179,7 @@ var parseCacheV = func() *parseCache {
 const parseCacheMaxLen = 10e3
 
 type parseCacheValue struct {
-	e   expr
+	e   metricsql.Expr
 	err error
 }
 

app/vmselect/promql/exec_test.go

@@ -8,44 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
-func TestExpandWithExprsSuccess(t *testing.T) {
-	f := func(q, qExpected string) {
-		t.Helper()
-		for i := 0; i < 3; i++ {
-			qExpanded, err := ExpandWithExprs(q)
-			if err != nil {
-				t.Fatalf("unexpected error when expanding %q: %s", q, err)
-			}
-			if qExpanded != qExpected {
-				t.Fatalf("unexpected expanded expression for %q;\ngot\n%q\nwant\n%q", q, qExpanded, qExpected)
-			}
-		}
-	}
-
-	f(`1`, `1`)
-	f(`foobar`, `foobar`)
-	f(`with (x = 1) x+x`, `2`)
-	f(`with (f(x) = x*x) 3+f(2)+2`, `9`)
-}
-
-func TestExpandWithExprsError(t *testing.T) {
-	f := func(q string) {
-		t.Helper()
-		for i := 0; i < 3; i++ {
-			qExpanded, err := ExpandWithExprs(q)
-			if err == nil {
-				t.Fatalf("expecting non-nil error when expanding %q", q)
-			}
-			if qExpanded != "" {
-				t.Fatalf("unexpected non-empty qExpanded=%q", qExpanded)
-			}
-		}
-	}
-
-	f(``)
-	f(`  with (`)
-}
-
 func TestExecSuccess(t *testing.T) {
 	start := int64(1000e3)
 	end := int64(2000e3)
@@ -59,7 +21,7 @@ func TestExecSuccess(t *testing.T) {
 			Start:    start,
 			End:      end,
 			Step:     step,
-			Deadline: netstorage.NewDeadline(time.Minute),
+			Deadline: netstorage.NewDeadline(time.Minute, ""),
 		}
 		for i := 0; i < 5; i++ {
 			result, err := Exec(ec, q, false)
@@ -570,6 +532,12 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run("absent_over_time(time())", func(t *testing.T) {
+		t.Parallel()
+		q := `absent_over_time(time())`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run("absent(123)", func(t *testing.T) {
 		t.Parallel()
 		q := `absent(123)`
@@ -593,6 +561,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("absent_over_time(nan[200s:10s])", func(t *testing.T) {
+		t.Parallel()
+		q := `absent_over_time(nan[200s:10s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`absent(scalar(multi-timeseries))`, func(t *testing.T) {
 		t.Parallel()
 		q := `
@@ -609,6 +588,34 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`absent_over_time(scalar(multi-timeseries))`, func(t *testing.T) {
+		t.Parallel()
+		q := `
+		absent_over_time(label_set(scalar(1 or label_set(2, "xx", "foo")), "yy", "foo"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("yy"),
+			Value: []byte("foo"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`absent(time() > 1500)`, func(t *testing.T) {
+		t.Parallel()
+		q := `
+		absent(time() > 1500)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("clamp_max(time(), 1400)", func(t *testing.T) {
 		t.Parallel()
 		q := `clamp_max(time(), 1400)`
@@ -1459,6 +1466,38 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`label_match()`, func(t *testing.T) {
+		t.Parallel()
+		q := `
+		label_match((
+			alias(time(), "foo"),
+			alias(2*time(), "bar"),
+		), "__name__", "f.+")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foo")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`label_mismatch()`, func(t *testing.T) {
+		t.Parallel()
+		q := `
+		label_mismatch((
+			alias(time(), "foo"),
+			alias(2*time(), "bar"),
+		), "__name__", "f.+")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2000, 2400, 2800, 3200, 3600, 4000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("bar")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`two_timeseries`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_desc(time() or label_set(2, "xx", "foo"))`
@@ -2301,18 +2340,36 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(scalar)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(123, time())`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-no-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, label_set(100, "foo", "bar"))`
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(single-value-no-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(123, label_set(100, "foo", "bar"))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-invalid-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, label_set(100, "le", "foobar"))`
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(single-value-invalid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(50, label_set(100, "le", "foobar"))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, label_set(100, "le", "200"))`
@@ -2324,6 +2381,39 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(single-value-valid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(80, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.4, 0.4, 0.4, 0.4, 0.4, 0.4},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_share(single-value-valid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(200, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_share(single-value-valid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(300, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(histogram_quantile(0.6, label_set(100, "le", "200"), "foobar"))`
@@ -2353,6 +2443,35 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_quantile(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_share(120, label_set(100, "le", "200"), "foobar"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foobar"),
+			Value: []byte("lower"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.6, 0.6, 0.6, 0.6, 0.6, 0.6},
+			Timestamps: timestampsExpected,
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foobar"),
+			Value: []byte("upper"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le-max-phi)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(1, (
@@ -2367,6 +2486,20 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(single-value-valid-le-max-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(200, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le-min-phi)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0, (
@@ -2381,6 +2514,48 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(single-value-valid-le-min-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(0, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_share(single-value-valid-le-low-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(55, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_share(single-value-valid-le-mid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(105, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le-min-phi-no-zero-bucket)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0, label_set(100, "le", "200"))`
@@ -2403,6 +2578,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(scalar-phi)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(time() / 8, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.625, 0.75, 0.875, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(valid)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(histogram_quantile(0.6,
@@ -2433,6 +2619,36 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(valid)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_share(25,
+			label_set(90, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+			or label_set(200, "tag", "xx", "le", "10")
+			or label_set(300, "tag", "xx", "le", "30")
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.325, 0.325, 0.325, 0.325, 0.325, 0.325},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("tag"),
+			Value: []byte("xx"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(negative-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6,
@@ -2461,7 +2677,7 @@ func TestExecSuccess(t *testing.T) {
 		)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, 10, 10, 10},
+			Values:     []float64{30, 30, 30, 30, 30, 30},
 			Timestamps: timestampsExpected,
 		}
 		r.MetricName.Tags = []storage.Tag{{
@@ -2490,6 +2706,25 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(normal-bucket-count)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(35,
+			label_set(0, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(normal-bucket-count, boundsLabel)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(histogram_quantile(0.2,
@@ -2540,6 +2775,56 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(normal-bucket-count, boundsLabel)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_share(22,
+			label_set(0, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf"),
+			"xxx"
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("lower"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.2, 0.2, 0.2, 0.2, 0.2, 0.2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333, 0.3333333333333333},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("xxx"),
+				Value: []byte("upper"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(zero-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6,
@@ -2963,6 +3248,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`range_over_time(time)`, func(t *testing.T) {
+		t.Parallel()
+		q := `range_over_time(alias(time()/100, "foobar")[3i])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`sum(multi-vector)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sum(label_set(10, "foo", "bar") or label_set(time()/100, "baz", "sss"))`
@@ -3121,6 +3417,150 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{13, 14, 12, 8, 12, 13},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("vmrange"),
+				Value: []byte("1.0e0...1.5e0"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 15, 12, 13, 15, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("vmrange"),
+				Value: []byte("2.0e0...2.5e0"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{13, 11, 16, 19, 13, 16},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("vmrange"),
+				Value: []byte("1.5e0...2.0e0"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{13, 14, 12, 8, 12, 13},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("vmrange"),
+				Value: []byte("1.0e0...1.5e0"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 15, 12, 13, 15, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("vmrange"),
+				Value: []byte("2.0e0...2.5e0"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{13, 11, 16, 19, 13, 16},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("vmrange"),
+				Value: []byte("1.5e0...2.0e0"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{40, 40, 40, 40, 40, 40},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`topk_max(histogram_over_time)`, func(t *testing.T) {
+		t.Parallel()
+		q := `topk_max(1, histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{13, 11, 16, 19, 13, 16},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+			{
+				Key:   []byte("vmrange"),
+				Value: []byte("1.5e0...2.0e0"),
+			},
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`share_gt_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `share_gt_over_time(rand(0)[200s:10s], 0.7)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.35, 0.3, 0.5, 0.3, 0.3, 0.25},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`share_le_over_time`, func(t *testing.T) {
+		t.Parallel()
+		q := `share_le_over_time(rand(0)[200s:10s], 0.7)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.65, 0.7, 0.5, 0.7, 0.7, 0.75},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`increases_over_time`, func(t *testing.T) {
 		t.Parallel()
 		q := `increases_over_time(rand(0)[200s:10s])`
@@ -4083,6 +4523,112 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`hoeffding_bound_lower()`, func(t *testing.T) {
+		t.Parallel()
+		q := `hoeffding_bound_lower(0.9, rand(0)[:10s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.2516770508510652, 0.2830570387745462, 0.27716232108436645, 0.3679356319931767, 0.3168460474120903, 0.23156726248243734},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`hoeffding_bound_upper()`, func(t *testing.T) {
+		t.Parallel()
+		q := `hoeffding_bound_upper(0.9, alias(rand(0), "foobar")[:10s])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.6510581320042821, 0.7261021731890429, 0.7245290097397009, 0.8113950442584258, 0.7736122275568004, 0.6658564048254882},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foobar")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`aggr_over_time(single-func)`, func(t *testing.T) {
+		t.Parallel()
+		q := `aggr_over_time("increase", rand(0)[:10s])`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{5.465672601448873, 6.642207999066246, 6.8400051805114295, 7.182425481980655, 5.1677922402706, 6.594060518641982},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("increase"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`aggr_over_time(multi-func)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(aggr_over_time(("min_over_time", "count_over_time", "max_over_time"), round(rand(0),0.1)[:10s]))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("min_over_time"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.8, 0.9, 1, 0.9, 1, 0.9},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("max_over_time"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{20, 20, 20, 20, 20, 20},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("count_over_time"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`avg(aggr_over_time(multi-func))`, func(t *testing.T) {
+		t.Parallel()
+		q := `avg(aggr_over_time(("min_over_time", "max_over_time"), time()[:10s]))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{905, 1105, 1305, 1505, 1705, 1905},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`avg(aggr_over_time(multi-func)) by (rollup)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(avg(aggr_over_time(("min_over_time", "max_over_time"), time()[:10s])) by (rollup))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{810, 1010, 1210, 1410, 1610, 1810},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("min_over_time"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("rollup"),
+			Value: []byte("max_over_time"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`rollup_candlestick()`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(rollup_candlestick(round(rand(0),0.01)[:10s]))`
@@ -4097,25 +4643,25 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.32, 0.82, 0.13, 0.28, 0.86, 0.57},
+			Values:     []float64{0.85, 0.15, 0.43, 0.76, 0.47, 0.21},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
-			Value: []byte("close"),
+			Value: []byte("open"),
 		}}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.9, 0.32, 0.82, 0.13, 0.28, 0.86},
+			Values:     []float64{0.32, 0.82, 0.13, 0.28, 0.86, 0.57},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
-			Value: []byte("open"),
+			Value: []byte("close"),
 		}}
 		r4 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.9, 0.94, 0.97, 0.93, 0.98, 0.92},
+			Values:     []float64{0.85, 0.94, 0.97, 0.93, 0.98, 0.92},
 			Timestamps: timestampsExpected,
 		}
 		r4.MetricName.Tags = []storage.Tag{{
@@ -4163,7 +4709,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(rollup(time()[:50s]))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{800, 1000, 1200, 1400, 1600, 1800},
+			Values:     []float64{850, 1050, 1250, 1450, 1650, 1850},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
@@ -4683,7 +5229,7 @@ func TestExecError(t *testing.T) {
 			Start:    1000,
 			End:      2000,
 			Step:     100,
-			Deadline: netstorage.NewDeadline(time.Minute),
+			Deadline: netstorage.NewDeadline(time.Minute, ""),
 		}
 		for i := 0; i < 4; i++ {
 			rv, err := Exec(ec, q, false)
@@ -4730,6 +5276,8 @@ func TestExecError(t *testing.T) {
 	f(`label_set(1, "foo")`)
 	f(`label_del()`)
 	f(`label_keep()`)
+	f(`label_match()`)
+	f(`label_mismatch()`)
 	f(`round()`)
 	f(`round(1,2,3)`)
 	f(`scalar()`)
@@ -4784,6 +5332,18 @@ func TestExecError(t *testing.T) {
 	f(`alias(1, "foo", "bar")`)
 	f(`lifetime()`)
 	f(`lag()`)
+	f(`aggr_over_time()`)
+	f(`aggr_over_time(foo)`)
+	f(`aggr_over_time("foo", bar, 1)`)
+	f(`sum(aggr_over_time())`)
+	f(`sum(aggr_over_time(foo))`)
+	f(`count(aggr_over_time("foo", bar, 1))`)
+	f(`hoeffding_bound_lower()`)
+	f(`hoeffding_bound_lower(1)`)
+	f(`hoeffding_bound_lower(0.99, foo, 1)`)
+	f(`hoeffding_bound_upper()`)
+	f(`hoeffding_bound_upper(1)`)
+	f(`hoeffding_bound_upper(0.99, foo, 1)`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)
@@ -4818,7 +5378,11 @@ func TestExecError(t *testing.T) {
 	f(`label_transform(1, "foo", 3, 4)`)
 	f(`label_transform(1, "foo", "bar", 4)`)
 	f(`label_transform(1, "foo", "invalid(regexp", "baz`)
+	f(`label_match(1, 2, 3)`)
+	f(`label_mismatch(1, 2, 3)`)
 	f(`alias(1, 2)`)
+	f(`aggr_over_time(1, 2)`)
+	f(`aggr_over_time(("foo", "bar"), 3)`)
 
 	// Duplicate timeseries
 	f(`(label_set(1, "foo", "bar") or label_set(2, "foo", "baz"))

app/vmselect/promql/parser_test.go

@@ -47,677 +47,3 @@ func TestParseMetricSelectorError(t *testing.T) {
 	f(`foo[5m]`)
 	f(`foo offset 5m`)
 }
-
-func TestParsePromQLSuccess(t *testing.T) {
-	another := func(s string, sExpected string) {
-		t.Helper()
-
-		e, err := parsePromQL(s)
-		if err != nil {
-			t.Fatalf("unexpected error when parsing %q: %s", s, err)
-		}
-		res := e.AppendString(nil)
-		if string(res) != sExpected {
-			t.Fatalf("unexpected string constructed;\ngot\n%q\nwant\n%q", res, sExpected)
-		}
-	}
-	same := func(s string) {
-		t.Helper()
-		another(s, s)
-	}
-
-	// metricExpr
-	same(`{}`)
-	same(`{}[5m]`)
-	same(`{}[5m:]`)
-	same(`{}[:]`)
-	another(`{}[: ]`, `{}[:]`)
-	same(`{}[:3s]`)
-	another(`{}[: 3s ]`, `{}[:3s]`)
-	same(`{}[5m:3s]`)
-	another(`{}[ 5m : 3s ]`, `{}[5m:3s]`)
-	same(`{} offset 5m`)
-	same(`{} offset -5m`)
-	same(`{}[5m] offset 10y`)
-	same(`{}[5.3m:3.4s] offset 10y`)
-	same(`{}[:3.4s] offset 10y`)
-	same(`{}[:3.4s] offset -10y`)
-	same(`{Foo="bAR"}`)
-	same(`{foo="bar"}`)
-	same(`{foo="bar"}[5m]`)
-	same(`{foo="bar"}[5m:]`)
-	same(`{foo="bar"}[5m:3s]`)
-	same(`{foo="bar"} offset 10y`)
-	same(`{foo="bar"} offset -10y`)
-	same(`{foo="bar"}[5m] offset 10y`)
-	same(`{foo="bar"}[5m:3s] offset 10y`)
-	another(`{foo="bar"}[5m] oFFSEt 10y`, `{foo="bar"}[5m] offset 10y`)
-	same("METRIC")
-	same("metric")
-	same("m_e:tri44:_c123")
-	another("-metric", "0 - metric")
-	same(`metric offset 10h`)
-	same("metric[5m]")
-	same("metric[5m:3s]")
-	same("metric[5m] offset 10h")
-	same("metric[5m:3s] offset 10h")
-	same("metric[5i:3i] offset 10i")
-	same(`metric{foo="bar"}`)
-	same(`metric{foo="bar"} offset 10h`)
-	same(`metric{foo!="bar"}[2d]`)
-	same(`metric{foo="bar"}[2d] offset 10h`)
-	same(`metric{foo="bar", b="sdfsdf"}[2d:3h] offset 10h`)
-	another(`  metric  {  foo  = "bar"  }  [  2d ]   offset   10h  `, `metric{foo="bar"}[2d] offset 10h`)
-	// metric name matching keywords
-	same("rate")
-	same("RATE")
-	same("by")
-	same("BY")
-	same("bool")
-	same("BOOL")
-	same("unless")
-	same("UNLESS")
-	same("Ignoring")
-	same("with")
-	same("WITH")
-	same("With")
-	same("alias")
-	same(`alias{foo="bar"}`)
-	same(`aLIas{alias="aa"}`)
-	another(`al\ias`, `alias`)
-	// identifiers with with escape chars
-	same(`foo\ bar`)
-	same(`foo\-bar\{{baz\+bar="aa"}`)
-	another(`\x2E\x2ef\oo{b\xEF\ar="aa"}`, `\x2e.foo{b\xefar="aa"}`)
-	// Duplicate filters
-	same(`foo{__name__="bar"}`)
-	same(`foo{a="b", a="c", __name__="aaa", b="d"}`)
-	// Metric filters ending with comma
-	another(`m{foo="bar",}`, `m{foo="bar"}`)
-	// String concat in tag value
-	another(`m{foo="bar" + "baz"}`, `m{foo="barbaz"}`)
-
-	// Valid regexp
-	same(`foo{bar=~"x"}`)
-	same(`foo{bar=~"^x"}`)
-	same(`foo{bar=~"^x$"}`)
-	same(`foo{bar=~"^(a[bc]|d)$"}`)
-	same(`foo{bar!~"x"}`)
-	same(`foo{bar!~"^x"}`)
-	same(`foo{bar!~"^x$"}`)
-	same(`foo{bar!~"^(a[bc]|d)$"}`)
-
-	// stringExpr
-	same(`""`)
-	same(`"\n\t\r 12:{}[]()44"`)
-	another(`''`, `""`)
-	another("``", `""`)
-	another("   `foo\"b'ar`  ", "\"foo\\\"b'ar\"")
-	another(`  'foo\'bar"BAZ'  `, `"foo'bar\"BAZ"`)
-	// string concat
-	another(`"foo"+'bar'`, `"foobar"`)
-
-	// numberExpr
-	same(`1`)
-	same(`1.23`)
-	same(`0.23`)
-	same(`1.2e+45`)
-	same(`1.2e-45`)
-	same(`-1`)
-	same(`-1.23`)
-	same(`-0.23`)
-	same(`-1.2e+45`)
-	same(`-1.2e-45`)
-	same(`-1.2e-45`)
-	another(`12.5E34`, `1.25e+35`)
-	another(`-.2`, `-0.2`)
-	another(`-.2E-2`, `-0.002`)
-	same(`NaN`)
-	another(`nan`, `NaN`)
-	another(`NAN`, `NaN`)
-	another(`nAN`, `NaN`)
-	another(`Inf`, `+Inf`)
-	another(`INF`, `+Inf`)
-	another(`inf`, `+Inf`)
-	another(`+Inf`, `+Inf`)
-	another(`-Inf`, `-Inf`)
-	another(`-inF`, `-Inf`)
-
-	// binaryOpExpr
-	another(`nan == nan`, `NaN`)
-	another(`nan ==bool nan`, `1`)
-	another(`nan !=bool nan`, `0`)
-	another(`nan !=bool 2`, `1`)
-	another(`2 !=bool nan`, `1`)
-	another(`nan >bool nan`, `0`)
-	another(`nan <bool nan`, `0`)
-	another(`1 ==bool nan`, `0`)
-	another(`NaN !=bool 1`, `1`)
-	another(`inf >=bool 2`, `1`)
-	another(`-1 >bool -inf`, `1`)
-	another(`-1 <bool -inf`, `0`)
-	another(`nan + 2 *3 * inf`, `NaN`)
-	another(`INF - Inf`, `NaN`)
-	another(`Inf + inf`, `+Inf`)
-	another(`1/0`, `+Inf`)
-	another(`0/0`, `NaN`)
-	another(`-m`, `0 - m`)
-	same(`m + ignoring () n[5m]`)
-	another(`M + IGNORING () N[5m]`, `M + ignoring () N[5m]`)
-	same(`m + on (foo) n[5m]`)
-	another(`m + ON (Foo) n[5m]`, `m + on (Foo) n[5m]`)
-	same(`m + ignoring (a, b) n[5m]`)
-	another(`1 or 2`, `1`)
-	another(`1 and 2`, `1`)
-	another(`1 unless 2`, `NaN`)
-	another(`1 default 2`, `1`)
-	another(`1 default NaN`, `1`)
-	another(`NaN default 2`, `2`)
-	another(`1 > 2`, `NaN`)
-	another(`1 > bool 2`, `0`)
-	another(`3 >= 2`, `3`)
-	another(`3 <= bool 2`, `0`)
-	another(`1 + -2 - 3`, `-4`)
-	another(`1 / 0 + 2`, `+Inf`)
-	another(`2 + -1 / 0`, `-Inf`)
-	another(`-1 ^ 0.5`, `NaN`)
-	another(`512.5 - (1 + 3) * (2 ^ 2) ^ 3`, `256.5`)
-	another(`1 == bool 1 != bool 24 < bool 4 > bool -1`, `1`)
-	another(`1 == bOOl 1 != BOOL 24 < Bool 4 > booL -1`, `1`)
-	another(`m1+on(foo)group_left m2`, `m1 + on (foo) group_left () m2`)
-	another(`M1+ON(FOO)GROUP_left M2`, `M1 + on (FOO) group_left () M2`)
-	same(`m1 + on (foo) group_right () m2`)
-	same(`m1 + on (foo, bar) group_right (x, y) m2`)
-	another(`m1 + on (foo, bar,) group_right (x, y,) m2`, `m1 + on (foo, bar) group_right (x, y) m2`)
-	same(`m1 == bool on (foo, bar) group_right (x, y) m2`)
-	another(`5 - 1 + 3 * 2 ^ 2 ^ 3 - 2  OR Metric {Bar= "Baz", aaa!="bb",cc=~"dd" ,zz !~"ff" } `,
-		`770 or Metric{Bar="Baz", aaa!="bb", cc=~"dd", zz!~"ff"}`)
-	same(`"foo" + bar()`)
-	same(`"foo" + bar{x="y"}`)
-	same(`("foo"[3s] + bar{x="y"})[5m:3s] offset 10s`)
-	same(`("foo"[3s] + bar{x="y"})[5i:3i] offset 10i`)
-	same(`bar + "foo" offset 3s`)
-	same(`bar + "foo" offset 3i`)
-	another(`1+2 if 2>3`, `NaN`)
-	another(`1+4 if 2<3`, `5`)
-	another(`2+6 default 3 if 2>3`, `8`)
-	another(`2+6 if 2>3 default NaN`, `NaN`)
-	another(`42 if 3>2 if 2+2<5`, `42`)
-	another(`42 if 3>2 if 2+2>=5`, `NaN`)
-	another(`1+2 ifnot 2>3`, `3`)
-	another(`1+4 ifnot 2<3`, `NaN`)
-	another(`2+6 default 3 ifnot 2>3`, `8`)
-	another(`2+6 ifnot 2>3 default NaN`, `8`)
-	another(`42 if 3>2 ifnot 2+2<5`, `NaN`)
-	another(`42 if 3>2 ifnot 2+2>=5`, `42`)
-
-	// parensExpr
-	another(`(-foo + ((bar) / (baz))) + ((23))`, `((0 - foo) + (bar / baz)) + 23`)
-	another(`(FOO + ((Bar) / (baZ))) + ((23))`, `(FOO + (Bar / baZ)) + 23`)
-	same(`(foo, bar)`)
-	another(`((foo, bar),(baz))`, `((foo, bar), baz)`)
-	same(`(foo, (bar, baz), ((x, y), (z, y), xx))`)
-	another(`1+(foo, bar,)`, `1 + (foo, bar)`)
-	another(`((foo(bar,baz)), (1+(2)+(3,4)+()))`, `(foo(bar, baz), (3 + (3, 4)) + ())`)
-	same(`()`)
-
-	// funcExpr
-	same(`f()`)
-	another(`f(x,)`, `f(x)`)
-	another(`-f()-Ff()`, `(0 - f()) - Ff()`)
-	same(`F()`)
-	another(`+F()`, `F()`)
-	another(`++F()`, `F()`)
-	another(`--F()`, `0 - (0 - F())`)
-	same(`f(http_server_request)`)
-	same(`f(http_server_request)[4s:5m] offset 10m`)
-	same(`f(http_server_request)[4i:5i] offset 10i`)
-	same(`F(HttpServerRequest)`)
-	same(`f(job, foo)`)
-	same(`F(Job, Foo)`)
-	another(` FOO (bar) + f  (  m  (  ),ff(1 + (  2.5)) ,M[5m ]  , "ff"  )`, `FOO(bar) + f(m(), ff(3.5), M[5m], "ff")`)
-	// funcName matching keywords
-	same(`by(2)`)
-	same(`BY(2)`)
-	same(`or(2)`)
-	same(`OR(2)`)
-	same(`bool(2)`)
-	same(`BOOL(2)`)
-	same(`rate(rate(m))`)
-	same(`rate(rate(m[5m]))`)
-	same(`rate(rate(m[5m])[1h:])`)
-	same(`rate(rate(m[5m])[1h:3s])`)
-	// funcName with escape chars
-	same(`foo\(ba\-r()`)
-
-	// aggrFuncExpr
-	same(`sum(http_server_request) by ()`)
-	same(`sum(http_server_request) by (job)`)
-	same(`sum(http_server_request) without (job, foo)`)
-	another(`sum(x,y,) without (a,b,)`, `sum(x, y) without (a, b)`)
-	another(`sum by () (xx)`, `sum(xx) by ()`)
-	another(`sum by (s) (xx)[5s]`, `(sum(xx) by (s))[5s]`)
-	another(`SUM BY (ZZ, aa) (XX)`, `sum(XX) by (ZZ, aa)`)
-	another(`sum without (a, b) (xx,2+2)`, `sum(xx, 4) without (a, b)`)
-	another(`Sum WIthout (a, B) (XX,2+2)`, `sum(XX, 4) without (a, B)`)
-	same(`sum(a) or sum(b)`)
-	same(`sum(a) by () or sum(b) without (x, y)`)
-	same(`sum(a) + sum(b)`)
-	same(`sum(x) * (1 + sum(a))`)
-
-	// All the above
-	another(`Sum(Ff(M) * M{X=""}[5m] Offset 7m - 123, 35) BY (X, y) * F2("Test")`,
-		`sum((Ff(M) * M{X=""}[5m] offset 7m) - 123, 35) by (X, y) * F2("Test")`)
-	another(`# comment
-		Sum(Ff(M) * M{X=""}[5m] Offset 7m - 123, 35) BY (X, y) # yet another comment
-		* F2("Test")`,
-		`sum((Ff(M) * M{X=""}[5m] offset 7m) - 123, 35) by (X, y) * F2("Test")`)
-
-	// withExpr
-	another(`with () x`, `x`)
-	another(`with (x=1,) x`, `1`)
-	another(`with (x = m offset 5h) x + x`, `m offset 5h + m offset 5h`)
-	another(`with (x = m offset 5i) x + x`, `m offset 5i + m offset 5i`)
-	another(`with (foo = bar{x="x"}) 1`, `1`)
-	another(`with (foo = bar{x="x"}) "x"`, `"x"`)
-	another(`with (f="x") f`, `"x"`)
-	another(`with (foo = bar{x="x"}) x{x="y"}`, `x{x="y"}`)
-	another(`with (foo = bar{x="x"}) 1+1`, `2`)
-	another(`with (foo = bar{x="x"}) f()`, `f()`)
-	another(`with (foo = bar{x="x"}) sum(x)`, `sum(x)`)
-	another(`with (foo = bar{x="x"}) baz{foo="bar"}`, `baz{foo="bar"}`)
-	another(`with (foo = bar) baz`, `baz`)
-	another(`with (foo = bar) foo + foo{a="b"}`, `bar + bar{a="b"}`)
-	another(`with (foo = bar, bar=baz + f()) test`, `test`)
-	another(`with (ct={job="test"}) a{ct} + ct() + f({ct="x"})`, `(a{job="test"} + {job="test"}) + f({ct="x"})`)
-	another(`with (ct={job="test", i="bar"}) ct + {ct, x="d"} + foo{ct, ct} + ctx(1)`,
-		`(({job="test", i="bar"} + {job="test", i="bar", x="d"}) + foo{job="test", i="bar"}) + ctx(1)`)
-	another(`with (foo = bar) {__name__=~"foo"}`, `{__name__=~"foo"}`)
-	another(`with (foo = bar) foo{__name__="foo"}`, `bar`)
-	another(`with (foo = bar) {__name__="foo", x="y"}`, `bar{x="y"}`)
-	another(`with (foo(bar) = {__name__!="bar"}) foo(x)`, `{__name__!="bar"}`)
-	another(`with (foo(bar) = bar{__name__="bar"}) foo(x)`, `x`)
-	another(`with (foo\-bar(baz) = baz + baz) foo\-bar((x,y))`, `(x, y) + (x, y)`)
-	another(`with (foo\-bar(baz) = baz + baz) foo\-bar(x*y)`, `(x * y) + (x * y)`)
-	another(`with (foo\-bar(baz) = baz + baz) foo\-bar(x\*y)`, `x\*y + x\*y`)
-	another(`with (foo\-bar(b\ az) = b\ az + b\ az) foo\-bar(x\*y)`, `x\*y + x\*y`)
-	// override ttf to something new.
-	another(`with (ttf = a) ttf + b`, `a + b`)
-	// override ttf to ru
-	another(`with (ttf = ru(m, n)) ttf`, `(clamp_min(n - clamp_min(m, 0), 0) / clamp_min(n, 0)) * 100`)
-
-	// Verify withExpr recursion and forward reference
-	another(`with (x = x+y, y = x+x) y ^ 2`, `((x + y) + (x + y)) ^ 2`)
-	another(`with (f1(x)=f2(x), f2(x)=f1(x)^2) f1(foobar)`, `f2(foobar)`)
-	another(`with (f1(x)=f2(x), f2(x)=f1(x)^2) f2(foobar)`, `f2(foobar) ^ 2`)
-
-	// Verify withExpr funcs
-	another(`with (x() = y+1) x`, `y + 1`)
-	another(`with (x(foo) = foo+1) x(a)`, `a + 1`)
-	another(`with (x(a, b) = a + b) x(foo, bar)`, `foo + bar`)
-	another(`with (x(a, b) = a + b) x(foo, x(1, 2))`, `foo + 3`)
-	another(`with (x(a) = sum(a) by (b)) x(xx) / x(y)`, `sum(xx) by (b) / sum(y) by (b)`)
-	another(`with (f(a,f,x)=ff(x,f,a)) f(f(x,y,z),1,2)`, `ff(2, 1, ff(z, y, x))`)
-	another(`with (f(x)=1+f(x)) f(foo{bar="baz"})`, `1 + f(foo{bar="baz"})`)
-	another(`with (a=foo, y=bar, f(a)= a+a+y) f(x)`, `(x + x) + bar`)
-	another(`with (f(a, b) = m{a, b}) f({a="x", b="y"}, {c="d"})`, `m{a="x", b="y", c="d"}`)
-	another(`with (xx={a="x"}, f(a, b) = m{a, b}) f({xx, b="y"}, {c="d"})`, `m{a="x", b="y", c="d"}`)
-	another(`with (x() = {b="c"}) foo{x}`, `foo{b="c"}`)
-	another(`with (f(x)=x{foo="bar"} offset 5m) f(m offset 10m)`, `(m{foo="bar"} offset 10m) offset 5m`)
-	another(`with (f(x)=x{foo="bar",bas="a"}[5m]) f(m[10m] offset 3s)`, `(m{foo="bar", bas="a"}[10m] offset 3s)[5m]`)
-	another(`with (f(x)=x{foo="bar"}[5m] offset 10m) f(m{x="y"})`, `m{x="y", foo="bar"}[5m] offset 10m`)
-	another(`with (f(x)=x{foo="bar"}[5m] offset 10m) f({x="y", foo="bar", foo="bar"})`, `{x="y", foo="bar"}[5m] offset 10m`)
-	another(`with (f(m, x)=m{x}[5m] offset 10m) f(foo, {})`, `foo[5m] offset 10m`)
-	another(`with (f(m, x)=m{x, bar="baz"}[5m] offset 10m) f(foo, {})`, `foo{bar="baz"}[5m] offset 10m`)
-	another(`with (f(x)=x[5m] offset 3s) f(foo[3m]+bar)`, `(foo[3m] + bar)[5m] offset 3s`)
-	another(`with (f(x)=x[5m:3s] oFFsEt 1.5m) f(sum(s) by (a,b))`, `(sum(s) by (a, b))[5m:3s] offset 1.5m`)
-	another(`with (x="a", y=x) y+"bc"`, `"abc"`)
-	another(`with (x="a", y="b"+x) "we"+y+"z"+f()`, `"webaz" + f()`)
-	another(`with (f(x) = m{foo=x+"y", bar="y"+x, baz=x} + x) f("qwe")`, `m{foo="qwey", bar="yqwe", baz="qwe"} + "qwe"`)
-	another(`with (f(a)=a) f`, `f`)
-	another(`with (f\q(a)=a) f\q`, `fq`)
-
-	// Verify withExpr for aggr func modifiers
-	another(`with (f(x) = x, y = sum(m) by (f)) y`, `sum(m) by (f)`)
-	another(`with (f(x) = sum(m) by (x)) f(foo)`, `sum(m) by (foo)`)
-	another(`with (f(x) = sum(m) by (x)) f((foo, bar, foo))`, `sum(m) by (foo, bar)`)
-	another(`with (f(x) = sum(m) without (x,y)) f((a, b))`, `sum(m) without (a, b, y)`)
-	another(`with (f(x) = sum(m) without (y,x)) f((a, y))`, `sum(m) without (y, a)`)
-	another(`with (f(x,y) = a + on (x,y) group_left (y,bar) b) f(foo,())`, `a + on (foo) group_left (bar) b`)
-	another(`with (f(x,y) = a + on (x,y) group_left (y,bar) b) f((foo),())`, `a + on (foo) group_left (bar) b`)
-	another(`with (f(x,y) = a + on (x,y) group_left (y,bar) b) f((foo,xx),())`, `a + on (foo, xx) group_left (bar) b`)
-
-	// Verify nested with exprs
-	another(`with (f(x) = (with(x=y) x) + x) f(z)`, `y + z`)
-	another(`with (x=foo) f(a, with (y=x) y)`, `f(a, foo)`)
-	another(`with (x=foo) a * x + (with (y=x) y) / y`, `(a * foo) + (foo / y)`)
-	another(`with (x = with (y = foo) y + x) x/x`, `(foo + x) / (foo + x)`)
-	another(`with (
-		x = {foo="bar"},
-		q = m{x, y="1"},
-		f(x) =
-			with (
-				z(y) = x + y * q
-			)
-			z(foo) / f(x)
-	)
-	f(a)`, `(a + (foo * m{foo="bar", y="1"})) / f(a)`)
-
-	// complex withExpr
-	another(`WITH (
-		treshold = (0.9),
-		commonFilters = {job="cacher", instance=~"1.2.3.4"},
-		hits = rate(cache{type="hit", commonFilters}[5m]),
-		miss = rate(cache{type="miss", commonFilters}[5m]),
-		sumByInstance(arg) = sum(arg) by (instance),
-		hitRatio = sumByInstance(hits) / sumByInstance(hits + miss)
-	)
-	hitRatio < treshold`,
-		`(sum(rate(cache{type="hit", job="cacher", instance=~"1.2.3.4"}[5m])) by (instance) / sum(rate(cache{type="hit", job="cacher", instance=~"1.2.3.4"}[5m]) + rate(cache{type="miss", job="cacher", instance=~"1.2.3.4"}[5m])) by (instance)) < 0.9`)
-	another(`WITH (
-		x2(x) = x^2,
-		f(x, y) = x2(x) + x*y + x2(y)
-	)
-	f(a, 3)
-	`, `((a ^ 2) + (a * 3)) + 9`)
-	another(`WITH (
-		x2(x) = x^2,
-		f(x, y) = x2(x) + x*y + x2(y)
-	)
-	f(2, 3)
-	`, `19`)
-	another(`WITH (
-		commonFilters = {instance="foo"},
-		timeToFuckup(currv, maxv) = (maxv - currv) / rate(currv)
-	)
-	timeToFuckup(diskUsage{commonFilters}, maxDiskSize{commonFilters})`,
-		`(maxDiskSize{instance="foo"} - diskUsage{instance="foo"}) / rate(diskUsage{instance="foo"})`)
-	another(`WITH (
-	       commonFilters = {job="foo", instance="bar"},
-	       sumRate(m, cf) = sum(rate(m{cf})) by (job, instance),
-	       hitRate(hits, misses) = sumRate(hits, commonFilters) / (sumRate(hits, commonFilters) + sumRate(misses, commonFilters))
-	   )
-	   hitRate(cacheHits, cacheMisses)`,
-		`sum(rate(cacheHits{job="foo", instance="bar"})) by (job, instance) / (sum(rate(cacheHits{job="foo", instance="bar"})) by (job, instance) + sum(rate(cacheMisses{job="foo", instance="bar"})) by (job, instance))`)
-	another(`with(y=123,z=5) union(with(y=3,f(x)=x*y) f(2) + f(3), with(x=5,y=2) x*y*z)`, `union(15, 50)`)
-}
-
-func TestParsePromQLError(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		e, err := parsePromQL(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error when parsing %q", s)
-		}
-		if e != nil {
-			t.Fatalf("expecting nil expr when parsing %q", s)
-		}
-	}
-
-	// an empty string
-	f("")
-	f("  \t\b\r\n  ")
-
-	// invalid metricExpr
-	f(`{__name__="ff"} offset 55`)
-	f(`foo[55]`)
-	f(`m[-5m]`)
-	f(`{`)
-	f(`foo{`)
-	f(`foo{bar`)
-	f(`foo{bar=`)
-	f(`foo{bar="baz"`)
-	f(`foo{bar="baz",  `)
-	f(`foo{123="23"}`)
-	f(`foo{foo}`)
-	f(`foo{,}`)
-	f(`foo{,foo="bar"}`)
-	f(`foo{foo=}`)
-	f(`foo{foo="ba}`)
-	f(`foo{"foo"="bar"}`)
-	f(`foo{$`)
-	f(`foo{a $`)
-	f(`foo{a="b",$`)
-	f(`foo{a="b"}$`)
-	f(`[`)
-	f(`[]`)
-	f(`f[5m]$`)
-	f(`[5m]`)
-	f(`[5m] offset 4h`)
-	f(`m[5m] offset $`)
-	f(`m[5m] offset 5h $`)
-	f(`m[]`)
-	f(`m[-5m]`)
-	f(`m[5m:`)
-	f(`m[5m:-`)
-	f(`m[5m:-1`)
-	f(`m[5m:-1]`)
-	f(`m[5m:-1s]`)
-	f(`m[-5m:1s]`)
-	f(`m[-5m:-1s]`)
-	f(`m[:`)
-	f(`m[:-`)
-	f(`m[:1]`)
-	f(`m[:-1m]`)
-	f(`m[5]`)
-	f(`m[[5m]]`)
-	f(`m[foo]`)
-	f(`m["ff"]`)
-	f(`m[10m`)
-	f(`m[123`)
-	f(`m["ff`)
-	f(`m[(f`)
-	f(`fd}`)
-	f(`]`)
-	f(`m $`)
-	f(`m{,}`)
-	f(`m{x=y}`)
-	f(`m{x=y/5}`)
-	f(`m{x=y+5}`)
-
-	// Invalid regexp
-	f(`foo{bar=~"x["}`)
-	f(`foo{bar=~"x("}`)
-	f(`foo{bar=~"x)"}`)
-	f(`foo{bar!~"x["}`)
-	f(`foo{bar!~"x("}`)
-	f(`foo{bar!~"x)"}`)
-
-	// invalid stringExpr
-	f(`'`)
-	f(`"`)
-	f("`")
-	f(`"foo`)
-	f(`'foo`)
-	f("`foo")
-	f(`"foo\"bar`)
-	f(`'foo\'bar`)
-	f("`foo\\`bar")
-	f(`"" $`)
-	f(`"foo" +`)
-	f(`n{"foo" + m`)
-
-	// invalid numberExpr
-	f(`12.`)
-	f(`1.2e`)
-	f(`23e-`)
-	f(`23E+`)
-	f(`.`)
-	f(`-12.`)
-	f(`-1.2e`)
-	f(`-23e-`)
-	f(`-23E+`)
-	f(`-.`)
-	f(`-1$$`)
-	f(`-$$`)
-	f(`+$$`)
-	f(`23 $$`)
-
-	// invalid binaryOpExpr
-	f(`+`)
-	f(`1 +`)
-	f(`1 + 2.`)
-	f(`3 unless`)
-	f(`23 + on (foo)`)
-	f(`m + on (,) m`)
-	f(`3 * ignoring`)
-	f(`m * on (`)
-	f(`m * on (foo`)
-	f(`m * on (foo,`)
-	f(`m * on (foo,)`)
-	f(`m * on (,foo)`)
-	f(`m * on (,)`)
-	f(`m == bool (bar) baz`)
-	f(`m == bool () baz`)
-	f(`m * by (baz) n`)
-	f(`m + bool group_left m2`)
-	f(`m + on () group_left (`)
-	f(`m + on () group_left (,`)
-	f(`m + on () group_left (,foo`)
-	f(`m + on () group_left (foo,)`)
-	f(`m + on () group_left (,foo)`)
-	f(`m + on () group_left (foo)`)
-	f(`m + on () group_right (foo) (m`)
-	f(`m or ignoring () group_left () n`)
-	f(`1 + bool 2`)
-	f(`m % bool n`)
-	f(`m * bool baz`)
-	f(`M * BOoL BaZ`)
-	f(`foo unless ignoring (bar) group_left xxx`)
-	f(`foo or bool bar`)
-	f(`foo == bool $$`)
-	f(`"foo" + bar`)
-
-	// invalid parensExpr
-	f(`(`)
-	f(`($`)
-	f(`(+`)
-	f(`(1`)
-	f(`(m+`)
-	f(`1)`)
-	f(`(,)`)
-	f(`(1)$`)
-
-	// invalid funcExpr
-	f(`f $`)
-	f(`f($)`)
-	f(`f[`)
-	f(`f()$`)
-	f(`f(`)
-	f(`f(foo`)
-	f(`f(f,`)
-	f(`f(,`)
-	f(`f(,)`)
-	f(`f(,foo)`)
-	f(`f(,foo`)
-	f(`f(foo,$`)
-	f(`f() by (a)`)
-	f(`f without (x) (y)`)
-	f(`f() foo (a)`)
-	f(`f bar (x) (b)`)
-	f(`f bar (x)`)
-
-	// invalid aggrFuncExpr
-	f(`sum(`)
-	f(`sum $`)
-	f(`sum [`)
-	f(`sum($)`)
-	f(`sum()$`)
-	f(`sum(foo) ba`)
-	f(`sum(foo) ba()`)
-	f(`sum(foo) by`)
-	f(`sum(foo) without x`)
-	f(`sum(foo) aaa`)
-	f(`sum(foo) aaa x`)
-	f(`sum() by $`)
-	f(`sum() by (`)
-	f(`sum() by ($`)
-	f(`sum() by (a`)
-	f(`sum() by (a $`)
-	f(`sum() by (a ]`)
-	f(`sum() by (a)$`)
-	f(`sum() by (,`)
-	f(`sum() by (a,$`)
-	f(`sum() by (,)`)
-	f(`sum() by (,a`)
-	f(`sum() by (,a)`)
-	f(`sum() on (b)`)
-	f(`sum() bool`)
-	f(`sum() group_left`)
-	f(`sum() group_right(x)`)
-	f(`sum ba`)
-	f(`sum ba ()`)
-	f(`sum by (`)
-	f(`sum by (a`)
-	f(`sum by (,`)
-	f(`sum by (,)`)
-	f(`sum by (,a`)
-	f(`sum by (,a)`)
-	f(`sum by (a)`)
-	f(`sum by (a) (`)
-	f(`sum by (a) [`)
-	f(`sum by (a) {`)
-	f(`sum by (a) (b`)
-	f(`sum by (a) (b,`)
-	f(`sum by (a) (,)`)
-	f(`avg by (a) (,b)`)
-	f(`sum by (x) (y) by (z)`)
-	f(`sum(m) by (1)`)
-
-	// invalid withExpr
-	f(`with $`)
-	f(`with a`)
-	f(`with a=b c`)
-	f(`with (`)
-	f(`with (x=b)$`)
-	f(`with ($`)
-	f(`with (foo`)
-	f(`with (foo $`)
-	f(`with (x y`)
-	f(`with (x =`)
-	f(`with (x = $`)
-	f(`with (x= y`)
-	f(`with (x= y $`)
-	f(`with (x= y)`)
-	f(`with (x=(`)
-	f(`with (x=[)`)
-	f(`with (x=() x)`)
-	f(`with ($$)`)
-	f(`with (x $$`)
-	f(`with (x = $$)`)
-	f(`with (x = foo) bar{x}`)
-	f(`with (x = {foo="bar"}[5m]) bar{x}`)
-	f(`with (x = {foo="bar"} offset 5m) bar{x}`)
-	f(`with (x = a, x = b) c`)
-	f(`with (x(a, a) = b) c`)
-	f(`with (x=m{f="x"}) foo{x}`)
-	f(`with (sum = x) y`)
-	f(`with (rate(a) = b) c`)
-	f(`with (clamp_min=x) y`)
-	f(`with (f()`)
-	f(`with (a=b c=d) e`)
-	f(`with (f(x)=x^2) m{x}`)
-	f(`with (f(x)=ff()) m{x}`)
-	f(`with (f(x`)
-	f(`with (x=m) a{x} + b`)
-	f(`with (x=m) b + a{x}`)
-	f(`with (x=m) f(b, a{x})`)
-	f(`with (x=m) sum(a{x})`)
-	f(`with (x=m) (a{x})`)
-	f(`with (f(a)=a) f(1, 2)`)
-	f(`with (f(x)=x{foo="bar"}) f(1)`)
-	f(`with (f(x)=x{foo="bar"}) f(m + n)`)
-	f(`with (f = with`)
-	f(`with (,)`)
-	f(`with (1) 2`)
-	f(`with (f(1)=2) 3`)
-	f(`with (f(,)=x) x`)
-	f(`with (x(a) = {b="c"}) foo{x}`)
-	f(`with (f(x) = m{foo=xx}) f("qwe")`)
-	f(`a + with(f(x)=x) f(1,2)`)
-	f(`with (f(x) = sum(m) by (x)) f({foo="bar"})`)
-	f(`with (f(x) = sum(m) by (x)) f((xx(), {foo="bar"}))`)
-	f(`with (f(x) = m + on (x) n) f(xx())`)
-	f(`with (f(x) = m + on (a) group_right (x) n) f(xx())`)
-}

app/vmselect/promql/rollup.go

@@ -8,12 +8,13 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/histogram"
 )
 
 var rollupFuncs = map[string]newRollupFunc{
-	"default_rollup": newRollupFuncOneArg(rollupDefault), // default rollup func
-
 	// Standard rollup funcs from PromQL.
 	// See funcs accepting range-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
 	"changes":            newRollupFuncOneArg(rollupChanges),
@@ -35,38 +36,98 @@ var rollupFuncs = map[string]newRollupFunc{
 	"quantile_over_time": newRollupQuantile,
 	"stddev_over_time":   newRollupFuncOneArg(rollupStddev),
 	"stdvar_over_time":   newRollupFuncOneArg(rollupStdvar),
+	"absent_over_time":   newRollupFuncOneArg(rollupAbsent),
 
 	// Additional rollup funcs.
-	"sum2_over_time":      newRollupFuncOneArg(rollupSum2),
-	"geomean_over_time":   newRollupFuncOneArg(rollupGeomean),
-	"first_over_time":     newRollupFuncOneArg(rollupFirst),
-	"last_over_time":      newRollupFuncOneArg(rollupLast),
-	"distinct_over_time":  newRollupFuncOneArg(rollupDistinct),
-	"increases_over_time": newRollupFuncOneArg(rollupIncreases),
-	"decreases_over_time": newRollupFuncOneArg(rollupDecreases),
-	"integrate":           newRollupFuncOneArg(rollupIntegrate),
-	"ideriv":              newRollupFuncOneArg(rollupIderiv),
-	"lifetime":            newRollupFuncOneArg(rollupLifetime),
-	"lag":                 newRollupFuncOneArg(rollupLag),
-	"scrape_interval":     newRollupFuncOneArg(rollupScrapeInterval),
-	"rollup":              newRollupFuncOneArg(rollupFake),
-	"rollup_rate":         newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_deriv":        newRollupFuncOneArg(rollupFake),
-	"rollup_delta":        newRollupFuncOneArg(rollupFake),
-	"rollup_increase":     newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
-	"rollup_candlestick":  newRollupFuncOneArg(rollupFake),
+	"default_rollup":        newRollupFuncOneArg(rollupDefault), // default rollup func
+	"range_over_time":       newRollupFuncOneArg(rollupRange),
+	"sum2_over_time":        newRollupFuncOneArg(rollupSum2),
+	"geomean_over_time":     newRollupFuncOneArg(rollupGeomean),
+	"first_over_time":       newRollupFuncOneArg(rollupFirst),
+	"last_over_time":        newRollupFuncOneArg(rollupLast),
+	"distinct_over_time":    newRollupFuncOneArg(rollupDistinct),
+	"increases_over_time":   newRollupFuncOneArg(rollupIncreases),
+	"decreases_over_time":   newRollupFuncOneArg(rollupDecreases),
+	"integrate":             newRollupFuncOneArg(rollupIntegrate),
+	"ideriv":                newRollupFuncOneArg(rollupIderiv),
+	"lifetime":              newRollupFuncOneArg(rollupLifetime),
+	"lag":                   newRollupFuncOneArg(rollupLag),
+	"scrape_interval":       newRollupFuncOneArg(rollupScrapeInterval),
+	"tmin_over_time":        newRollupFuncOneArg(rollupTmin),
+	"tmax_over_time":        newRollupFuncOneArg(rollupTmax),
+	"share_le_over_time":    newRollupShareLE,
+	"share_gt_over_time":    newRollupShareGT,
+	"histogram_over_time":   newRollupFuncOneArg(rollupHistogram),
+	"rollup":                newRollupFuncOneArg(rollupFake),
+	"rollup_rate":           newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_deriv":          newRollupFuncOneArg(rollupFake),
+	"rollup_delta":          newRollupFuncOneArg(rollupFake),
+	"rollup_increase":       newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
+	"rollup_candlestick":    newRollupFuncOneArg(rollupFake),
+	"aggr_over_time":        newRollupFuncTwoArgs(rollupFake),
+	"hoeffding_bound_upper": newRollupHoeffdingBoundUpper,
+	"hoeffding_bound_lower": newRollupHoeffdingBoundLower,
 }
 
-var rollupFuncsMayAdjustWindow = map[string]bool{
-	"default_rollup":  true,
-	"first_over_time": true,
-	"last_over_time":  true,
-	"deriv":           true,
-	"deriv_fast":      true,
-	"irate":           true,
-	"rate":            true,
-	"lifetime":        true,
-	"scrape_interval": true,
+// rollupAggrFuncs are functions that can be passed to `aggr_over_time()`
+var rollupAggrFuncs = map[string]rollupFunc{
+	// Standard rollup funcs from PromQL.
+	"changes":          rollupChanges,
+	"delta":            rollupDelta,
+	"deriv":            rollupDerivSlow,
+	"deriv_fast":       rollupDerivFast,
+	"idelta":           rollupIdelta,
+	"increase":         rollupIncrease,  // + rollupFuncsRemoveCounterResets
+	"irate":            rollupIderiv,    // + rollupFuncsRemoveCounterResets
+	"rate":             rollupDerivFast, // + rollupFuncsRemoveCounterResets
+	"resets":           rollupResets,
+	"avg_over_time":    rollupAvg,
+	"min_over_time":    rollupMin,
+	"max_over_time":    rollupMax,
+	"sum_over_time":    rollupSum,
+	"count_over_time":  rollupCount,
+	"stddev_over_time": rollupStddev,
+	"stdvar_over_time": rollupStdvar,
+	"absent_over_time": rollupAbsent,
+
+	// Additional rollup funcs.
+	"range_over_time":     rollupRange,
+	"sum2_over_time":      rollupSum2,
+	"geomean_over_time":   rollupGeomean,
+	"first_over_time":     rollupFirst,
+	"last_over_time":      rollupLast,
+	"distinct_over_time":  rollupDistinct,
+	"increases_over_time": rollupIncreases,
+	"decreases_over_time": rollupDecreases,
+	"integrate":           rollupIntegrate,
+	"ideriv":              rollupIderiv,
+	"lifetime":            rollupLifetime,
+	"lag":                 rollupLag,
+	"scrape_interval":     rollupScrapeInterval,
+	"tmin_over_time":      rollupTmin,
+	"tmax_over_time":      rollupTmax,
+}
+
+var rollupFuncsCannotAdjustWindow = map[string]bool{
+	"changes":             true,
+	"delta":               true,
+	"holt_winters":        true,
+	"idelta":              true,
+	"increase":            true,
+	"predict_linear":      true,
+	"resets":              true,
+	"sum_over_time":       true,
+	"count_over_time":     true,
+	"quantile_over_time":  true,
+	"stddev_over_time":    true,
+	"stdvar_over_time":    true,
+	"absent_over_time":    true,
+	"sum2_over_time":      true,
+	"geomean_over_time":   true,
+	"distinct_over_time":  true,
+	"increases_over_time": true,
+	"decreases_over_time": true,
+	"integrate":           true,
 }
 
 var rollupFuncsRemoveCounterResets = map[string]bool{
@@ -78,13 +139,64 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
 }
 
 var rollupFuncsKeepMetricGroup = map[string]bool{
-	"default_rollup":     true,
-	"avg_over_time":      true,
-	"min_over_time":      true,
-	"max_over_time":      true,
-	"quantile_over_time": true,
-	"rollup":             true,
-	"geomean_over_time":  true,
+	"default_rollup":        true,
+	"avg_over_time":         true,
+	"min_over_time":         true,
+	"max_over_time":         true,
+	"quantile_over_time":    true,
+	"rollup":                true,
+	"geomean_over_time":     true,
+	"hoeffding_bound_lower": true,
+	"hoeffding_bound_upper": true,
+}
+
+func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
+	afe, ok := expr.(*metricsql.AggrFuncExpr)
+	if ok {
+		// This is for incremental aggregate function case:
+		//
+		//     sum(aggr_over_time(...))
+		//
+		// See aggr_incremental.go for details.
+		expr = afe.Args[0]
+	}
+	fe, ok := expr.(*metricsql.FuncExpr)
+	if !ok {
+		logger.Panicf("BUG: unexpected expression; want metricsql.FuncExpr; got %T; value: %s", expr, expr.AppendString(nil))
+	}
+	if fe.Name != "aggr_over_time" {
+		logger.Panicf("BUG: unexpected function name: %q; want `aggr_over_time`", fe.Name)
+	}
+	if len(fe.Args) != 2 {
+		return nil, fmt.Errorf("unexpected number of args to aggr_over_time(); got %d; want %d", len(fe.Args), 2)
+	}
+	arg := fe.Args[0]
+	var aggrFuncNames []string
+	if se, ok := arg.(*metricsql.StringExpr); ok {
+		aggrFuncNames = append(aggrFuncNames, se.S)
+	} else {
+		fe, ok := arg.(*metricsql.FuncExpr)
+		if !ok || fe.Name != "" {
+			return nil, fmt.Errorf("%s cannot be passed to aggr_over_time(); expecting quoted aggregate function name or a list of quoted aggregate function names",
+				arg.AppendString(nil))
+		}
+		for _, e := range fe.Args {
+			se, ok := e.(*metricsql.StringExpr)
+			if !ok {
+				return nil, fmt.Errorf("%s cannot be passed here; expecting quoted aggregate function name", e.AppendString(nil))
+			}
+			aggrFuncNames = append(aggrFuncNames, se.S)
+		}
+	}
+	if len(aggrFuncNames) == 0 {
+		return nil, fmt.Errorf("aggr_over_time() must contain at least a single aggregate function name")
+	}
+	for _, s := range aggrFuncNames {
+		if rollupAggrFuncs[s] == nil {
+			return nil, fmt.Errorf("%q cannot be used in `aggr_over_time` function; expecting quoted aggregate function name", s)
+		}
+	}
+	return aggrFuncNames, nil
 }
 
 func getRollupArgIdx(funcName string) int {
@@ -92,10 +204,84 @@ func getRollupArgIdx(funcName string) int {
 	if rollupFuncs[funcName] == nil {
 		logger.Panicf("BUG: getRollupArgIdx is called for non-rollup func %q", funcName)
 	}
-	if funcName == "quantile_over_time" {
+	switch funcName {
+	case "quantile_over_time", "aggr_over_time",
+		"hoeffding_bound_lower", "hoeffding_bound_upper":
 		return 1
+	default:
+		return 0
 	}
-	return 0
+}
+
+func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) (
+	func(values []float64, timestamps []int64), []*rollupConfig, error) {
+	preFunc := func(values []float64, timestamps []int64) {}
+	if rollupFuncsRemoveCounterResets[name] {
+		preFunc = func(values []float64, timestamps []int64) {
+			removeCounterResets(values)
+		}
+	}
+	newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig {
+		return &rollupConfig{
+			TagValue:        tagValue,
+			Func:            rf,
+			Start:           start,
+			End:             end,
+			Step:            step,
+			Window:          window,
+			MayAdjustWindow: !rollupFuncsCannotAdjustWindow[name],
+			LookbackDelta:   lookbackDelta,
+			Timestamps:      sharedTimestamps,
+		}
+	}
+	appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig {
+		dst = append(dst, newRollupConfig(rollupMin, "min"))
+		dst = append(dst, newRollupConfig(rollupMax, "max"))
+		dst = append(dst, newRollupConfig(rollupAvg, "avg"))
+		return dst
+	}
+	var rcs []*rollupConfig
+	switch name {
+	case "rollup":
+		rcs = appendRollupConfigs(rcs)
+	case "rollup_rate", "rollup_deriv":
+		preFuncPrev := preFunc
+		preFunc = func(values []float64, timestamps []int64) {
+			preFuncPrev(values, timestamps)
+			derivValues(values, timestamps)
+		}
+		rcs = appendRollupConfigs(rcs)
+	case "rollup_increase", "rollup_delta":
+		preFuncPrev := preFunc
+		preFunc = func(values []float64, timestamps []int64) {
+			preFuncPrev(values, timestamps)
+			deltaValues(values)
+		}
+		rcs = appendRollupConfigs(rcs)
+	case "rollup_candlestick":
+		rcs = append(rcs, newRollupConfig(rollupFirst, "open"))
+		rcs = append(rcs, newRollupConfig(rollupLast, "close"))
+		rcs = append(rcs, newRollupConfig(rollupMin, "low"))
+		rcs = append(rcs, newRollupConfig(rollupMax, "high"))
+	case "aggr_over_time":
+		aggrFuncNames, err := getRollupAggrFuncNames(expr)
+		if err != nil {
+			return nil, nil, fmt.Errorf("invalid args to %s: %s", expr.AppendString(nil), err)
+		}
+		for _, aggrFuncName := range aggrFuncNames {
+			if rollupFuncsRemoveCounterResets[aggrFuncName] {
+				// There is no need to save the previous preFunc, since it is either empty or the same.
+				preFunc = func(values []float64, timestamps []int64) {
+					removeCounterResets(values)
+				}
+			}
+			rf := rollupAggrFuncs[aggrFuncName]
+			rcs = append(rcs, newRollupConfig(rf, aggrFuncName))
+		}
+	default:
+		rcs = append(rcs, newRollupConfig(rf, ""))
+	}
+	return preFunc, rcs, nil
 }
 
 func getRollupFunc(funcName string) newRollupFunc {
@@ -103,10 +289,6 @@ func getRollupFunc(funcName string) newRollupFunc {
 	return rollupFuncs[funcName]
 }
 
-func isRollupFunc(funcName string) bool {
-	return getRollupFunc(funcName) != nil
-}
-
 type rollupFuncArg struct {
 	prevValue     float64
 	prevTimestamp int64
@@ -116,7 +298,12 @@ type rollupFuncArg struct {
 	currTimestamp int64
 	idx           int
 	step          int64
+
+	// Real previous value even if it is located too far from the current window.
+	// It matches prevValue if prevValue is not nan.
 	realPrevValue float64
+
+	tsm *timeseriesMap
 }
 
 func (rfa *rollupFuncArg) reset() {
@@ -128,6 +315,7 @@ func (rfa *rollupFuncArg) reset() {
 	rfa.idx = 0
 	rfa.step = 0
 	rfa.realPrevValue = nan
+	rfa.tsm = nil
 }
 
 // rollupFunc must return rollup value for the given rfa.
@@ -166,6 +354,54 @@ var (
 // The maximum interval without previous rows.
 const maxSilenceInterval = 5 * 60 * 1000
 
+type timeseriesMap struct {
+	origin    *timeseries
+	labelName string
+	h         metrics.Histogram
+	m         map[string]*timeseries
+}
+
+func newTimeseriesMap(funcName string, sharedTimestamps []int64, mnSrc *storage.MetricName) *timeseriesMap {
+	if funcName != "histogram_over_time" {
+		return nil
+	}
+
+	values := make([]float64, len(sharedTimestamps))
+	for i := range values {
+		values[i] = nan
+	}
+	var origin timeseries
+	origin.MetricName.CopyFrom(mnSrc)
+	origin.MetricName.ResetMetricGroup()
+	origin.Timestamps = sharedTimestamps
+	origin.Values = values
+	return &timeseriesMap{
+		origin:    &origin,
+		labelName: "vmrange",
+		m:         make(map[string]*timeseries),
+	}
+}
+
+func (tsm *timeseriesMap) AppendTimeseriesTo(dst []*timeseries) []*timeseries {
+	for _, ts := range tsm.m {
+		dst = append(dst, ts)
+	}
+	return dst
+}
+
+func (tsm *timeseriesMap) GetOrCreateTimeseries(labelValue string) *timeseries {
+	ts := tsm.m[labelValue]
+	if ts != nil {
+		return ts
+	}
+	ts = &timeseries{}
+	ts.CopyFromShallowTimestamps(tsm.origin)
+	ts.MetricName.RemoveTag(tsm.labelName)
+	ts.MetricName.AddTag(tsm.labelName, labelValue)
+	tsm.m[labelValue] = ts
+	return ts
+}
+
 // Do calculates rollups for the given timestamps and values, appends
 // them to dstValues and returns results.
 //
@@ -173,8 +409,19 @@ const maxSilenceInterval = 5 * 60 * 1000
 //
 // timestamps must cover time range [rc.Start - rc.Window - maxSilenceInterval ... rc.End + rc.Step].
 //
-// Cannot be called from concurrent goroutines.
+// Do cannot be called from concurrent goroutines.
 func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []int64) []float64 {
+	return rc.doInternal(dstValues, nil, values, timestamps)
+}
+
+// DoTimeseriesMap calculates rollups for the given timestamps and values and puts them to tsm.
+func (rc *rollupConfig) DoTimeseriesMap(tsm *timeseriesMap, values []float64, timestamps []int64) {
+	ts := getTimeseries()
+	ts.Values = rc.doInternal(ts.Values[:0], tsm, values, timestamps)
+	putTimeseries(ts)
+}
+
+func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, values []float64, timestamps []int64) []float64 {
 	// Sanity checks.
 	if rc.Step <= 0 {
 		logger.Panicf("BUG: Step must be bigger than 0; got %d", rc.Step)
@@ -192,7 +439,8 @@ func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []i
 	// Extend dstValues in order to remove mallocs below.
 	dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
 
-	maxPrevInterval := getMaxPrevInterval(timestamps)
+	scrapeInterval := getScrapeInterval(timestamps)
+	maxPrevInterval := getMaxPrevInterval(scrapeInterval)
 	if rc.LookbackDelta > 0 && maxPrevInterval > rc.LookbackDelta {
 		maxPrevInterval = rc.LookbackDelta
 	}
@@ -207,6 +455,7 @@ func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []i
 	rfa.idx = 0
 	rfa.step = rc.Step
 	rfa.realPrevValue = nan
+	rfa.tsm = tsm
 
 	i := 0
 	j := 0
@@ -296,7 +545,7 @@ func binarySearchInt64(a []int64, v int64) uint {
 	return i
 }
 
-func getMaxPrevInterval(timestamps []int64) int64 {
+func getScrapeInterval(timestamps []int64) int64 {
 	if len(timestamps) < 2 {
 		return int64(maxSilenceInterval)
 	}
@@ -312,30 +561,34 @@ func getMaxPrevInterval(timestamps []int64) int64 {
 		h.Update(float64(ts - tsPrev))
 		tsPrev = ts
 	}
-	d := int64(h.Quantile(0.6))
+	scrapeInterval := int64(h.Quantile(0.6))
 	histogram.PutFast(h)
-	if d <= 0 {
+	if scrapeInterval <= 0 {
 		return int64(maxSilenceInterval)
 	}
-	// Increase d more for smaller scrape intervals in order to hide possible gaps
+	return scrapeInterval
+}
+
+func getMaxPrevInterval(scrapeInterval int64) int64 {
+	// Increase scrapeInterval more for smaller scrape intervals in order to hide possible gaps
 	// when high jitter is present.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/139 .
-	if d <= 2*1000 {
-		return d + 4*d
+	if scrapeInterval <= 2*1000 {
+		return scrapeInterval + 4*scrapeInterval
 	}
-	if d <= 4*1000 {
-		return d + 2*d
+	if scrapeInterval <= 4*1000 {
+		return scrapeInterval + 2*scrapeInterval
 	}
-	if d <= 8*1000 {
-		return d + d
+	if scrapeInterval <= 8*1000 {
+		return scrapeInterval + scrapeInterval
 	}
-	if d <= 16*1000 {
-		return d + d/2
+	if scrapeInterval <= 16*1000 {
+		return scrapeInterval + scrapeInterval/2
 	}
-	if d <= 32*1000 {
-		return d + d/4
+	if scrapeInterval <= 32*1000 {
+		return scrapeInterval + scrapeInterval/4
 	}
-	return d + d/8
+	return scrapeInterval + scrapeInterval/8
 }
 
 func removeCounterResets(values []float64) {
@@ -414,6 +667,15 @@ func newRollupFuncOneArg(rf rollupFunc) newRollupFunc {
 	}
 }
 
+func newRollupFuncTwoArgs(rf rollupFunc) newRollupFunc {
+	return func(args []interface{}) (rollupFunc, error) {
+		if err := expectRollupArgsNum(args, 2); err != nil {
+			return nil, err
+		}
+		return rf, nil
+	}
+}
+
 func newRollupHoltWinters(args []interface{}) (rollupFunc, error) {
 	if err := expectRollupArgsNum(args, 3); err != nil {
 		return nil, err
@@ -522,6 +784,116 @@ func linearRegression(rfa *rollupFuncArg) (float64, float64) {
 	return v, k
 }
 
+func newRollupShareLE(args []interface{}) (rollupFunc, error) {
+	return newRollupShareFilter(args, countFilterLE)
+}
+
+func countFilterLE(values []float64, le float64) int {
+	n := 0
+	for _, v := range values {
+		if v <= le {
+			n++
+		}
+	}
+	return n
+}
+
+func newRollupShareGT(args []interface{}) (rollupFunc, error) {
+	return newRollupShareFilter(args, countFilterGT)
+}
+
+func countFilterGT(values []float64, gt float64) int {
+	n := 0
+	for _, v := range values {
+		if v > gt {
+			n++
+		}
+	}
+	return n
+}
+
+func newRollupShareFilter(args []interface{}, countFilter func(values []float64, limit float64) int) (rollupFunc, error) {
+	if err := expectRollupArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	limits, err := getScalar(args[1], 1)
+	if err != nil {
+		return nil, err
+	}
+	rf := func(rfa *rollupFuncArg) float64 {
+		// There is no need in handling NaNs here, since they must be cleaned up
+		// before calling rollup funcs.
+		values := rfa.values
+		if len(values) == 0 {
+			return nan
+		}
+		limit := limits[rfa.idx]
+		n := countFilter(values, limit)
+		return float64(n) / float64(len(values))
+	}
+	return rf, nil
+}
+
+func newRollupHoeffdingBoundLower(args []interface{}) (rollupFunc, error) {
+	if err := expectRollupArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	phis, err := getScalar(args[0], 0)
+	if err != nil {
+		return nil, err
+	}
+	rf := func(rfa *rollupFuncArg) float64 {
+		bound, avg := rollupHoeffdingBoundInternal(rfa, phis)
+		return avg - bound
+	}
+	return rf, nil
+}
+
+func newRollupHoeffdingBoundUpper(args []interface{}) (rollupFunc, error) {
+	if err := expectRollupArgsNum(args, 2); err != nil {
+		return nil, err
+	}
+	phis, err := getScalar(args[0], 0)
+	if err != nil {
+		return nil, err
+	}
+	rf := func(rfa *rollupFuncArg) float64 {
+		bound, avg := rollupHoeffdingBoundInternal(rfa, phis)
+		return avg + bound
+	}
+	return rf, nil
+}
+
+func rollupHoeffdingBoundInternal(rfa *rollupFuncArg, phis []float64) (float64, float64) {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	if len(values) == 0 {
+		return nan, nan
+	}
+	if len(values) == 1 {
+		return 0, values[0]
+	}
+	vMax := rollupMax(rfa)
+	vMin := rollupMin(rfa)
+	vAvg := rollupAvg(rfa)
+	vRange := vMax - vMin
+	if vRange <= 0 {
+		return 0, vAvg
+	}
+	phi := phis[rfa.idx]
+	if phi >= 1 {
+		return inf, vAvg
+	}
+	if phi <= 0 {
+		return 0, vAvg
+	}
+	// See https://en.wikipedia.org/wiki/Hoeffding%27s_inequality
+	// and https://www.youtube.com/watch?v=6UwcqiNsZ8U&feature=youtu.be&t=1237
+	bound := vRange * math.Sqrt(math.Log(1/(1-phi))/(2*float64(len(values))))
+	return bound, vAvg
+}
+
 func newRollupQuantile(args []interface{}) (rollupFunc, error) {
 	if err := expectRollupArgsNum(args, 2); err != nil {
 		return nil, err
@@ -553,6 +925,21 @@ func newRollupQuantile(args []interface{}) (rollupFunc, error) {
 	return rf, nil
 }
 
+func rollupHistogram(rfa *rollupFuncArg) float64 {
+	values := rfa.values
+	tsm := rfa.tsm
+	tsm.h.Reset()
+	for _, v := range values {
+		tsm.h.Update(v)
+	}
+	idx := rfa.idx
+	tsm.h.VisitNonZeroBuckets(func(vmrange string, count uint64) {
+		ts := tsm.GetOrCreateTimeseries(vmrange)
+		ts.Values[idx] = float64(count)
+	})
+	return nan
+}
+
 func rollupAvg(rfa *rollupFuncArg) float64 {
 	// Do not use `Rapid calculation methods` at https://en.wikipedia.org/wiki/Standard_deviation,
 	// since it is slower and has no significant benefits in precision.
@@ -561,7 +948,10 @@ func rollupAvg(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		return rfa.prevValue
+		// Do not take into account rfa.prevValue, since it may lead
+		// to inconsistent results comparing to Prometheus on broken time series
+		// with irregular data points.
+		return nan
 	}
 	var sum float64
 	for _, v := range values {
@@ -573,14 +963,14 @@ func rollupAvg(rfa *rollupFuncArg) float64 {
 func rollupMin(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
-	minValue := rfa.prevValue
 	values := rfa.values
-	if math.IsNaN(minValue) {
-		if len(values) == 0 {
-			return nan
-		}
-		minValue = values[0]
+	if len(values) == 0 {
+		// Do not take into account rfa.prevValue, since it may lead
+		// to inconsistent results comparing to Prometheus on broken time series
+		// with irregular data points.
+		return nan
 	}
+	minValue := values[0]
 	for _, v := range values {
 		if v < minValue {
 			minValue = v
@@ -592,14 +982,14 @@ func rollupMin(rfa *rollupFuncArg) float64 {
 func rollupMax(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
-	maxValue := rfa.prevValue
 	values := rfa.values
-	if math.IsNaN(maxValue) {
-		if len(values) == 0 {
-			return nan
-		}
-		maxValue = values[0]
+	if len(values) == 0 {
+		// Do not take into account rfa.prevValue, since it may lead
+		// to inconsistent results comparing to Prometheus on broken time series
+		// with irregular data points.
+		return nan
 	}
+	maxValue := values[0]
 	for _, v := range values {
 		if v > maxValue {
 			maxValue = v
@@ -608,6 +998,44 @@ func rollupMax(rfa *rollupFuncArg) float64 {
 	return maxValue
 }
 
+func rollupTmin(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	timestamps := rfa.timestamps
+	if len(values) == 0 {
+		return nan
+	}
+	minValue := values[0]
+	minTimestamp := timestamps[0]
+	for i, v := range values {
+		if v < minValue {
+			minValue = v
+			minTimestamp = timestamps[i]
+		}
+	}
+	return float64(minTimestamp) * 1e-3
+}
+
+func rollupTmax(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	values := rfa.values
+	timestamps := rfa.timestamps
+	if len(values) == 0 {
+		return nan
+	}
+	maxValue := values[0]
+	maxTimestamp := timestamps[0]
+	for i, v := range values {
+		if v > maxValue {
+			maxValue = v
+			maxTimestamp = timestamps[i]
+		}
+	}
+	return float64(maxTimestamp) * 1e-3
+}
+
 func rollupSum(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
@@ -625,6 +1053,12 @@ func rollupSum(rfa *rollupFuncArg) float64 {
 	return sum
 }
 
+func rollupRange(rfa *rollupFuncArg) float64 {
+	max := rollupMax(rfa)
+	min := rollupMin(rfa)
+	return max - min
+}
+
 func rollupSum2(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
@@ -653,6 +1087,13 @@ func rollupGeomean(rfa *rollupFuncArg) float64 {
 	return math.Pow(p, 1/float64(len(values)))
 }
 
+func rollupAbsent(rfa *rollupFuncArg) float64 {
+	if len(rfa.values) == 0 {
+		return 1
+	}
+	return nan
+}
+
 func rollupCount(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
@@ -716,16 +1157,11 @@ func rollupDeltaInternal(rfa *rollupFuncArg, canUseRealPrevValue bool) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		if len(values) == 1 {
-			if canUseRealPrevValue && !math.IsNaN(rfa.realPrevValue) {
-				// Fix against removeCounterResets.
-				return values[0] - rfa.realPrevValue
-			}
-			// Assume that the previous non-existing value was 0.
-			return values[0]
+		// Assume that the previous non-existing value was 0.
+		prevValue = 0
+		if canUseRealPrevValue && !math.IsNaN(rfa.prevValue) {
+			prevValue = rfa.prevValue
 		}
-		prevValue = values[0]
-		values = values[1:]
 	}
 	if len(values) == 0 {
 		// Assume that the value didn't change on the given interval.
@@ -773,16 +1209,25 @@ func rollupDerivFast(rfa *rollupFuncArg) float64 {
 	prevValue := rfa.prevValue
 	prevTimestamp := rfa.prevTimestamp
 	if math.IsNaN(prevValue) {
-		if len(values) < 2 {
-			// It is impossible to calculate derivative on 0 or 1 values.
+		if len(values) == 0 {
+			return nan
+		}
+		if len(values) == 1 {
+			// It is impossible to determine the duration during which the value changed
+			// from 0 to the current value.
+			// The following attempts didn't work well:
+			// - using scrape interval as the duration. It fails on Prometheus restarts when it
+			//   skips scraping for the counter. This results in too high rate() value for the first point
+			//   after Prometheus restarts.
+			// - using window or step as the duration. It results in too small rate() values for the first
+			//   points of time series.
+			//
+			// So just return nan
 			return nan
 		}
 		prevValue = values[0]
 		prevTimestamp = timestamps[0]
-		values = values[1:]
-		timestamps = timestamps[1:]
-	}
-	if len(values) == 0 {
+	} else if len(values) == 0 {
 		// Assume that the value didn't change on the given interval.
 		return 0
 	}
@@ -799,8 +1244,20 @@ func rollupIderiv(rfa *rollupFuncArg) float64 {
 	values := rfa.values
 	timestamps := rfa.timestamps
 	if len(values) < 2 {
-		if len(values) == 0 || math.IsNaN(rfa.prevValue) {
-			// It is impossible to calculate derivative on 0 or 1 values.
+		if len(values) == 0 {
+			return nan
+		}
+		if math.IsNaN(rfa.prevValue) {
+			// It is impossible to determine the duration during which the value changed
+			// from 0 to the current value.
+			// The following attempts didn't work well:
+			// - using scrape interval as the duration. It fails on Prometheus restarts when it
+			//   skips scraping for the counter. This results in too high rate() value for the first point
+			//   after Prometheus restarts.
+			// - using window or step as the duration. It results in too small rate() values for the first
+			//   points of time series.
+			//
+			// So just return nan
 			return nan
 		}
 		return (values[0] - rfa.prevValue) / (float64(timestamps[0]-rfa.prevTimestamp) * 1e-3)
@@ -955,16 +1412,13 @@ func rollupResets(rfa *rollupFuncArg) float64 {
 }
 
 func rollupFirst(rfa *rollupFuncArg) float64 {
-	// See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness
-	v := rfa.prevValue
-	if !math.IsNaN(v) {
-		return v
-	}
-
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
+		// Do not take into account rfa.prevValue, since it may lead
+		// to inconsistent results comparing to Prometheus on broken time series
+		// with irregular data points.
 		return nan
 	}
 	return values[0]
@@ -977,7 +1431,10 @@ func rollupLast(rfa *rollupFuncArg) float64 {
 	// before calling rollup funcs.
 	values := rfa.values
 	if len(values) == 0 {
-		return rfa.prevValue
+		// Do not take into account rfa.prevValue, since it may lead
+		// to inconsistent results comparing to Prometheus on broken time series
+		// with irregular data points.
+		return nan
 	}
 	return values[len(values)-1]
 }

app/vmselect/promql/rollup_result_cache.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/VictoriaMetrics/metrics"
@@ -73,8 +74,8 @@ func InitRollupResultCache(cachePath string) {
 		return stats
 	}
 	if len(rollupResultCachePath) > 0 {
-		logger.Infof("loaded rollupResult cache from %q in %s; entriesCount: %d, sizeBytes: %d",
-			rollupResultCachePath, time.Since(startTime), fcs().EntriesCount, fcs().BytesSize)
+		logger.Infof("loaded rollupResult cache from %q in %.3f seconds; entriesCount: %d, sizeBytes: %d",
+			rollupResultCachePath, time.Since(startTime).Seconds(), fcs().EntriesCount, fcs().BytesSize)
 	}
 
 	metrics.NewGauge(`vm_cache_entries{type="promql/rollupResult"}`, func() float64 {
@@ -112,8 +113,8 @@ func StopRollupResultCache() {
 	rollupResultCacheV.c.UpdateStats(&fcs)
 	rollupResultCacheV.c.Stop()
 	rollupResultCacheV.c = nil
-	logger.Infof("saved rollupResult cache to %q in %s; entriesCount: %d, sizeBytes: %d",
-		rollupResultCachePath, time.Since(startTime), fcs.EntriesCount, fcs.BytesSize)
+	logger.Infof("saved rollupResult cache to %q in %.3f seconds; entriesCount: %d, sizeBytes: %d",
+		rollupResultCachePath, time.Since(startTime).Seconds(), fcs.EntriesCount, fcs.BytesSize)
 }
 
 type rollupResultCache struct {
@@ -128,7 +129,7 @@ func ResetRollupResultCache() {
 	rollupResultCacheV.c.Reset()
 }
 
-func (rrc *rollupResultCache) Get(funcName string, ec *EvalConfig, me *metricExpr, iafc *incrementalAggrFuncContext, window int64) (tss []*timeseries, newStart int64) {
+func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window int64) (tss []*timeseries, newStart int64) {
 	if *disableCache || !ec.mayCache() {
 		return nil, ec.Start
 	}
@@ -137,7 +138,7 @@ func (rrc *rollupResultCache) Get(funcName string, ec *EvalConfig, me *metricExp
 	bb := bbPool.Get()
 	defer bbPool.Put(bb)
 
-	bb.B = marshalRollupResultCacheKey(bb.B[:0], funcName, me, iafc, window, ec.Step)
+	bb.B = marshalRollupResultCacheKey(bb.B[:0], expr, window, ec.Step)
 	metainfoBuf := rrc.c.Get(nil, bb.B)
 	if len(metainfoBuf) == 0 {
 		return nil, ec.Start
@@ -157,7 +158,7 @@ func (rrc *rollupResultCache) Get(funcName string, ec *EvalConfig, me *metricExp
 	if len(compressedResultBuf.B) == 0 {
 		mi.RemoveKey(key)
 		metainfoBuf = mi.Marshal(metainfoBuf[:0])
-		bb.B = marshalRollupResultCacheKey(bb.B[:0], funcName, me, iafc, window, ec.Step)
+		bb.B = marshalRollupResultCacheKey(bb.B[:0], expr, window, ec.Step)
 		rrc.c.Set(bb.B, metainfoBuf)
 		return nil, ec.Start
 	}
@@ -209,7 +210,7 @@ func (rrc *rollupResultCache) Get(funcName string, ec *EvalConfig, me *metricExp
 
 var resultBufPool bytesutil.ByteBufferPool
 
-func (rrc *rollupResultCache) Put(funcName string, ec *EvalConfig, me *metricExpr, iafc *incrementalAggrFuncContext, window int64, tss []*timeseries) {
+func (rrc *rollupResultCache) Put(ec *EvalConfig, expr metricsql.Expr, window int64, tss []*timeseries) {
 	if *disableCache || len(tss) == 0 || !ec.mayCache() {
 		return
 	}
@@ -260,7 +261,7 @@ func (rrc *rollupResultCache) Put(funcName string, ec *EvalConfig, me *metricExp
 	bb.B = key.Marshal(bb.B[:0])
 	rrc.c.SetBig(bb.B, compressedResultBuf.B)
 
-	bb.B = marshalRollupResultCacheKey(bb.B[:0], funcName, me, iafc, window, ec.Step)
+	bb.B = marshalRollupResultCacheKey(bb.B[:0], expr, window, ec.Step)
 	metainfoBuf := rrc.c.Get(nil, bb.B)
 	var mi rollupResultCacheMetainfo
 	if len(metainfoBuf) > 0 {
@@ -288,23 +289,13 @@ var (
 var tooBigRollupResults = metrics.NewCounter("vm_too_big_rollup_results_total")
 
 // Increment this value every time the format of the cache changes.
-const rollupResultCacheVersion = 6
+const rollupResultCacheVersion = 7
 
-func marshalRollupResultCacheKey(dst []byte, funcName string, me *metricExpr, iafc *incrementalAggrFuncContext, window, step int64) []byte {
+func marshalRollupResultCacheKey(dst []byte, expr metricsql.Expr, window, step int64) []byte {
 	dst = append(dst, rollupResultCacheVersion)
-	if iafc == nil {
-		dst = append(dst, 0)
-	} else {
-		dst = append(dst, 1)
-		dst = iafc.ae.AppendString(dst)
-	}
-	dst = encoding.MarshalUint64(dst, uint64(len(funcName)))
-	dst = append(dst, funcName...)
 	dst = encoding.MarshalInt64(dst, window)
 	dst = encoding.MarshalInt64(dst, step)
-	for i := range me.TagFilters {
-		dst = me.TagFilters[i].Marshal(dst)
-	}
+	dst = expr.AppendString(dst)
 	return dst
 }
 

app/vmselect/promql/rollup_result_cache_test.go

@@ -3,12 +3,12 @@ package promql
 import (
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 func TestRollupResultCache(t *testing.T) {
 	ResetRollupResultCache()
-	funcName := "foo"
 	window := int64(456)
 	ec := &EvalConfig{
 		Start: 1000,
@@ -17,21 +17,24 @@ func TestRollupResultCache(t *testing.T) {
 
 		MayCache: true,
 	}
-	me := &metricExpr{
-		TagFilters: []storage.TagFilter{{
-			Key:   []byte("aaa"),
-			Value: []byte("xxx"),
+	me := &metricsql.MetricExpr{
+		LabelFilters: []metricsql.LabelFilter{{
+			Label: "aaa",
+			Value: "xxx",
 		}},
 	}
-	iafc := &incrementalAggrFuncContext{
-		ae: &aggrFuncExpr{
-			Name: "foobar",
-		},
+	fe := &metricsql.FuncExpr{
+		Name: "foo",
+		Args: []metricsql.Expr{me},
+	}
+	ae := &metricsql.AggrFuncExpr{
+		Name: "foobar",
+		Args: []metricsql.Expr{fe},
 	}
 
 	// Try obtaining an empty value.
 	t.Run("empty", func(t *testing.T) {
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != ec.Start {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, ec.Start)
 		}
@@ -41,7 +44,7 @@ func TestRollupResultCache(t *testing.T) {
 	})
 
 	// Store timeseries overlapping with start
-	t.Run("start-overlap-no-iafc", func(t *testing.T) {
+	t.Run("start-overlap-no-ae", func(t *testing.T) {
 		ResetRollupResultCache()
 		tss := []*timeseries{
 			{
@@ -49,8 +52,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 1400 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
 		}
@@ -62,7 +65,7 @@ func TestRollupResultCache(t *testing.T) {
 		}
 		testTimeseriesEqual(t, tss, tssExpected)
 	})
-	t.Run("start-overlap-with-iafc", func(t *testing.T) {
+	t.Run("start-overlap-with-ae", func(t *testing.T) {
 		ResetRollupResultCache()
 		tss := []*timeseries{
 			{
@@ -70,8 +73,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, iafc, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, iafc, window)
+		rollupResultCacheV.Put(ec, ae, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, ae, window)
 		if newStart != 1400 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
 		}
@@ -93,8 +96,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{333, 0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -112,8 +115,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -131,8 +134,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -150,8 +153,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 1000 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
 		}
@@ -169,8 +172,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2, 3, 4, 5, 6, 7},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 2200 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
 		}
@@ -192,8 +195,8 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{1, 2, 3, 4, 5, 6},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 2200 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
 		}
@@ -217,8 +220,8 @@ func TestRollupResultCache(t *testing.T) {
 			}
 			tss = append(tss, ts)
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss)
-		tssResult, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss)
+		tssResult, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 2200 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
 		}
@@ -246,10 +249,10 @@ func TestRollupResultCache(t *testing.T) {
 				Values:     []float64{0, 1, 2},
 			},
 		}
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss1)
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss2)
-		rollupResultCacheV.Put(funcName, ec, me, nil, window, tss3)
-		tss, newStart := rollupResultCacheV.Get(funcName, ec, me, nil, window)
+		rollupResultCacheV.Put(ec, fe, window, tss1)
+		rollupResultCacheV.Put(ec, fe, window, tss2)
+		rollupResultCacheV.Put(ec, fe, window, tss3)
+		tss, newStart := rollupResultCacheV.Get(ec, fe, window)
 		if newStart != 1400 {
 			t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
 		}

app/vmselect/promql/rollup_test.go

@@ -3,6 +3,8 @@ package promql
 import (
 	"math"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 )
 
 var (
@@ -57,7 +59,7 @@ func TestRollupIderivDuplicateTimestamps(t *testing.T) {
 	}
 	n = rollupIderiv(rfa)
 	if n != 500 {
-		t.Fatalf("unexpected value; got %v; want %v", n, 0.5)
+		t.Fatalf("unexpected value; got %v; want %v", n, 500)
 	}
 
 	rfa = &rollupFuncArg{
@@ -157,7 +159,7 @@ func TestDerivValues(t *testing.T) {
 	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
 }
 
-func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpected *metricExpr, vExpected float64) {
+func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpected *metricsql.MetricExpr, vExpected float64) {
 	t.Helper()
 	nrf := getRollupFunc(funcName)
 	if nrf == nil {
@@ -190,6 +192,52 @@ func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpecte
 	}
 }
 
+func TestRollupShareLEOverTime(t *testing.T) {
+	f := func(le, vExpected float64) {
+		t.Helper()
+		les := []*timeseries{{
+			Values:     []float64{le},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
+		testRollupFunc(t, "share_le_over_time", args, &me, vExpected)
+	}
+
+	f(-123, 0)
+	f(0, 0)
+	f(10, 0)
+	f(12, 0.08333333333333333)
+	f(30, 0.16666666666666666)
+	f(50, 0.75)
+	f(100, 0.9166666666666666)
+	f(123, 1)
+	f(1000, 1)
+}
+
+func TestRollupShareGTOverTime(t *testing.T) {
+	f := func(gt, vExpected float64) {
+		t.Helper()
+		gts := []*timeseries{{
+			Values:     []float64{gt},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, gts}
+		testRollupFunc(t, "share_gt_over_time", args, &me, vExpected)
+	}
+
+	f(-123, 1)
+	f(0, 1)
+	f(10, 1)
+	f(12, 0.9166666666666666)
+	f(30, 0.8333333333333334)
+	f(50, 0.25)
+	f(100, 0.08333333333333333)
+	f(123, 0)
+	f(1000, 0)
+}
+
 func TestRollupQuantileOverTime(t *testing.T) {
 	f := func(phi, vExpected float64) {
 		t.Helper()
@@ -197,8 +245,8 @@ func TestRollupQuantileOverTime(t *testing.T) {
 			Values:     []float64{phi},
 			Timestamps: []int64{123},
 		}}
-		var me metricExpr
-		args := []interface{}{phis, &rollupExpr{Expr: &me}}
+		var me metricsql.MetricExpr
+		args := []interface{}{phis, &metricsql.RollupExpr{Expr: &me}}
 		testRollupFunc(t, "quantile_over_time", args, &me, vExpected)
 	}
 
@@ -219,8 +267,8 @@ func TestRollupPredictLinear(t *testing.T) {
 			Values:     []float64{sec},
 			Timestamps: []int64{123},
 		}}
-		var me metricExpr
-		args := []interface{}{&rollupExpr{Expr: &me}, secs}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, secs}
 		testRollupFunc(t, "predict_linear", args, &me, vExpected)
 	}
 
@@ -241,8 +289,8 @@ func TestRollupHoltWinters(t *testing.T) {
 			Values:     []float64{tf},
 			Timestamps: []int64{123},
 		}}
-		var me metricExpr
-		args := []interface{}{&rollupExpr{Expr: &me}, sfs, tfs}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, sfs, tfs}
 		testRollupFunc(t, "holt_winters", args, &me, vExpected)
 	}
 
@@ -262,27 +310,72 @@ func TestRollupHoltWinters(t *testing.T) {
 	f(0.9, 0.9, 33.99637566941818)
 }
 
+func TestRollupHoeffdingBoundLower(t *testing.T) {
+	f := func(phi, vExpected float64) {
+		t.Helper()
+		phis := []*timeseries{{
+			Values:     []float64{phi},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{phis, &metricsql.RollupExpr{Expr: &me}}
+		testRollupFunc(t, "hoeffding_bound_lower", args, &me, vExpected)
+	}
+
+	f(0.5, 28.21949401521037)
+	f(-1, 47.083333333333336)
+	f(0, 47.083333333333336)
+	f(1, -inf)
+	f(2, -inf)
+	f(0.1, 39.72878000047643)
+	f(0.9, 12.701803086472331)
+}
+
+func TestRollupHoeffdingBoundUpper(t *testing.T) {
+	f := func(phi, vExpected float64) {
+		t.Helper()
+		phis := []*timeseries{{
+			Values:     []float64{phi},
+			Timestamps: []int64{123},
+		}}
+		var me metricsql.MetricExpr
+		args := []interface{}{phis, &metricsql.RollupExpr{Expr: &me}}
+		testRollupFunc(t, "hoeffding_bound_upper", args, &me, vExpected)
+	}
+
+	f(0.5, 65.9471726514563)
+	f(-1, 47.083333333333336)
+	f(0, 47.083333333333336)
+	f(1, inf)
+	f(2, inf)
+	f(0.1, 54.43788666619024)
+	f(0.9, 81.46486358019433)
+}
+
 func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f := func(funcName string, vExpected float64) {
 		t.Helper()
-		var me metricExpr
-		args := []interface{}{&rollupExpr{Expr: &me}}
+		var me metricsql.MetricExpr
+		args := []interface{}{&metricsql.RollupExpr{Expr: &me}}
 		testRollupFunc(t, funcName, args, &me, vExpected)
 	}
 
 	f("default_rollup", 34)
 	f("changes", 11)
-	f("delta", -89)
+	f("delta", 34)
 	f("deriv", -266.85860231406065)
 	f("deriv_fast", -712)
 	f("idelta", 0)
-	f("increase", 275)
+	f("increase", 398)
 	f("irate", 0)
 	f("rate", 2200)
 	f("resets", 5)
+	f("range_over_time", 111)
 	f("avg_over_time", 47.083333333333336)
 	f("min_over_time", 12)
 	f("max_over_time", 123)
+	f("tmin_over_time", 0.08)
+	f("tmax_over_time", 0.005)
 	f("sum_over_time", 565)
 	f("sum2_over_time", 37951)
 	f("geomean_over_time", 39.33466603189148)
@@ -327,7 +420,7 @@ func TestRollupNewRollupFuncError(t *testing.T) {
 		Values:     []float64{321},
 		Timestamps: []int64{123},
 	}}
-	me := &metricExpr{}
+	me := &metricsql.MetricExpr{}
 	f("holt_winters", []interface{}{123, 123, 321})
 	f("holt_winters", []interface{}{me, 123, 321})
 	f("holt_winters", []interface{}{me, scalarTs, 321})
@@ -409,7 +502,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 123, 123, 34, 34}
+		valuesExpected := []float64{nan, 123, nan, 34, nan, 44}
 		timestampsExpected := []int64{0, 5, 10, 15, 20, 25}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -423,7 +516,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{12, 44, 34, nan}
+		valuesExpected := []float64{44, 32, 34, nan}
 		timestampsExpected := []int64{100, 120, 140, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -437,7 +530,7 @@ func TestRollupNoWindowPartialPoints(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, nan, 123, 54, 44}
+		valuesExpected := []float64{nan, nan, 123, 34, 32}
 		timestampsExpected := []int64{-50, 0, 50, 100, 150}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -499,7 +592,7 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{99, 12, 44, nan, 32, 34, nan}
+		valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
 		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -513,7 +606,7 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{99, 12, 44, 44, 32, 34, nan}
+		valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
 		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -527,7 +620,7 @@ func TestRollupFuncsLookbackDelta(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{34, 12, 12, 44, 44, 34, nan}
+		valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan}
 		timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -544,7 +637,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 123, 21, 12, 34}
+		valuesExpected := []float64{nan, 123, 54, 44, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -572,7 +665,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, 21, 12, 12, 34}
+		valuesExpected := []float64{nan, 21, 12, 32, 34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -614,7 +707,7 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		}
 		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
 		values := rc.Do(nil, testValues, testTimestamps)
-		valuesExpected := []float64{nan, -102, -9, 22, 0}
+		valuesExpected := []float64{nan, nan, -9, 22, 0}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
@@ -772,6 +865,20 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
+	t.Run("deriv_fast", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:   rollupDerivFast,
+			Start:  0,
+			End:    20,
+			Step:   4,
+			Window: 0,
+		}
+		rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step)
+		values := rc.Do(nil, testValues, testTimestamps)
+		valuesExpected := []float64{nan, nan, nan, 0, -8900, 0}
+		timestampsExpected := []int64{0, 4, 8, 12, 16, 20}
+		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
 	t.Run("ideriv", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:   rollupIderiv,

app/vmselect/promql/transform.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/valyala/histogram"
 )
@@ -64,6 +65,8 @@ var transformFuncs = map[string]transformFunc{
 	"label_move":         transformLabelMove,
 	"label_transform":    transformLabelTransform,
 	"label_value":        transformLabelValue,
+	"label_match":        transformLabelMatch,
+	"label_mismatch":     transformLabelMismatch,
 	"union":              transformUnion,
 	"":                   transformUnion, // empty func is a synonim to union
 	"keep_last_value":    transformKeepLastValue,
@@ -92,6 +95,7 @@ var transformFuncs = map[string]transformFunc{
 	"asin":               newTransformFuncOneArg(transformAsin),
 	"acos":               newTransformFuncOneArg(transformAcos),
 	"prometheus_buckets": transformPrometheusBuckets,
+	"histogram_share":    transformHistogramShare,
 }
 
 func getTransformFunc(s string) transformFunc {
@@ -99,13 +103,9 @@ func getTransformFunc(s string) transformFunc {
 	return transformFuncs[s]
 }
 
-func isTransformFunc(s string) bool {
-	return getTransformFunc(s) != nil
-}
-
 type transformFuncArg struct {
 	ec   *EvalConfig
-	fe   *funcExpr
+	fe   *metricsql.FuncExpr
 	args [][]*timeseries
 }
 
@@ -126,7 +126,7 @@ func newTransformFuncOneArg(tf func(v float64) float64) transformFunc {
 	}
 }
 
-func doTransformValues(arg []*timeseries, tf func(values []float64), fe *funcExpr) ([]*timeseries, error) {
+func doTransformValues(arg []*timeseries, tf func(values []float64), fe *metricsql.FuncExpr) ([]*timeseries, error) {
 	name := strings.ToLower(fe.Name)
 	keepMetricGroup := transformFuncsKeepMetricGroup[name]
 	for _, ts := range arg {
@@ -149,28 +149,10 @@ func transformAbsent(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 	arg := args[0]
-
 	if len(arg) == 0 {
-		// Copy tags from arg
-		rvs := evalNumber(tfa.ec, 1)
-		rv := rvs[0]
-		me, ok := tfa.fe.Args[0].(*metricExpr)
-		if !ok {
-			return rvs, nil
-		}
-		for i := range me.TagFilters {
-			tf := &me.TagFilters[i]
-			if len(tf.Key) == 0 {
-				continue
-			}
-			if tf.IsRegexp || tf.IsNegative {
-				continue
-			}
-			rv.MetricName.AddTagBytes(tf.Key, tf.Value)
-		}
+		rvs := getAbsentTimeseries(tfa.ec, tfa.fe.Args[0])
 		return rvs, nil
 	}
-
 	for _, ts := range arg {
 		ts.MetricName.ResetMetricGroup()
 		for i, v := range ts.Values {
@@ -185,6 +167,28 @@ func transformAbsent(tfa *transformFuncArg) ([]*timeseries, error) {
 	return arg, nil
 }
 
+func getAbsentTimeseries(ec *EvalConfig, arg metricsql.Expr) []*timeseries {
+	// Copy tags from arg
+	rvs := evalNumber(ec, 1)
+	rv := rvs[0]
+	me, ok := arg.(*metricsql.MetricExpr)
+	if !ok {
+		return rvs
+	}
+	tfs := toTagFilters(me.LabelFilters)
+	for i := range tfs {
+		tf := &tfs[i]
+		if len(tf.Key) == 0 {
+			continue
+		}
+		if tf.IsRegexp || tf.IsNegative {
+			continue
+		}
+		rv.MetricName.AddTagBytes(tf.Key, tf.Value)
+	}
+	return rvs
+}
+
 func transformCeil(v float64) float64 {
 	return math.Ceil(v)
 }
@@ -398,6 +402,105 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 	return rvs
 }
 
+func transformHistogramShare(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if len(args) < 2 || len(args) > 3 {
+		return nil, fmt.Errorf("unexpected number of args; got %d; want 2...3", len(args))
+	}
+	les, err := getScalar(args[0], 0)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse le: %s", err)
+	}
+
+	// Convert buckets with `vmrange` labels to buckets with `le` labels.
+	tss := vmrangeBucketsToLE(args[1])
+
+	// Parse boundsLabel. See https://github.com/prometheus/prometheus/issues/5706 for details.
+	var boundsLabel string
+	if len(args) > 2 {
+		s, err := getString(args[2], 2)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse boundsLabel (arg #3): %s", err)
+		}
+		boundsLabel = s
+	}
+
+	// Group metrics by all tags excluding "le"
+	m := groupLeTimeseries(tss)
+
+	// Calculate share for les
+
+	share := func(i int, les []float64, xss []leTimeseries) (q, lower, upper float64) {
+		leReq := les[i]
+		if math.IsNaN(leReq) || len(xss) == 0 {
+			return nan, nan, nan
+		}
+		fixBrokenBuckets(i, xss)
+		if leReq < 0 {
+			return 0, 0, 0
+		}
+		if math.IsInf(leReq, 1) {
+			return 1, 1, 1
+		}
+		var vPrev, lePrev float64
+		for _, xs := range xss {
+			v := xs.ts.Values[i]
+			le := xs.le
+			if leReq >= le {
+				vPrev = v
+				lePrev = le
+				continue
+			}
+			// precondition: lePrev <= leReq < le
+			vLast := xss[len(xss)-1].ts.Values[i]
+			lower = vPrev / vLast
+			if math.IsInf(le, 1) {
+				return lower, lower, 1
+			}
+			if lePrev == leReq {
+				return lower, lower, lower
+			}
+			upper = v / vLast
+			q = lower + (v-vPrev)/vLast*(leReq-lePrev)/(le-lePrev)
+			return q, lower, upper
+		}
+		// precondition: leReq > leLast
+		return 1, 1, 1
+	}
+	rvs := make([]*timeseries, 0, len(m))
+	for _, xss := range m {
+		sort.Slice(xss, func(i, j int) bool {
+			return xss[i].le < xss[j].le
+		})
+		dst := xss[0].ts
+		var tsLower, tsUpper *timeseries
+		if len(boundsLabel) > 0 {
+			tsLower = &timeseries{}
+			tsLower.CopyFromShallowTimestamps(dst)
+			tsLower.MetricName.RemoveTag(boundsLabel)
+			tsLower.MetricName.AddTag(boundsLabel, "lower")
+			tsUpper = &timeseries{}
+			tsUpper.CopyFromShallowTimestamps(dst)
+			tsUpper.MetricName.RemoveTag(boundsLabel)
+			tsUpper.MetricName.AddTag(boundsLabel, "upper")
+		}
+		for i := range dst.Values {
+			q, lower, upper := share(i, les, xss)
+			dst.Values[i] = q
+			if len(boundsLabel) > 0 {
+				tsLower.Values[i] = lower
+				tsUpper.Values[i] = upper
+			}
+		}
+		rvs = append(rvs, dst)
+		if len(boundsLabel) > 0 {
+			rvs = append(rvs, tsLower)
+			rvs = append(rvs, tsUpper)
+		}
+	}
+	return rvs, nil
+}
+
 func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
 	if len(args) < 2 || len(args) > 3 {
@@ -422,73 +525,35 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	}
 
 	// Group metrics by all tags excluding "le"
-	type x struct {
-		le float64
-		ts *timeseries
-	}
-	m := make(map[string][]x)
-	bb := bbPool.Get()
-	for _, ts := range tss {
-		tagValue := ts.MetricName.GetTagValue("le")
-		if len(tagValue) == 0 {
-			continue
-		}
-		le, err := strconv.ParseFloat(bytesutil.ToUnsafeString(tagValue), 64)
-		if err != nil {
-			continue
-		}
-		ts.MetricName.ResetMetricGroup()
-		ts.MetricName.RemoveTag("le")
-		bb.B = marshalMetricTagsSorted(bb.B[:0], &ts.MetricName)
-		m[string(bb.B)] = append(m[string(bb.B)], x{
-			le: le,
-			ts: ts,
-		})
-	}
-	bbPool.Put(bb)
+	m := groupLeTimeseries(tss)
 
 	// Calculate quantile for each group in m
 
-	lastNonInf := func(i int, xss []x) float64 {
+	lastNonInf := func(i int, xss []leTimeseries) float64 {
 		for len(xss) > 0 {
 			xsLast := xss[len(xss)-1]
 			v := xsLast.ts.Values[i]
 			if v == 0 {
 				return nan
 			}
-			if !math.IsNaN(v) && !math.IsInf(xsLast.le, 0) {
+			if !math.IsInf(xsLast.le, 0) {
 				return xsLast.le
 			}
 			xss = xss[:len(xss)-1]
 		}
 		return nan
 	}
-	quantile := func(i int, phis []float64, xss []x) (q, lower, upper float64) {
+	quantile := func(i int, phis []float64, xss []leTimeseries) (q, lower, upper float64) {
 		phi := phis[i]
 		if math.IsNaN(phi) {
 			return nan, nan, nan
 		}
-		// Fix broken buckets.
-		// They are already sorted by le, so their values must be in ascending order,
-		// since the next bucket value includes all the previous buckets.
-		vPrev := float64(0)
-		for _, xs := range xss {
-			v := xs.ts.Values[i]
-			if v < vPrev {
-				xs.ts.Values[i] = vPrev
-			} else if !math.IsNaN(v) {
-				vPrev = v
-			}
-		}
-		vLast := nan
-		for len(xss) > 0 {
+		fixBrokenBuckets(i, xss)
+		vLast := float64(0)
+		if len(xss) > 0 {
 			vLast = xss[len(xss)-1].ts.Values[i]
-			if !math.IsNaN(vLast) {
-				break
-			}
-			xss = xss[:len(xss)-1]
 		}
-		if vLast == 0 || math.IsNaN(vLast) {
+		if vLast == 0 {
 			return nan, nan, nan
 		}
 		if phi < 0 {
@@ -498,15 +563,10 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 			return inf, vLast, inf
 		}
 		vReq := vLast * phi
-		vPrev = 0
+		vPrev := float64(0)
 		lePrev := float64(0)
 		for _, xs := range xss {
 			v := xs.ts.Values[i]
-			if math.IsNaN(v) {
-				// Skip NaNs - they may appear if the selected time range
-				// contains multiple different bucket sets.
-				continue
-			}
 			le := xs.le
 			if v <= 0 {
 				// Skip zero buckets.
@@ -565,6 +625,50 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 	return rvs, nil
 }
 
+type leTimeseries struct {
+	le float64
+	ts *timeseries
+}
+
+func groupLeTimeseries(tss []*timeseries) map[string][]leTimeseries {
+	m := make(map[string][]leTimeseries)
+	bb := bbPool.Get()
+	for _, ts := range tss {
+		tagValue := ts.MetricName.GetTagValue("le")
+		if len(tagValue) == 0 {
+			continue
+		}
+		le, err := strconv.ParseFloat(bytesutil.ToUnsafeString(tagValue), 64)
+		if err != nil {
+			continue
+		}
+		ts.MetricName.ResetMetricGroup()
+		ts.MetricName.RemoveTag("le")
+		bb.B = marshalMetricTagsSorted(bb.B[:0], &ts.MetricName)
+		m[string(bb.B)] = append(m[string(bb.B)], leTimeseries{
+			le: le,
+			ts: ts,
+		})
+	}
+	bbPool.Put(bb)
+	return m
+}
+
+func fixBrokenBuckets(i int, xss []leTimeseries) {
+	// Fix broken buckets.
+	// They are already sorted by le, so their values must be in ascending order,
+	// since the next bucket includes all the previous buckets.
+	vPrev := float64(0)
+	for _, xs := range xss {
+		v := xs.ts.Values[i]
+		if v < vPrev || math.IsNaN(v) {
+			xs.ts.Values[i] = vPrev
+		} else {
+			vPrev = v
+		}
+	}
+}
+
 func transformHour(t time.Time) int {
 	return t.Hour()
 }
@@ -1023,7 +1127,7 @@ func transformLabelTransform(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 
-	r, err := compileRegexp(regex)
+	r, err := metricsql.CompileRegexp(regex)
 	if err != nil {
 		return nil, fmt.Errorf(`cannot compile regex %q: %s`, regex, err)
 	}
@@ -1052,7 +1156,7 @@ func transformLabelReplace(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 
-	r, err := compileRegexpAnchored(regex)
+	r, err := metricsql.CompileRegexpAnchored(regex)
 	if err != nil {
 		return nil, fmt.Errorf(`cannot compile regex %q: %s`, regex, err)
 	}
@@ -1101,6 +1205,62 @@ func transformLabelValue(tfa *transformFuncArg) ([]*timeseries, error) {
 	return rvs, nil
 }
 
+func transformLabelMatch(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if err := expectTransformArgsNum(args, 3); err != nil {
+		return nil, err
+	}
+	labelName, err := getString(args[1], 1)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get label name: %s", err)
+	}
+	labelRe, err := getString(args[2], 2)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get regexp: %s", err)
+	}
+	r, err := metricsql.CompileRegexpAnchored(labelRe)
+	if err != nil {
+		return nil, fmt.Errorf(`cannot compile regexp %q: %s`, labelRe, err)
+	}
+	tss := args[0]
+	rvs := tss[:0]
+	for _, ts := range tss {
+		labelValue := ts.MetricName.GetTagValue(labelName)
+		if r.Match(labelValue) {
+			rvs = append(rvs, ts)
+		}
+	}
+	return rvs, nil
+}
+
+func transformLabelMismatch(tfa *transformFuncArg) ([]*timeseries, error) {
+	args := tfa.args
+	if err := expectTransformArgsNum(args, 3); err != nil {
+		return nil, err
+	}
+	labelName, err := getString(args[1], 1)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get label name: %s", err)
+	}
+	labelRe, err := getString(args[2], 2)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get regexp: %s", err)
+	}
+	r, err := metricsql.CompileRegexpAnchored(labelRe)
+	if err != nil {
+		return nil, fmt.Errorf(`cannot compile regexp %q: %s`, labelRe, err)
+	}
+	tss := args[0]
+	rvs := tss[:0]
+	for _, ts := range tss {
+		labelValue := ts.MetricName.GetTagValue(labelName)
+		if !r.Match(labelValue) {
+			rvs = append(rvs, ts)
+		}
+	}
+	return rvs, nil
+}
+
 func transformLn(v float64) float64 {
 	return math.Log(v)
 }
@@ -1163,7 +1323,7 @@ func transformScalar(tfa *transformFuncArg) ([]*timeseries, error) {
 
 	// Verify whether the arg is a string.
 	// Then try converting the string to number.
-	if se, ok := tfa.fe.Args[0].(*stringExpr); ok {
+	if se, ok := tfa.fe.Args[0].(*metricsql.StringExpr); ok {
 		n, err := strconv.ParseFloat(se.S, 64)
 		if err != nil {
 			n = nan

app/vmstorage/main.go

@@ -62,8 +62,8 @@ func InitWithoutMetrics() {
 	blocksCount := tm.SmallBlocksCount + tm.BigBlocksCount
 	rowsCount := tm.SmallRowsCount + tm.BigRowsCount
 	sizeBytes := tm.SmallSizeBytes + tm.BigSizeBytes
-	logger.Infof("successfully opened storage %q in %s; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
-		*DataPath, time.Since(startTime), partsCount, blocksCount, rowsCount, sizeBytes)
+	logger.Infof("successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
+		*DataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
 }
 
 // Storage is a storage.
@@ -133,7 +133,7 @@ func Stop() {
 	startTime := time.Now()
 	WG.WaitAndBlock()
 	Storage.MustClose()
-	logger.Infof("successfully closed the storage in %s", time.Since(startTime))
+	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 
 	logger.Infof("the storage has been stopped")
 }

deployment/docker/Makefile

@@ -1,7 +1,7 @@
 # All these commands must run from repository root.
 
 DOCKER_NAMESPACE := docker.io/victoriametrics
-BUILDER_IMAGE := local/builder:go1.13.5
+BUILDER_IMAGE := local/builder:go1.13.6
 CERTS_IMAGE := local/certs:1.0.3
 
 package-certs:

deployment/docker/builder/Dockerfile

@@ -1,2 +1,2 @@
-FROM golang:1.13.5
+FROM golang:1.13.6
 STOPSIGNAL SIGINT

deployment/docker/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.5'
 services:
   prometheus:
     container_name: prometheus
-    image: prom/prometheus:v2.14.0
+    image: prom/prometheus:v2.15.2
     depends_on:
       - "victoriametrics"
     ports:
@@ -35,7 +35,7 @@ services:
     restart: always
   grafana:
     container_name: grafana
-    image: grafana/grafana:6.5.0
+    image: grafana/grafana:6.5.2
     entrypoint: >
       /bin/sh -c "
       cd /var/lib/grafana &&

docs/Articles.md

@@ -20,3 +20,4 @@
 * [Evaluation performance and correctness: VictoriaMetrics response](https://medium.com/@valyala/evaluating-performance-and-correctness-victoriametrics-response-e27315627e87)
 * [Improving histogram usability for Prometheus and Grafana](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
 * [Prometheus storage: tech terms for humans](https://medium.com/@valyala/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
+* [Billy: how VictoriaMetrics deals with more than 500 billion rows](https://medium.com/@valyala/billy-how-victoriametrics-deals-with-more-than-500-billion-rows-e82ff8f725da)

docs/CaseStudies.md

@@ -1,8 +1,28 @@
-## Case studies
+## Case studies and talks
 
-Below are approved public case studies from VictoriaMetrics users. Join our [community Slack channel](http://slack.victoriametrics.com/)
+Below are approved public case studies and talks from VictoriaMetrics users. Join our [community Slack channel](http://slack.victoriametrics.com/)
 and feel free asking for references, reviews and additional case studies from real VictoriaMetrics users there.
 
+### Adidas
+
+See [slides](https://promcon.io/2019-munich/slides/remote-write-storage-wars.pdf) and [video](https://youtu.be/OsH6gPdxR4s)
+from [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk at [PromCon 2019](https://promcon.io/2019-munich/).
+VictoriaMetrics is compared to Thanos, Corex and M3DB in the talk.
+
+
+### COLOPL
+
+[COLOPL](http://www.colopl.co.jp/en/) is Japaneese Game Development company. It started using VictoriaMetrics
+after evaulating the following remote storage solutions for Prometheus:
+
+* Cortex
+* Thanos
+* M3DB
+* VictoriaMetrics
+
+See [slides](https://speakerdeck.com/inletorder/monitoring-platform-with-victoria-metrics) and [video](https://www.youtube.com/watch?v=hUpHIluxw80)
+from `Large-scale, super-load system monitoring platform built with VictoriaMetrics` talk at [Prometheus Meetup Tokyo #3](https://prometheus.connpass.com/event/157721/).
+
 
 ### Wix.com
 

docs/Cluster-VictoriaMetrics.md

@@ -3,7 +3,7 @@
 VictoriaMetrics is fast, cost-effective and scalable time series database. It can be used as a long-term remote storage for Prometheus.
 
 It is recommended using [single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics) instead of cluster version
-for ingestion rates lower than 10 million of data points per second.
+for ingestion rates lower than a million of data points per second.
 Single-node version [scales perfectly](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
 with the number of CPU cores, RAM and available storage space.
 Single-node version is easier to configure and operate comparing to cluster version, so think twice before sticking to cluster version.
@@ -133,6 +133,7 @@ with [the official Grafana dashboard for VictoriaMetrics cluster](https://grafan
   - `<suffix>` may have the following values:
      - `prometheus` - for inserting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write)
      - `influx/write` or `influx/api/v2/write` - for inserting data with [Influx line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/)
+     - `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html).
      - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` on `vmselect` (see below).
 
 * URLs for querying: `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:

docs/ExtendedPromQL.md

@@ -2,6 +2,7 @@
 
 VictoriaMetrics implements MetricsQL - query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 It is backwards compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work the same after switching from Prometheus to VictoriaMetrics.
+[Standalone MetricsQL package](https://godoc.org/github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql) can be used for parsing MetricsQL in external apps.
 
 The following functionality is implemented differently in MetricsQL comparing to PromQL in order to improve user experience:
 * MetricsQL takes into account the previous point before the window in square brackets for range functions such as `rate` and `increase`.
@@ -51,6 +52,7 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
   - `label_move(q, src_label1, dst_label1, ... src_labelN, dst_labelN)` for moving label values from `src_*` to `dst_*`.
   - `label_transform(q, label, regexp, replacement)` for replacing all the `regexp` occurences with `replacement` in the `label` values from `q`.
   - `label_value(q, label)` - returns numeric values for the given `label` from `q`.
+- `label_match(q, label, regexp)` and `label_mismatch(q, label, regexp)` for filtering time series with labels matching (or not matching) the given regexps.
 - `step()` function for returning the step in seconds used in the query.
 - `start()` and `end()` functions for returning the start and end timestamps of the `[start ... end]` range used in the query.
 - `integrate(m[d])` for returning integral over the given duration `d` for the given metric `m`.
@@ -64,6 +66,7 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `lifetime(q[d])` - returns lifetime of `q` over `d` in seconds. It is expected that `d` exceeds the lifetime of `q`.
 - `scrape_interval(q[d])` - returns the average interval in seconds between data points of `q` over `d` aka `scrape interval`.
 - Trigonometric functions - `sin(q)`, `cos(q)`, `asin(q)`, `acos(q)` and `pi()`.
+- `range_over_time(m[d])` - returns value range for `m` over `d` time window, i.e. `max_over_time(m[d])-min_over_time(m[d])`.
 - `median_over_time(m[d])` - calculates median values for `m` over `d` time window. Shorthand to `quantile_over_time(0.5, m[d])`.
 - `median(q)` - median aggregate. Shorthand to `quantile(0.5, q)`.
 - `limitk(k, q)` - limits the number of time series returned from `q` to `k`.
@@ -78,6 +81,11 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `increases_over_time(m[d])` and `decreases_over_time(m[d])` - returns the number of `m` increases or decreases over the given duration `d`.
 - `prometheus_buckets(q)` - converts [VictoriaMetrics histogram](https://godoc.org/github.com/VictoriaMetrics/metrics#Histogram) buckets to Prometheus buckets with `le` labels.
 - `histogram(q)` - calculates aggregate histogram over `q` time series for each point on the graph. See [this article](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) for more details.
+- `histogram_over_time(m[d])` - calculates [VictoriaMetrics histogram](https://godoc.org/github.com/VictoriaMetrics/metrics#Histogram) for `m` over `d`.
+  For example, the following query calculates median temperature by country over the last 24 hours:
+  `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmbucket, country))`.
+- `histogram_share(le, buckets)` - returns share (in the range 0..1) for `buckets`. Useful for calculating SLI and SLO.
+  For instance, the following query returns the share of requests which are performed under 1.5 seconds: `histogram_share(1.5, sum(request_duration_seconds_bucket) by (le))`.
 - `topk_*` and `bottomk_*` aggregate functions, which return up to K time series. Note that the standard `topk` function may return more than K time series -
    see [this article](https://www.robustperception.io/graph-top-n-time-series-in-grafana) for details.
    - `topk_min(k, q)` - returns top K time series with the max minimums on the given time range
@@ -88,3 +96,14 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
    - `bottomk_max(k, q)` - returns bottom K time series with the min maximums on the given time range
    - `bottomk_avg(k, q)` - returns bottom K time series with the min averages on the given time range
    - `bottomk_median(k, q)` - returns bottom K time series with the min medians on the given time range
+- `share_le_over_time(m[d], le)` - returns share (in the range 0..1) of values in `m` over `d`, which are smaller or equal to `le`. Useful for calculating SLI and SLO.
+  Example: `share_le_over_time(memory_usage_bytes[24h], 100*1024*1024)` returns the share of time series values for the last 24 hours when memory usage was below or equal to 100MB.
+- `share_gt_over_time(m[d], gt)` - returns share (in the range 0..1) of values in `m` over `d`, which are bigger than `gt`. Useful for calculating SLI and SLO.
+  Example: `share_gt_over_time(up[24h], 0)` - returns service availability for the last 24 hours.
+- `tmin_over_time(m[d])` - returns timestamp for the minimum value for `m` over `d` time range.
+- `tmax_over_time(m[d])` - returns timestamp for the maximum value for `m` over `d` time range.
+- `aggr_over_time(("aggr_func1", "aggr_func2", ...), m[d])` - simultaneously calculates all the listed `aggr_func*` for `m` over `d` time range.
+  `aggr_func*` can contain any functions that accept range vector. For instance, `aggr_over_time(("min_over_time", "max_over_time", "rate"), m[d])`
+  would calculate `min_over_time`, `max_over_time` and `rate` for `m[d]`.
+- `hoeffding_bound_upper(phi, m[d])` and `hoeffding_bound_lower(phi, m[d])` - return upper and lower [Hoeffding bounds](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality)
+  for the given `phi` in the range `[0..1]`.

docs/FAQ.md

@@ -16,7 +16,7 @@ To provide the best long-term [remote storage](https://prometheus.io/docs/operat
 * High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
   may be crammed into a limited storage comparing to TimescaleDB.
 * Optimized for storage with high-latency IO and low iops (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
-* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, Uber M3, Cortex, InfluxDB or TimescaleDB.
+* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB.
   See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
   and [comparing Thanos to VictoriaMetrics](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
 * Easy operation:
@@ -54,14 +54,56 @@ Yes. Prometheus continues writing data to local storage after enabling remote st
 and new data is available for querying via Prometheus as usual.
 
 
-### How does VictoriaMetrics compare to other clustered TSDBs on top of Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/improbable-eng/thanos), [Cortex](https://github.com/cortexproject/cortex), etc.?
+### How does VictoriaMetrics compare to other clustered TSDBs on top of Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/thanos-io/thanos), [Cortex](https://github.com/cortexproject/cortex), etc.?
 
-VictoriaMetrics is simpler, faster, more cost-effective and it provides [useful extensions for PromQL](ExtendedPromQL). The simplicity is twofold:
-- It is simpler to configure and operate. There is no need in configuring third-party [sidecars](https://github.com/improbable-eng/thanos/blob/master/docs/components/sidecar.md)
-  or fighting with [gossip protocol](https://github.com/improbable-eng/thanos/blob/master/docs/proposals/completed/201809_gossip-removal.md).
-- VictoriaMetrics has simpler architecture, which means less bugs and more useful features in a long run comparing to competing TSDBs.
+VictoriaMetrics is simpler, faster, more cost-effective and it provides [MetricsQL with useful extensions for PromQL](ExtendedPromQL). The simplicity is twofold:
+- It is simpler to configure and operate. There is no need in configuring third-party [sidecars](https://github.com/thanos-io/thanos/blob/master/docs/components/sidecar.md)
+  or fighting with [gossip protocol](https://github.com/thanos-io/thanos/blob/master/docs/proposals/completed/201809_gossip-removal.md).
+- VictoriaMetrics has simpler architecture, which means less bugs and more useful features in the long run comparing to competing TSDBs.
 
-See [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
+See [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
+and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk from [PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
+
+VictoriaMetrics also [uses less RAM than Thanos components](https://github.com/thanos-io/thanos/issues/448).
+
+
+### What is the difference between VictoriaMetrics and [Cortex](https://github.com/cortexproject/cortex)?
+
+VictoriaMetrics is similar to Cortex in the following aspects:
+- Both systems accept data from Prometheus via standard [remote_write API](https://prometheus.io/docs/practices/remote_write/),
+  i.e. there is no need in running sidecars unlike in [Thanos](https://github.com/thanos-io/thanos) case.
+- Both systems support multi-tenancy out of the box. See [the corresponding docs for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#url-format).
+
+The main differences between Corex and VictoriaMetrics:
+- Cortex re-uses Prometheus source code, while VictoriaMetrics is written from scratch.
+- Cortex provides [Ruler](https://github.com/cortexproject/cortex/blob/master/docs/architecture.md#ruler) and [Alertmanager](https://github.com/cortexproject/cortex/blob/master/docs/architecture.md#alertmanager) components,
+  which are currently missing in VictoriaMetrics. However, these components can be substituted by [Promxy](https://github.com/jacksontj/promxy#how-do-i-use-alertingrecording-rules-in-promxy).
+- Cortex heavily relies on third-party services such as Consul, Memcache, DynamoDB, BigTable, Cassandra, etc.
+  This may increase operational complexity and reduce system reliability comparing to VictoriaMetrics' case,
+  which doesn't use any external services. Compare [Cortex Architecture](https://github.com/cortexproject/cortex/blob/master/docs/architecture.md)
+  to [VictoriaMetrics architecture](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#architecture-overview).
+- VictoriaMetrics provides [production-ready single-node solution](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md),
+  which is much easier to setup and operate than Cortex cluster.
+- Cortex may lose up to 12 hours of recent data on Ingestor failure - see [the corresponding docs](https://github.com/cortexproject/cortex/blob/master/docs/architecture.md#ingesters-failure-and-data-loss).
+  VictoriaMetrics may lose only a few seconds of recent data, which isn't synced to persistent storage yet.
+  See [this article for details](https://medium.com/@valyala/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
+- Cortex is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
+
+
+### What is the difference between VictoriaMetrics and [Thanos](https://github.com/thanos-io/thanos)?
+
+- Thanos re-uses Prometheus source code, while VictoriaMetrics is written from scratch.
+- Thanos provides [Ruler component](https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md),
+  while VictoriaMetrics relies on [Promxy for alerting and recording rules](https://github.com/jacksontj/promxy#how-do-i-use-alertingrecording-rules-in-promxy).
+- VictoriaMetrics accepts data via [standard remote_write API for Prometheus](https://prometheus.io/docs/practices/remote_write/),
+  while Thanos uses non-standard [Sidecar](https://github.com/thanos-io/thanos/blob/master/docs/components/sidecar.md), which must run alongside each Prometheus instance.
+- Thanos Sidecar requires disabling data compaction in Prometheus, which may hurt Prometheus performance and increase RAM usage.
+- Thanos stores data on object storage (Amazon S3 or Google GCS), while VictoriaMetrics stores data on block storage (GCP persistent disks, Amazon EBS or bare metal HDD).
+- Thanos may lose up to 2 hours of recent data, which wasn't uploaded yet to object storage. VictoriaMetrics may lose only a few seconds of recent data,
+  which isn't synced to persistent storage yet. See [this article for details](https://medium.com/@valyala/wal-usage-looks-broken-in-modern-time-series-databases-b62a627ab704).
+- Thanos may be harder to setup and operate comparing to VictoriaMetrics, since it has more moving parts, which can be connected with less reliable networks.
+  See [this article for details](https://medium.com/faun/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
+- Thanos is usually slower and requires more CPU and RAM than VictoriaMetrics. See [this talk from Adidas at PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
 
 
 ### How does VictoriaMetrics compare to [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/)?
@@ -76,12 +118,13 @@ TimescaleDB insists on using SQL as a query language. While SQL is more powerful
 Additionally, VictoriaMetrics requires [up to 70x less storage space comparing to TimescaleDB](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) for storing the same amount of time series data.
 
 
-### Does VictoriaMetrics use Prometheus technologies like other clustered TSDBs built on top of Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/improbable-eng/thanos), [Cortex](https://github.com/cortexproject/cortex)?
+### Does VictoriaMetrics use Prometheus technologies like other clustered TSDBs built on top of Prometheus such as [M3 from Uber](https://eng.uber.com/m3/), [Thanos](https://github.com/thanos-io/thanos), [Cortex](https://github.com/cortexproject/cortex)?
 
 No. VictoriaMetrics core is written in Go from scratch by [fasthttp](https://github.com/valyala/fasthttp) [author](https://github.com/valyala).
 The architecture is [optimized for storing and querying large amounts of time series data with high cardinality](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac). VictoriaMetrics storage uses [certain ideas from ClickHouse](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282). Special thanks to [Alexey Milovidov](https://github.com/alexey-milovidov).
 
 
+
 ### Are there performance comparisons with other solutions?
 
 Yes:
@@ -115,7 +158,7 @@ This is slow and expensive.
 Prometheus remote read API isn't intended for querying foreign data aka `global query view`. See [this issue](https://github.com/prometheus/prometheus/issues/4456) for details.
 
 So just query VictoriaMetrics directly via [Prometheus Querying API](https://prometheus.io/docs/prometheus/latest/querying/api/)
-or via [Prometheus datasoruce in Grafana](http://docs.grafana.org/features/datasources/prometheus/).
+or via [Prometheus datasource in Grafana](http://docs.grafana.org/features/datasources/prometheus/).
 
 
 ### Does VictoriaMetrics deduplicate data from Prometheus instances scraping the same targets (aka `HA pairs`)?
@@ -133,8 +176,8 @@ The deduplication for Prometheus HA pair may be easily implemented on top of Vic
 ### Where is the source code of VictoriaMetrics?
 
 Source code for the following versions is available in the following places:
-* [Single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics).
-* [Cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
+* [Single-node version](https://github.com/VictoriaMetrics/VictoriaMetrics)
+* [Cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster)
 
 
 ### Does VictoriaMetrics fit for data from IoT sensors and industrial sensors?
@@ -147,7 +190,11 @@ and scales horizontally to multiple nodes.
 
 ### Where can I ask questions about VictoriaMetrics?
 
-See [VictoriaMetrics-users group](https://groups.google.com/forum/#!forum/victorametrics-users).
+Questions about VictoriaMetrics can be asked via the following channels:
+
+- [Slack channel](http://slack.victoriametrics.com/)
+- [Telegram channel](https://t.me/VictoriaMetrics_en)
+- [Google group](https://groups.google.com/forum/#!forum/victorametrics-users)
 
 
 ### Where can I file bugs and feature requests regarding VictoriaMetrics?

docs/Single-server-VictoriaMetrics.md

@@ -8,8 +8,10 @@ in [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). Just downl
 Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 
 
-## Case studies
+## Case studies and talks
 
+* [Adidas](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#adidas)
+* [COLOPL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#colopl)
 * [Wix.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wixcom)
 * [Wedos.com](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#wedoscom)
 * [Dreamteam](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/CaseStudies#dreamteam)
@@ -28,9 +30,11 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
 * High data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
   may be crammed into limited storage comparing to TimescaleDB.
 * Optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
-* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, Uber M3, Cortex, InfluxDB or TimescaleDB.
-  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
-  and [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683).
+* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB.
+  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae),
+  [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
+  and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk
+  from [PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
 * Easy operation:
   * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
@@ -491,6 +495,7 @@ Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/iss
 ### Third-party contributions
 
 * [Unofficial yum repository](https://copr.fedorainfracloud.org/coprs/antonpatsev/VictoriaMetrics/) ([source code](https://github.com/patsevanton/victoriametrics-rpm))
+* [Prometheus -> VictoriaMetrics exporter](https://github.com/ryotarai/prometheus-tsdb-dump)
 
 
 ### How to work with snapshots?
@@ -532,6 +537,19 @@ the deleted time series isn't freed instantly - it is freed during subsequent me
 It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
 before actually deleting the metrics.
 
+The delete API is intended mainly for the following cases:
+
+- One-off deleting of accidentally written invalid (or undesired) time series.
+- One-off deleting of user data due to [GDPR](https://en.wikipedia.org/wiki/General_Data_Protection_Regulation).
+
+It isn't recommended using delete API for the following cases, since it brings non-zero overhead:
+
+- Regular cleanups for unneded data. Just prevent writing unneeded data into VictoriaMetrics.
+- Reducing disk space usage by deleting unneded time series. This doesn't work as expected, since the deleted
+  time series occupy disk space until the next merge operation, which can never occur.
+
+It is better using `-retentionPeriod` command-line flag for efficient pruning of old data.
+
 
 ### How to export time series?
 
@@ -615,7 +633,7 @@ A rough estimation of the required resources for ingestion path:
 * RAM size: less than 1KB per active time series. So, ~1GB of RAM is required for 1M active time series.
   Time series is considered active if new data points have been added to it recently or if it has been recently queried.
   The number of active time series may be obtained from `vm_cache_entries{type="storage/hour_metric_ids"}` metric
-  exproted on the `/metrics` page.
+  exported on the `/metrics` page.
   VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited by `-memory.allowedPercent` flag.
 
 * CPU cores: a CPU core per 300K inserted data points per second. So, ~4 CPU cores are required for processing
@@ -696,7 +714,7 @@ There is no downsampling support at the moment, but:
 - VictoriaMetrics has good compression for on-disk data. See [this article](https://medium.com/@valyala/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932)
   for details.
 
-These properties reduce the need in downsampling. We plan to implement downsampling in the future.
+These properties reduce the need of downsampling. We plan to implement downsampling in the future.
 See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/36) for details.
 
 
@@ -718,8 +736,10 @@ horizontally scalable long-term remote storage for really large Prometheus deplo
 
 ### Alerting
 
-VictoriaMetrics doesn't support rule evaluation and alerting yet, so these actions must be performed either
-on [Prometheus side](https://prometheus.io/docs/alerting/overview/) or on [Grafana side](https://grafana.com/docs/alerting/rules/).
+VictoriaMetrics doesn't support rule evaluation and alerting yet, so these actions can be performed at the following places:
+* At Prometheus - see [the corresponding docs](https://prometheus.io/docs/alerting/overview/).
+* At Promxy - see [the corresponding docs](https://github.com/jacksontj/promxy/blob/master/README.md#how-do-i-use-alertingrecording-rules-in-promxy).
+* At Grafana - see [the corresponding docs](https://grafana.com/docs/alerting/rules/).
 
 
 ### Security
@@ -739,14 +759,14 @@ For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<i
 
 ### Tuning
 
-* There is no need in VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
+* There is no need for VictoriaMetrics tuning since it uses reasonable defaults for command-line flags,
   which are automatically adjusted for the available CPU and RAM resources.
-* There is no need in Operating System tuning since VictoriaMetrics is optimized for default OS settings.
+* There is no need for Operating System tuning since VictoriaMetrics is optimized for default OS settings.
   The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a),
   so Prometheus instances could establish more connections to VictoriaMetrics.
 * The recommended filesystem is `ext4`, the recommended persistent storage is [persistent HDD-based disk on GCP](https://cloud.google.com/compute/docs/disks/#pdspecs),
   since it is protected from hardware failures via internal replication and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).
-  If you plan storing more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
+  If you plan to store more than 1TB of data on `ext4` partition or plan extending it to more than 16TB,
   then the following options are recommended to pass to `mkfs.ext4`:
 
 ```
@@ -777,7 +797,7 @@ The most interesting metrics are:
 ### Troubleshooting
 
 * It is recommended to use default command-line flag values (i.e. don't set them explicitly) until the need
-  in tweaking these flag values arises.
+  of tweaking these flag values arises.
 
 * If VictoriaMetrics works slowly and eats more than a CPU core per 100K ingested data points per second,
   then it is likely you have too many active time series for the current amount of RAM.
@@ -791,13 +811,14 @@ The most interesting metrics are:
   has at least 20% of free space comparing to disk size.
 
 * If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
-  then just remove directoreis with broken parts. This will recover VictoriaMetrics at the cost
+  then just remove directories with broken parts. This will recover VictoriaMetrics at the cost
   of data loss stored in the broken parts. In the future, `vmrecover` tool will be created
   for automatic recovering from such errors.
 
 
 ### Backfilling
 
+VictoriaMetrics accepts historical data in arbitrary order of time.
 Make sure that configured `-retentionPeriod` covers timestamps for the backfilled data.
 
 It is recommended disabling query cache with `-search.disableCache` command-line flag when writing
@@ -842,7 +863,7 @@ The collected profiles may be analyzed with [go tool pprof](https://github.com/g
 - [ ] CLI tool for data migration, re-balancing and adding/removing nodes [#103](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/103)
 
 
-The discussion happens [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/129). Feel free to comment any item or add own one.
+The discussion happens [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/129). Feel free to comment on any item or add you own one.
 
 
 ## Contacts
@@ -855,6 +876,7 @@ Contact us with any questions regarding VictoriaMetrics at [info@victoriametrics
 Feel free asking any questions regarding VictoriaMetrics:
 
 - [slack](http://slack.victoriametrics.com/)
+- [reddit](https://www.reddit.com/r/VictoriaMetrics/)
 - [telegram-en](https://t.me/VictoriaMetrics_en)
 - [telegram-ru](https://t.me/VictoriaMetrics_ru1)
 - [google groups](https://groups.google.com/forum/#!forum/victorametrics-users)
@@ -885,7 +907,7 @@ Report bugs and propose new features [here](https://github.com/VictoriaMetrics/V
 
 ## Victoria Metrics Logo
 
-[Zip](VM_logo.zip) contains three folders with different image orientation (main color and inverted version).
+[Zip](VM_logo.zip) contains three folders with different image orientations (main color and inverted version).
 
 Files included in each folder:
 

go.mod

@@ -1,32 +1,27 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
 require (
-	cloud.google.com/go v0.49.0 // indirect
-	cloud.google.com/go/storage v1.4.0
-	github.com/VictoriaMetrics/fastcache v1.5.4
+	cloud.google.com/go v0.51.0 // indirect
+	cloud.google.com/go/storage v1.5.0
+	github.com/VictoriaMetrics/fastcache v1.5.7
 	github.com/VictoriaMetrics/metrics v1.9.3
-	github.com/aws/aws-sdk-go v1.26.0
+	github.com/aws/aws-sdk-go v1.28.7
 	github.com/cespare/xxhash/v2 v2.1.1
-	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/snappy v0.0.1
-	github.com/jstemmer/go-junit-report v0.9.1 // indirect
-	github.com/klauspost/compress v1.9.4
-	github.com/valyala/fastjson v1.4.1
+	github.com/klauspost/compress v1.9.8
+	github.com/valyala/fastjson v1.4.5
 	github.com/valyala/fastrand v1.0.0
-	github.com/valyala/gozstd v1.6.3
+	github.com/valyala/gozstd v1.6.4
 	github.com/valyala/histogram v1.0.1
 	github.com/valyala/quicktemplate v1.4.1
-	go.opencensus.io v0.22.2 // indirect
-	golang.org/x/exp v0.0.0-20191129062945-2f5052295587 // indirect
-	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect
-	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449
-	golang.org/x/tools v0.0.0-20191212051200-825cb0626375 // indirect
-	google.golang.org/api v0.14.0
-	google.golang.org/appengine v1.6.5 // indirect
-	google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f // indirect
-	google.golang.org/grpc v1.25.1 // indirect
+	golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a // indirect
+	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20200122134326-e047566fdf82
+	golang.org/x/tools v0.0.0-20200122042241-dc16b66866f1 // indirect
+	google.golang.org/api v0.15.0
+	google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24 // indirect
 )
 
 go 1.12

go.sum

@@ -5,45 +5,53 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.49.0 h1:CH+lkubJzcPYB1Ggupcq0+k8Ni2ILdG2lYjDIgavDBQ=
-cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY=
-cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM=
+cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0 h1:9/vpR43S4aJaROxqQHQ3nH9lfyKKV0dC3vOmnw8ebQQ=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.4.0 h1:KDdqY5VTXBTqpSbctVTt0mVvfanP6JZzNzLE0qNY100=
-cloud.google.com/go/storage v1.4.0/go.mod h1:ZusYJWlOshgSBGbt6K3GnB3MT3H1xs2id9+TCl4fDBA=
+cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/VictoriaMetrics/fastcache v1.5.4 h1:0BaXbRH01RycJk79OOBwMCXlNryko9z4yEf6RqbP+Xo=
-github.com/VictoriaMetrics/fastcache v1.5.4/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
+github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
+github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
 github.com/VictoriaMetrics/metrics v1.9.3 h1:+1kZnOIb8RY825Nb9q9yMrPcOYuPE2GrZWxUh59XnHI=
 github.com/VictoriaMetrics/metrics v1.9.3/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
-github.com/aws/aws-sdk-go v1.26.0 h1:uddjCeunG1nroOMjcMO57g/vfwY7jJ1zYSEGmdeCeEs=
-github.com/aws/aws-sdk-go v1.26.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.28.7 h1:8RUfzsEmyXR8a9G7o2snfUKwrSuqks/k4C7TIfXDDrY=
+github.com/aws/aws-sdk-go v1.28.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -56,18 +64,21 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -76,8 +87,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.9.4 h1:xhvAeUPQ2drNUhKtrGdTGNvV9nNafHMUkRyLkzxJoB4=
-github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
+github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -90,19 +101,18 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
-github.com/valyala/fastjson v1.4.1 h1:hrltpHpIpkaxll8QltMU8c3QZ5+qIiCL8yKqPFJI/yE=
-github.com/valyala/fastjson v1.4.1/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o=
+github.com/valyala/fastjson v1.4.5 h1:uSuLfXk2LzRtzwd3Fy5zGRBe0Vs7zhs11vjdko32xb4=
+github.com/valyala/fastjson v1.4.5/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o=
 github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
-github.com/valyala/gozstd v1.6.3 h1:kr3oF/F1RvxYr8wgPjrH04gvHuMEL99pPC9e+5pPQdU=
-github.com/valyala/gozstd v1.6.3/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
+github.com/valyala/gozstd v1.6.4 h1:nFLddjEf90SFl5cVWyElSHozQDsbvLljPK703/skBS0=
+github.com/valyala/gozstd v1.6.4/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg=
 github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto=
 github.com/valyala/quicktemplate v1.4.1 h1:tEtkSN6mTCJlYVT7As5x4wjtkk2hj2thsb0M+AcAVeM=
@@ -115,14 +125,17 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587 h1:5Uz0rkjCFu9BC9gCRN7EkwVvhNyQgGWb8KNJrPwBoHY=
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a h1:7Wlg8L54In96HTWOaI4sreLJ6qfyGuvSau5el3fK41Y=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -131,7 +144,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
@@ -139,6 +151,7 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -150,20 +163,23 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -172,8 +188,11 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -195,17 +214,23 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191212051200-825cb0626375 h1:hrBCt+pkb1irxFjg/GeadSn24gLxi5/Z861I53OkQbc=
-golang.org/x/tools v0.0.0-20191212051200-825cb0626375/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122042241-dc16b66866f1 h1:468gVSKEm8NObiNTQ3it08aAGsPfuvz+WXUHmnq8Wws=
+golang.org/x/tools v0.0.0-20200122042241-dc16b66866f1/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -220,15 +245,18 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f h1:naitw5DILWPQvG0oG04mR9jF8fmKpRdW3E3zzKA4D0Y=
-google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24 h1:wDju+RU97qa0FZT0QnZDg9Uc2dH0Ql513kFvHocz+WM=
+google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

lib/backup/actions/backup.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -41,17 +42,33 @@ type Backup struct {
 
 // Run runs b with the provided settings.
 func (b *Backup) Run() error {
-	startTime := time.Now()
-
 	concurrency := b.Concurrency
 	src := b.Src
 	dst := b.Dst
 	origin := b.Origin
 
+	if origin != nil && origin.String() == dst.String() {
+		origin = nil
+	}
 	if origin == nil {
 		origin = &fsnil.FS{}
 	}
 
+	if err := dst.DeleteFile(fscommon.BackupCompleteFilename); err != nil {
+		return fmt.Errorf("cannot delete `backup complete` file at %s: %s", dst, err)
+	}
+	if err := runBackup(src, dst, origin, concurrency); err != nil {
+		return err
+	}
+	if err := dst.CreateFile(fscommon.BackupCompleteFilename, []byte("ok")); err != nil {
+		return fmt.Errorf("cannot create `backup complete` file at %s: %s", dst, err)
+	}
+	return nil
+}
+
+func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, concurrency int) error {
+	startTime := time.Now()
+
 	logger.Infof("starting backup from %s to %s using origin %s", src, dst, origin)
 
 	logger.Infof("obtaining list of parts at %s", src)
@@ -149,8 +166,8 @@ func (b *Backup) Run() error {
 		}
 	}
 
-	logger.Infof("backed up %d bytes in %s; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
-		backupSize, time.Since(startTime), deleteSize, copySize, uploadSize)
+	logger.Infof("backed up %d bytes in %.3f seconds; deleted %d bytes; server-side copied %d bytes; uploaded %d bytes",
+		backupSize, time.Since(startTime).Seconds(), deleteSize, copySize, uploadSize)
 
 	return nil
 }

lib/backup/actions/restore.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -29,6 +30,11 @@ type Restore struct {
 	// If dst points to existing directory, then incremental restore is performed,
 	// i.e. only new data is downloaded from src.
 	Dst *fslocal.FS
+
+	// SkipBackupCompleteCheck may be set in order to skip for `backup complete` file in Src.
+	//
+	// This may be needed for restoring from old backups with missing `backup complete` file.
+	SkipBackupCompleteCheck bool
 }
 
 // Run runs r with the provided settings.
@@ -48,6 +54,18 @@ func (r *Restore) Run() error {
 	concurrency := r.Concurrency
 	src := r.Src
 	dst := r.Dst
+
+	if !r.SkipBackupCompleteCheck {
+		ok, err := src.HasFile(fscommon.BackupCompleteFilename)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			return fmt.Errorf("cannot find %s file in %s; this means either incomplete backup or old backup; "+
+				"pass `-skipBackupCompleteCheck` command-line flag if you still need restoring from this backup", fscommon.BackupCompleteFilename, src)
+		}
+	}
+
 	logger.Infof("starting restore from %s to %s", src, dst)
 
 	logger.Infof("obtaining list of parts at %s", src)
@@ -164,7 +182,8 @@ func (r *Restore) Run() error {
 		}
 	}
 
-	logger.Infof("restored %d bytes from backup in %s; deleted %d bytes; downloaded %d bytes", backupSize, time.Since(startTime), deleteSize, downloadSize)
+	logger.Infof("restored %d bytes from backup in %.3f seconds; deleted %d bytes; downloaded %d bytes",
+		backupSize, time.Since(startTime).Seconds(), deleteSize, downloadSize)
 
 	return nil
 }

lib/backup/common/fs.go

@@ -38,4 +38,13 @@ type RemoteFS interface {
 
 	// UploadPart must upload part p from r to RemoteFS.
 	UploadPart(p Part, r io.Reader) error
+
+	// DeleteFile deletes filePath at RemoteFS
+	DeleteFile(filePath string) error
+
+	// CreateFile creates filePath at RemoteFS and puts data into it.
+	CreateFile(filePath string, data []byte) error
+
+	// HasFile returns true if filePath exists at RemoteFS.
+	HasFile(filePath string) (bool, error)
 }

lib/backup/fscommon/fscommon.go

@@ -260,3 +260,11 @@ func removeEmptyDirsInternal(d *os.File) (bool, error) {
 	}
 	return true, nil
 }
+
+// IgnorePath returns true if the given path must be ignored.
+func IgnorePath(path string) bool {
+	return strings.HasSuffix(path, ".ignore")
+}
+
+// BackupCompleteFilename is a filename, which is created in the destination fs when backup is complete.
+const BackupCompleteFilename = "backup_complete.ignore"

lib/backup/fsremote/fsremote.go

@@ -3,6 +3,7 @@ package fsremote
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -48,6 +49,9 @@ func (fs *FS) ListParts() ([]common.Part, error) {
 		if !strings.HasPrefix(file, dir) {
 			logger.Panicf("BUG: unexpected prefix for file %q; want %q", file, dir)
 		}
+		if fscommon.IgnorePath(file) {
+			continue
+		}
 		var p common.Part
 		if !p.ParseFromRemotePath(file[len(dir):]) {
 			logger.Infof("skipping unknown file %s", file)
@@ -188,3 +192,42 @@ func (fs *FS) mkdirAll(filePath string) error {
 func (fs *FS) path(p common.Part) string {
 	return p.RemotePath(fs.Dir)
 }
+
+// DeleteFile deletes filePath at fs.
+//
+// The function does nothing if the filePath doesn't exist.
+func (fs *FS) DeleteFile(filePath string) error {
+	path := filepath.Join(fs.Dir, filePath)
+	err := os.Remove(path)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("cannot remove %q: %s", path, err)
+	}
+	return nil
+}
+
+// CreateFile creates filePath at fs and puts data into it.
+//
+// The file is overwritten if it exists.
+func (fs *FS) CreateFile(filePath string, data []byte) error {
+	path := filepath.Join(fs.Dir, filePath)
+	if err := ioutil.WriteFile(path, data, 0600); err != nil {
+		return fmt.Errorf("cannot write %d bytes to %q: %s", len(data), path, err)
+	}
+	return nil
+}
+
+// HasFile returns true if filePath exists at fs.
+func (fs *FS) HasFile(filePath string) (bool, error) {
+	path := filepath.Join(fs.Dir, filePath)
+	fi, err := os.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("cannot stat %q: %s", path, err)
+	}
+	if fi.IsDir() {
+		return false, fmt.Errorf("%q is directory, while file is needed", path)
+	}
+	return true, nil
+}

lib/backup/gcsremote/gcs.go

@@ -8,6 +8,7 @@ import (
 
 	"cloud.google.com/go/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
@@ -97,6 +98,9 @@ func (fs *FS) ListParts() ([]common.Part, error) {
 		if !strings.HasPrefix(file, dir) {
 			return nil, fmt.Errorf("unexpected prefix for gcs key %q; want %q", file, dir)
 		}
+		if fscommon.IgnorePath(file) {
+			continue
+		}
 		var p common.Part
 		if !p.ParseFromRemotePath(file[len(dir):]) {
 			logger.Infof("skipping unknown object %q", file)
@@ -187,3 +191,56 @@ func (fs *FS) object(p common.Part) *storage.ObjectHandle {
 	path := p.RemotePath(fs.Dir)
 	return fs.bkt.Object(path)
 }
+
+// DeleteFile deletes filePath at fs if it exists.
+//
+// The function does nothing if the filePath doesn't exists.
+func (fs *FS) DeleteFile(filePath string) error {
+	path := fs.Dir + filePath
+	o := fs.bkt.Object(path)
+	ctx := context.Background()
+	if err := o.Delete(ctx); err != nil {
+		if err != storage.ErrObjectNotExist {
+			return fmt.Errorf("cannot delete %q at %s (remote path %q): %s", filePath, fs, o.ObjectName(), err)
+		}
+	}
+	return nil
+}
+
+// CreateFile creates filePath at fs and puts data into it.
+//
+// The file is overwritten if it exists.
+func (fs *FS) CreateFile(filePath string, data []byte) error {
+	path := fs.Dir + filePath
+	o := fs.bkt.Object(path)
+	ctx := context.Background()
+	w := o.NewWriter(ctx)
+	n, err := w.Write(data)
+	if err != nil {
+		_ = w.Close()
+		return fmt.Errorf("cannot upload %d bytes to %q at %s (remote path %q): %s", len(data), filePath, fs, o.ObjectName(), err)
+	}
+	if n != len(data) {
+		_ = w.Close()
+		return fmt.Errorf("wrong data size uploaded to %q at %s (remote path %q); got %d bytes; want %d bytes", filePath, fs, o.ObjectName(), n, len(data))
+	}
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("cannot close %q at %s (remote path %q): %s", filePath, fs, o.ObjectName(), err)
+	}
+	return nil
+}
+
+// HasFile returns ture if filePath exists at fs.
+func (fs *FS) HasFile(filePath string) (bool, error) {
+	path := fs.Dir + filePath
+	o := fs.bkt.Object(path)
+	ctx := context.Background()
+	_, err := o.Attrs(ctx)
+	if err != nil {
+		if err == storage.ErrObjectNotExist {
+			return false, nil
+		}
+		return false, fmt.Errorf("unexpected error when obtaining attributes for %q at %s (remote path %q): %s", filePath, fs, o.ObjectName(), err)
+	}
+	return true, nil
+}

lib/backup/s3remote/s3.go

@@ -1,14 +1,17 @@
 package s3remote
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -113,6 +116,9 @@ func (fs *FS) ListParts() ([]common.Part, error) {
 				errOuter = fmt.Errorf("unexpected prefix for s3 key %q; want %q", file, dir)
 				return false
 			}
+			if fscommon.IgnorePath(file) {
+				continue
+			}
 			var p common.Part
 			if !p.ParseFromRemotePath(file[len(dir):]) {
 				logger.Infof("skipping unknown object %q", file)
@@ -220,6 +226,78 @@ func (fs *FS) UploadPart(p common.Part, r io.Reader) error {
 	return nil
 }
 
+// DeleteFile deletes filePath from fs if it exists.
+//
+// The function does nothing if the file doesn't exist.
+func (fs *FS) DeleteFile(filePath string) error {
+	// It looks like s3 may return `AccessDenied: Access Denied` instead of `s3.ErrCodeNoSuchKey`
+	// on an attempt to delete non-existing file.
+	// so just check whether the filePath exists before deleting it.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/284 for details.
+	ok, err := fs.HasFile(filePath)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		// Missing file - nothing to delete.
+		return nil
+	}
+
+	path := fs.Dir + filePath
+	input := &s3.DeleteObjectInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+	}
+	if _, err := fs.s3.DeleteObject(input); err != nil {
+		return fmt.Errorf("cannot delete %q at %s (remote path %q): %s", filePath, fs, path, err)
+	}
+	return nil
+}
+
+// CreateFile creates filePath at fs and puts data into it.
+//
+// The file is overwritten if it already exists.
+func (fs *FS) CreateFile(filePath string, data []byte) error {
+	path := fs.Dir + filePath
+	sr := &statReader{
+		r: bytes.NewReader(data),
+	}
+	input := &s3manager.UploadInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+		Body:   sr,
+	}
+	_, err := fs.uploader.Upload(input)
+	if err != nil {
+		return fmt.Errorf("cannot upoad data to %q at %s (remote path %q): %s", filePath, fs, path, err)
+	}
+	l := int64(len(data))
+	if sr.size != l {
+		return fmt.Errorf("wrong data size uploaded to %q at %s; got %d bytes; want %d bytes", filePath, fs, sr.size, l)
+	}
+	return nil
+}
+
+// HasFile returns true if filePath exists at fs.
+func (fs *FS) HasFile(filePath string) (bool, error) {
+	path := fs.Dir + filePath
+	input := &s3.GetObjectInput{
+		Bucket: aws.String(fs.Bucket),
+		Key:    aws.String(path),
+	}
+	o, err := fs.s3.GetObject(input)
+	if err != nil {
+		if ae, ok := err.(awserr.Error); ok && ae.Code() == s3.ErrCodeNoSuchKey {
+			return false, nil
+		}
+		return false, fmt.Errorf("cannot open %q at %s (remote path %q): %s", filePath, fs, path, err)
+	}
+	if err := o.Body.Close(); err != nil {
+		return false, fmt.Errorf("cannot close %q at %s (remote path %q): %s", filePath, fs, path, err)
+	}
+	return true, nil
+}
+
 func (fs *FS) path(p common.Part) string {
 	return p.RemotePath(fs.Dir)
 }

lib/encoding/encoding.go

@@ -166,24 +166,24 @@ func unmarshalInt64Array(dst []int64, src []byte, mt MarshalType, firstValue int
 		bb := bbPool.Get()
 		bb.B, err = DecompressZSTD(bb.B[:0], src)
 		if err != nil {
-			return nil, fmt.Errorf("cannot decompress zstd data of size %d: %s", len(src), err)
+			return nil, fmt.Errorf("cannot decompress zstd data of size %d: %s; src_zstd=%X", len(src), err, src)
 		}
 		dst, err = unmarshalInt64NearestDelta(dst, bb.B, firstValue, itemsCount)
 		bbPool.Put(bb)
 		if err != nil {
-			return nil, fmt.Errorf("cannot unmarshal nearest delta data after zstd decompression: %s", err)
+			return nil, fmt.Errorf("cannot unmarshal nearest delta data after zstd decompression: %s; src_zstd=%X", err, src)
 		}
 		return dst, nil
 	case MarshalTypeZSTDNearestDelta2:
 		bb := bbPool.Get()
 		bb.B, err = DecompressZSTD(bb.B[:0], src)
 		if err != nil {
-			return nil, fmt.Errorf("cannot decompress zstd data of size %d: %s", len(src), err)
+			return nil, fmt.Errorf("cannot decompress zstd data of size %d: %s; src_zstd=%X", len(src), err, src)
 		}
 		dst, err = unmarshalInt64NearestDelta2(dst, bb.B, firstValue, itemsCount)
 		bbPool.Put(bb)
 		if err != nil {
-			return nil, fmt.Errorf("cannot unmarshal nearest delta2 data after zstd decompression: %s", err)
+			return nil, fmt.Errorf("cannot unmarshal nearest delta2 data after zstd decompression: %s; src_zstd=%X", err, src)
 		}
 		return dst, nil
 	case MarshalTypeNearestDelta:

lib/encoding/nearest_delta.go

@@ -60,10 +60,10 @@ func unmarshalInt64NearestDelta(dst []int64, src []byte, firstValue int64, items
 
 	tail, err := UnmarshalVarInt64s(is.A, src)
 	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal nearest delta: %s", err)
+		return nil, fmt.Errorf("cannot unmarshal nearest delta from %d bytes; src=%X: %s", len(src), src, err)
 	}
 	if len(tail) > 0 {
-		return nil, fmt.Errorf("unexpected tail left after unmarshaling %d items; tail size=%d, value=%X", itemsCount, len(tail), tail)
+		return nil, fmt.Errorf("unexpected tail left after unmarshaling %d items from %d bytes; tail size=%d; src=%X; tail=%X", itemsCount, len(src), len(tail), src, tail)
 	}
 
 	v := firstValue

lib/encoding/nearest_delta2.go

@@ -63,10 +63,10 @@ func unmarshalInt64NearestDelta2(dst []int64, src []byte, firstValue int64, item
 
 	tail, err := UnmarshalVarInt64s(is.A, src)
 	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal nearest delta: %s", err)
+		return nil, fmt.Errorf("cannot unmarshal nearest delta from %d bytes; src=%X: %s", len(src), src, err)
 	}
 	if len(tail) > 0 {
-		return nil, fmt.Errorf("unexpected tail left after unmarshaling %d items; tail size=%d, value=%X", itemsCount, len(tail), tail)
+		return nil, fmt.Errorf("unexpected tail left after unmarshaling %d items from %d bytes; tail size=%d; src=%X; tail=%X", itemsCount, len(src), len(tail), src, tail)
 	}
 
 	v := firstValue

lib/fs/fadvise_unix.go

@@ -1,4 +1,4 @@
-// +buid linux freebsd
+// +build linux freebsd
 
 package fs
 

lib/httpserver/httpserver.go

@@ -2,7 +2,6 @@ package httpserver
 
 import (
 	"bufio"
-	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"flag"
@@ -20,6 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/klauspost/compress/gzip"
 )
 
 var (

lib/mergeset/part.go

@@ -5,7 +5,7 @@ import (
 	"path/filepath"
 	"sync"
 	"sync/atomic"
-	"unsafe"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -44,7 +44,7 @@ var (
 	maxCachedInmemoryBlocksPerPartOnce sync.Once
 )
 
-type partInternals struct {
+type part struct {
 	ph partHeader
 
 	path string
@@ -56,16 +56,9 @@ type partInternals struct {
 	indexFile fs.ReadAtCloser
 	itemsFile fs.ReadAtCloser
 	lensFile  fs.ReadAtCloser
-}
 
-type part struct {
-	partInternals
-
-	// Align atomic counters inside caches by 8 bytes on 32-bit architectures.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212 .
-	_         [(8 - (unsafe.Sizeof(partInternals{}) % 8)) % 8]byte
-	idxbCache indexBlockCache
-	ibCache   inmemoryBlockCache
+	idxbCache *indexBlockCache
+	ibCache   *inmemoryBlockCache
 }
 
 func openFilePart(path string) (*part, error) {
@@ -132,8 +125,8 @@ func newPart(ph *partHeader, path string, size uint64, metaindexReader filestrea
 	p.lensFile = lensFile
 
 	p.ph.CopyFrom(ph)
-	p.idxbCache.Init()
-	p.ibCache.Init()
+	p.idxbCache = newIndexBlockCache()
+	p.ibCache = newInmemoryBlockCache()
 
 	if len(errors) > 0 {
 		// Return only the first error, since it has no sense in returning all errors.
@@ -149,8 +142,8 @@ func (p *part) MustClose() {
 	p.itemsFile.MustClose()
 	p.lensFile.MustClose()
 
-	p.idxbCache.Reset()
-	p.ibCache.Reset()
+	p.idxbCache.MustClose()
+	p.ibCache.MustClose()
 }
 
 type indexBlock struct {
@@ -179,27 +172,72 @@ type indexBlockCache struct {
 	requests uint64
 	misses   uint64
 
-	m         map[uint64]*indexBlock
-	missesMap map[uint64]uint64
-	mu        sync.RWMutex
+	m  map[uint64]indexBlockCacheEntry
+	mu sync.RWMutex
+
+	cleanerStopCh chan struct{}
+	cleanerWG     sync.WaitGroup
 }
 
-func (idxbc *indexBlockCache) Init() {
-	idxbc.m = make(map[uint64]*indexBlock)
-	idxbc.missesMap = make(map[uint64]uint64)
-	idxbc.requests = 0
-	idxbc.misses = 0
+type indexBlockCacheEntry struct {
+	// Atomically updated counters must go first in the struct, so they are properly
+	// aligned to 8 bytes on 32-bit architectures.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
+	lastAccessTime uint64
+
+	idxb *indexBlock
 }
 
-func (idxbc *indexBlockCache) Reset() {
+func newIndexBlockCache() *indexBlockCache {
+	var idxbc indexBlockCache
+	idxbc.m = make(map[uint64]indexBlockCacheEntry)
+	idxbc.cleanerStopCh = make(chan struct{})
+	idxbc.cleanerWG.Add(1)
+	go func() {
+		defer idxbc.cleanerWG.Done()
+		idxbc.cleaner()
+	}()
+	return &idxbc
+}
+
+func (idxbc *indexBlockCache) MustClose() {
+	close(idxbc.cleanerStopCh)
+	idxbc.cleanerWG.Wait()
+
 	atomic.AddUint64(&indexBlockCacheRequests, idxbc.requests)
 	atomic.AddUint64(&indexBlockCacheMisses, idxbc.misses)
 	// It is safe returning idxbc.m to pool, since the Reset must be called
 	// when the idxbc entries are no longer accessed by concurrent goroutines.
-	for _, idxb := range idxbc.m {
-		putIndexBlock(idxb)
+	for _, idxbe := range idxbc.m {
+		putIndexBlock(idxbe.idxb)
 	}
-	idxbc.Init()
+	idxbc.m = nil
+}
+
+// cleaner periodically cleans least recently used items.
+func (idxbc *indexBlockCache) cleaner() {
+	t := time.NewTimer(5 * time.Second)
+	for {
+		select {
+		case <-t.C:
+			idxbc.cleanByTimeout()
+		case <-idxbc.cleanerStopCh:
+			t.Stop()
+			return
+		}
+	}
+}
+
+func (idxbc *indexBlockCache) cleanByTimeout() {
+	currentTime := atomic.LoadUint64(&currentTimestamp)
+	idxbc.mu.Lock()
+	for k, idxbe := range idxbc.m {
+		// Delete items accessed more than 10 minutes ago.
+		if currentTime-atomic.LoadUint64(&idxbe.lastAccessTime) > 10*60 {
+			delete(idxbc.m, k)
+		}
+	}
+	idxbc.mu.Unlock()
 }
 
 var (
@@ -210,16 +248,17 @@ var (
 func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
 	atomic.AddUint64(&idxbc.requests, 1)
 	idxbc.mu.RLock()
-	idxb := idxbc.m[k]
+	idxbe, ok := idxbc.m[k]
 	idxbc.mu.RUnlock()
 
-	if idxb != nil {
-		return idxb
+	if ok {
+		currentTime := atomic.LoadUint64(&currentTimestamp)
+		if atomic.LoadUint64(&idxbe.lastAccessTime) != currentTime {
+			atomic.StoreUint64(&idxbe.lastAccessTime, currentTime)
+		}
+		return idxbe.idxb
 	}
 	atomic.AddUint64(&idxbc.misses, 1)
-	idxbc.mu.Lock()
-	idxbc.missesMap[k]++
-	idxbc.mu.Unlock()
 	return nil
 }
 
@@ -229,13 +268,6 @@ func (idxbc *indexBlockCache) Get(k uint64) *indexBlock {
 func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) bool {
 	idxbc.mu.Lock()
 
-	if idxbc.missesMap[k] < 2 {
-		// Do not pollute cache with infrequently accessed items, since they may
-		// evict frequently accessed items.
-		idxbc.mu.Unlock()
-		return false
-	}
-
 	// Remove superflouos entries.
 	if overflow := len(idxbc.m) - getMaxCachedIndexBlocksPerPart(); overflow > 0 {
 		// Remove 10% of items from the cache.
@@ -250,21 +282,13 @@ func (idxbc *indexBlockCache) Put(k uint64, idxb *indexBlock) bool {
 			}
 		}
 	}
-	if overflow := len(idxbc.missesMap) - 8*getMaxCachedIndexBlocksPerPart(); overflow > 0 {
-		// Remove 10% of items from the cache.
-		overflow = int(float64(len(idxbc.missesMap)) * 0.1)
-		for k := range idxbc.missesMap {
-			delete(idxbc.missesMap, k)
-			overflow--
-			if overflow == 0 {
-				break
-			}
-		}
-	}
 
-	// Store the frequently accessed idxb in the cache.
-	delete(idxbc.missesMap, k)
-	idxbc.m[k] = idxb
+	// Store idxb in the cache.
+	idxbe := indexBlockCacheEntry{
+		lastAccessTime: atomic.LoadUint64(&currentTimestamp),
+		idxb:           idxb,
+	}
+	idxbc.m[k] = idxbe
 	idxbc.mu.Unlock()
 	return true
 }
@@ -291,9 +315,11 @@ type inmemoryBlockCache struct {
 	requests uint64
 	misses   uint64
 
-	m         map[inmemoryBlockCacheKey]*inmemoryBlock
-	missesMap map[inmemoryBlockCacheKey]uint64
-	mu        sync.RWMutex
+	m  map[inmemoryBlockCacheKey]*inmemoryBlockCacheEntry
+	mu sync.RWMutex
+
+	cleanerStopCh chan struct{}
+	cleanerWG     sync.WaitGroup
 }
 
 type inmemoryBlockCacheKey struct {
@@ -309,22 +335,66 @@ func (ibck *inmemoryBlockCacheKey) Init(bh *blockHeader) {
 	ibck.itemsBlockOffset = bh.itemsBlockOffset
 }
 
-func (ibc *inmemoryBlockCache) Init() {
-	ibc.m = make(map[inmemoryBlockCacheKey]*inmemoryBlock)
-	ibc.missesMap = make(map[inmemoryBlockCacheKey]uint64)
-	ibc.requests = 0
-	ibc.misses = 0
+type inmemoryBlockCacheEntry struct {
+	// Atomically updated counters must go first in the struct, so they are properly
+	// aligned to 8 bytes on 32-bit architectures.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
+	lastAccessTime uint64
+
+	ib *inmemoryBlock
 }
 
-func (ibc *inmemoryBlockCache) Reset() {
+func newInmemoryBlockCache() *inmemoryBlockCache {
+	var ibc inmemoryBlockCache
+	ibc.m = make(map[inmemoryBlockCacheKey]*inmemoryBlockCacheEntry)
+
+	ibc.cleanerStopCh = make(chan struct{})
+	ibc.cleanerWG.Add(1)
+	go func() {
+		defer ibc.cleanerWG.Done()
+		ibc.cleaner()
+	}()
+	return &ibc
+}
+
+func (ibc *inmemoryBlockCache) MustClose() {
+	close(ibc.cleanerStopCh)
+	ibc.cleanerWG.Wait()
+
 	atomic.AddUint64(&inmemoryBlockCacheRequests, ibc.requests)
 	atomic.AddUint64(&inmemoryBlockCacheMisses, ibc.misses)
 	// It is safe returning ibc.m entries to pool, since the Reset function may be called
 	// only if no other goroutines access ibc entries.
-	for _, ib := range ibc.m {
-		putInmemoryBlock(ib)
+	for _, ibe := range ibc.m {
+		putInmemoryBlock(ibe.ib)
 	}
-	ibc.Init()
+	ibc.m = nil
+}
+
+// cleaner periodically cleans least recently used items.
+func (ibc *inmemoryBlockCache) cleaner() {
+	t := time.NewTimer(5 * time.Second)
+	for {
+		select {
+		case <-t.C:
+			ibc.cleanByTimeout()
+		case <-ibc.cleanerStopCh:
+			t.Stop()
+			return
+		}
+	}
+}
+
+func (ibc *inmemoryBlockCache) cleanByTimeout() {
+	currentTime := atomic.LoadUint64(&currentTimestamp)
+	ibc.mu.Lock()
+	for k, ibe := range ibc.m {
+		// Delete items accessed more than 10 minutes ago.
+		if currentTime-atomic.LoadUint64(&ibe.lastAccessTime) > 10*60 {
+			delete(ibc.m, k)
+		}
+	}
+	ibc.mu.Unlock()
 }
 
 var (
@@ -336,16 +406,17 @@ func (ibc *inmemoryBlockCache) Get(k inmemoryBlockCacheKey) *inmemoryBlock {
 	atomic.AddUint64(&ibc.requests, 1)
 
 	ibc.mu.RLock()
-	ib := ibc.m[k]
+	ibe := ibc.m[k]
 	ibc.mu.RUnlock()
 
-	if ib != nil {
-		return ib
+	if ibe != nil {
+		currentTime := atomic.LoadUint64(&currentTimestamp)
+		if atomic.LoadUint64(&ibe.lastAccessTime) != currentTime {
+			atomic.StoreUint64(&ibe.lastAccessTime, currentTime)
+		}
+		return ibe.ib
 	}
 	atomic.AddUint64(&ibc.misses, 1)
-	ibc.mu.Lock()
-	ibc.missesMap[k]++
-	ibc.mu.Unlock()
 	return nil
 }
 
@@ -355,14 +426,7 @@ func (ibc *inmemoryBlockCache) Get(k inmemoryBlockCacheKey) *inmemoryBlock {
 func (ibc *inmemoryBlockCache) Put(k inmemoryBlockCacheKey, ib *inmemoryBlock) bool {
 	ibc.mu.Lock()
 
-	if ibc.missesMap[k] < 2 {
-		// Do not cache entry with low number of accesses, since it may evict
-		// frequently accessed entries from the cache.
-		ibc.mu.Unlock()
-		return false
-	}
-
-	// Clean superflouos entries in ibc.m and ibc.missesMap.
+	// Clean superflouos entries in cache.
 	if overflow := len(ibc.m) - getMaxCachedInmemoryBlocksPerPart(); overflow > 0 {
 		// Remove 10% of items from the cache.
 		overflow = int(float64(len(ibc.m)) * 0.1)
@@ -376,21 +440,13 @@ func (ibc *inmemoryBlockCache) Put(k inmemoryBlockCacheKey, ib *inmemoryBlock) b
 			}
 		}
 	}
-	if overflow := len(ibc.missesMap) - 8*getMaxCachedInmemoryBlocksPerPart(); overflow > 0 {
-		// Remove 10% of items from the cache.
-		overflow = int(float64(len(ibc.missesMap)) * 0.1)
-		for k := range ibc.missesMap {
-			delete(ibc.missesMap, k)
-			overflow--
-			if overflow == 0 {
-				break
-			}
-		}
-	}
 
-	// The entry is frequently accessed, so store it in the cache.
-	delete(ibc.missesMap, k)
-	ibc.m[k] = ib
+	// Store ib in the cache.
+	ibe := &inmemoryBlockCacheEntry{
+		lastAccessTime: atomic.LoadUint64(&currentTimestamp),
+		ib:             ib,
+	}
+	ibc.m[k] = ibe
 	ibc.mu.Unlock()
 	return true
 }
@@ -409,3 +465,15 @@ func (ibc *inmemoryBlockCache) Requests() uint64 {
 func (ibc *inmemoryBlockCache) Misses() uint64 {
 	return atomic.LoadUint64(&ibc.misses)
 }
+
+func init() {
+	go func() {
+		t := time.NewTimer(time.Second)
+		for tm := range t.C {
+			t := uint64(tm.Unix())
+			atomic.StoreUint64(&currentTimestamp, t)
+		}
+	}()
+}
+
+var currentTimestamp uint64

lib/mergeset/part_search.go

@@ -82,8 +82,8 @@ func (ps *partSearch) Init(p *part, shouldCacheBlock func(item []byte) bool) {
 	ps.reset()
 
 	ps.p = p
-	ps.idxbCache = &p.idxbCache
-	ps.ibCache = &p.ibCache
+	ps.idxbCache = p.idxbCache
+	ps.ibCache = p.ibCache
 }
 
 // Seek seeks for the first item greater or equal to k in ps.

lib/mergeset/table.go

@@ -187,8 +187,8 @@ func OpenTable(path string, flushCallback func(), prepareBlock PrepareBlockCallb
 
 	var m TableMetrics
 	tb.UpdateMetrics(&m)
-	logger.Infof("table %q has been opened in %s; partsCount: %d; blocksCount: %d, itemsCount: %d; sizeBytes: %d",
-		path, time.Since(startTime), m.PartsCount, m.BlocksCount, m.ItemsCount, m.SizeBytes)
+	logger.Infof("table %q has been opened in %.3f seconds; partsCount: %d; blocksCount: %d, itemsCount: %d; sizeBytes: %d",
+		path, time.Since(startTime).Seconds(), m.PartsCount, m.BlocksCount, m.ItemsCount, m.SizeBytes)
 
 	tb.convertersWG.Add(1)
 	go func() {
@@ -206,17 +206,17 @@ func (tb *Table) MustClose() {
 	logger.Infof("waiting for raw items flusher to stop on %q...", tb.path)
 	startTime := time.Now()
 	tb.rawItemsFlusherWG.Wait()
-	logger.Infof("raw items flusher stopped in %s on %q", time.Since(startTime), tb.path)
+	logger.Infof("raw items flusher stopped in %.3f seconds on %q", time.Since(startTime).Seconds(), tb.path)
 
 	logger.Infof("waiting for converters to stop on %q...", tb.path)
 	startTime = time.Now()
 	tb.convertersWG.Wait()
-	logger.Infof("converters stopped in %s on %q", time.Since(startTime), tb.path)
+	logger.Infof("converters stopped in %.3f seconds on %q", time.Since(startTime).Seconds(), tb.path)
 
 	logger.Infof("waiting for part mergers to stop on %q...", tb.path)
 	startTime = time.Now()
 	tb.partMergersWG.Wait()
-	logger.Infof("part mergers stopped in %s on %q", time.Since(startTime), tb.path)
+	logger.Infof("part mergers stopped in %.3f seconds on %q", time.Since(startTime).Seconds(), tb.path)
 
 	logger.Infof("flushing inmemory parts to files on %q...", tb.path)
 	startTime = time.Now()
@@ -242,7 +242,7 @@ func (tb *Table) MustClose() {
 	if err := tb.mergePartsOptimal(pws, nil); err != nil {
 		logger.Panicf("FATAL: cannot flush inmemory parts to files in %q: %s", tb.path, err)
 	}
-	logger.Infof("%d inmemory parts have been flushed to files in %s on %q", len(pws), time.Since(startTime), tb.path)
+	logger.Infof("%d inmemory parts have been flushed to files in %.3f seconds on %q", len(pws), time.Since(startTime).Seconds(), tb.path)
 
 	// Remove references to parts from the tb, so they may be eventually closed
 	// after all the searches are done.
@@ -447,7 +447,7 @@ func (tb *Table) convertToV1280() {
 			logger.Errorf("failed round 1 of background conversion of %q to v1.28.0 format: %s", tb.path, err)
 			return
 		}
-		logger.Infof("finished round 1 of background conversion of %q to v1.28.0 format in %s", tb.path, time.Since(startTime))
+		logger.Infof("finished round 1 of background conversion of %q to v1.28.0 format in %.3f seconds", tb.path, time.Since(startTime).Seconds())
 
 		// The second round is needed in order to merge small blocks
 		// with tag->metricIDs rows left after the first round.
@@ -460,7 +460,7 @@ func (tb *Table) convertToV1280() {
 				return
 			}
 		}
-		logger.Infof("finished round 2 of background conversion of %q to v1.28.0 format in %s", tb.path, time.Since(startTime))
+		logger.Infof("finished round 2 of background conversion of %q to v1.28.0 format in %.3f seconds", tb.path, time.Since(startTime).Seconds())
 	}
 
 	if err := fs.WriteFileAtomically(flagFilePath, []byte("ok")); err != nil {
@@ -853,7 +853,8 @@ func (tb *Table) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isOuterP
 
 	d := time.Since(startTime)
 	if d > 10*time.Second {
-		logger.Infof("merged %d items in %s at %d items/sec to %q; sizeBytes: %d", outItemsCount, d, int(float64(outItemsCount)/d.Seconds()), dstPartPath, newPSize)
+		logger.Infof("merged %d items in %.3f seconds at %d items/sec to %q; sizeBytes: %d",
+			outItemsCount, d.Seconds(), int(float64(outItemsCount)/d.Seconds()), dstPartPath, newPSize)
 	}
 
 	return nil
@@ -1057,7 +1058,7 @@ func (tb *Table) CreateSnapshotAt(dstDir string) error {
 	parentDir := filepath.Dir(dstDir)
 	fs.MustSyncPath(parentDir)
 
-	logger.Infof("created Table snapshot of %q at %q in %s", srcDir, dstDir, time.Since(startTime))
+	logger.Infof("created Table snapshot of %q at %q in %.3f seconds", srcDir, dstDir, time.Since(startTime).Seconds())
 	return nil
 }
 

lib/metricsql/aggr.go

@@ -0,0 +1,51 @@
+package metricsql
+
+import (
+	"strings"
+)
+
+var aggrFuncs = map[string]bool{
+	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#aggregation-operators
+	"sum":          true,
+	"min":          true,
+	"max":          true,
+	"avg":          true,
+	"stddev":       true,
+	"stdvar":       true,
+	"count":        true,
+	"count_values": true,
+	"bottomk":      true,
+	"topk":         true,
+	"quantile":     true,
+
+	// MetricsQL extension funcs
+	"median":         true,
+	"limitk":         true,
+	"distinct":       true,
+	"sum2":           true,
+	"geomean":        true,
+	"histogram":      true,
+	"topk_min":       true,
+	"topk_max":       true,
+	"topk_avg":       true,
+	"topk_median":    true,
+	"bottomk_min":    true,
+	"bottomk_max":    true,
+	"bottomk_avg":    true,
+	"bottomk_median": true,
+}
+
+func isAggrFunc(s string) bool {
+	s = strings.ToLower(s)
+	return aggrFuncs[s]
+}
+
+func isAggrFuncModifier(s string) bool {
+	s = strings.ToLower(s)
+	switch s {
+	case "by", "without":
+		return true
+	default:
+		return false
+	}
+}

lib/metricsql/aggr_test.go

@@ -1,4 +1,4 @@
-package promql
+package metricsql
 
 import (
 	"testing"

lib/metricsql/binary_op.go

@@ -0,0 +1,205 @@
+package metricsql
+
+import (
+	"fmt"
+	"math"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql/binaryop"
+)
+
+var binaryOps = map[string]bool{
+	"+": true,
+	"-": true,
+	"*": true,
+	"/": true,
+	"%": true,
+	"^": true,
+
+	// cmp ops
+	"==": true,
+	"!=": true,
+	">":  true,
+	"<":  true,
+	">=": true,
+	"<=": true,
+
+	// logical set ops
+	"and":    true,
+	"or":     true,
+	"unless": true,
+
+	// New ops for MetricsQL
+	"if":      true,
+	"ifnot":   true,
+	"default": true,
+}
+
+var binaryOpPriorities = map[string]int{
+	"default": -1,
+
+	"if":    0,
+	"ifnot": 0,
+
+	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence
+	"or": 1,
+
+	"and":    2,
+	"unless": 2,
+
+	"==": 3,
+	"!=": 3,
+	"<":  3,
+	">":  3,
+	"<=": 3,
+	">=": 3,
+
+	"+": 4,
+	"-": 4,
+
+	"*": 5,
+	"/": 5,
+	"%": 5,
+
+	"^": 6,
+}
+
+func isBinaryOp(op string) bool {
+	op = strings.ToLower(op)
+	return binaryOps[op]
+}
+
+func binaryOpPriority(op string) int {
+	op = strings.ToLower(op)
+	return binaryOpPriorities[op]
+}
+
+func scanBinaryOpPrefix(s string) int {
+	n := 0
+	for op := range binaryOps {
+		if len(s) < len(op) {
+			continue
+		}
+		ss := strings.ToLower(s[:len(op)])
+		if ss == op && len(op) > n {
+			n = len(op)
+		}
+	}
+	return n
+}
+
+func isRightAssociativeBinaryOp(op string) bool {
+	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence
+	return op == "^"
+}
+
+func isBinaryOpGroupModifier(s string) bool {
+	s = strings.ToLower(s)
+	switch s {
+	// See https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
+	case "on", "ignoring":
+		return true
+	default:
+		return false
+	}
+}
+
+func isBinaryOpJoinModifier(s string) bool {
+	s = strings.ToLower(s)
+	switch s {
+	case "group_left", "group_right":
+		return true
+	default:
+		return false
+	}
+}
+
+func isBinaryOpBoolModifier(s string) bool {
+	s = strings.ToLower(s)
+	return s == "bool"
+}
+
+// IsBinaryOpCmp returns true if op is comparison operator such as '==', '!=', etc.
+func IsBinaryOpCmp(op string) bool {
+	switch op {
+	case "==", "!=", ">", "<", ">=", "<=":
+		return true
+	default:
+		return false
+	}
+}
+
+func isBinaryOpLogicalSet(op string) bool {
+	op = strings.ToLower(op)
+	switch op {
+	case "and", "or", "unless":
+		return true
+	default:
+		return false
+	}
+}
+
+func binaryOpEval(op string, left, right float64, isBool bool) float64 {
+	if IsBinaryOpCmp(op) {
+		evalCmp := func(cf func(left, right float64) bool) float64 {
+			if isBool {
+				if cf(left, right) {
+					return 1
+				}
+				return 0
+			}
+			if cf(left, right) {
+				return left
+			}
+			return nan
+		}
+		switch op {
+		case "==":
+			left = evalCmp(binaryop.Eq)
+		case "!=":
+			left = evalCmp(binaryop.Neq)
+		case ">":
+			left = evalCmp(binaryop.Gt)
+		case "<":
+			left = evalCmp(binaryop.Lt)
+		case ">=":
+			left = evalCmp(binaryop.Gte)
+		case "<=":
+			left = evalCmp(binaryop.Lte)
+		default:
+			panic(fmt.Errorf("BUG: unexpected comparison binaryOp: %q", op))
+		}
+	} else {
+		switch op {
+		case "+":
+			left = binaryop.Plus(left, right)
+		case "-":
+			left = binaryop.Minus(left, right)
+		case "*":
+			left = binaryop.Mul(left, right)
+		case "/":
+			left = binaryop.Div(left, right)
+		case "%":
+			left = binaryop.Mod(left, right)
+		case "^":
+			left = binaryop.Pow(left, right)
+		case "and":
+			// Nothing to do
+		case "or":
+			// Nothing to do
+		case "unless":
+			left = nan
+		case "default":
+			left = binaryop.Default(left, right)
+		case "if":
+			left = binaryop.If(left, right)
+		case "ifnot":
+			left = binaryop.Ifnot(left, right)
+		default:
+			panic(fmt.Errorf("BUG: unexpected non-comparison binaryOp: %q", op))
+		}
+	}
+	return left
+}
+
+var nan = math.NaN()

lib/metricsql/binary_op_test.go

@@ -1,4 +1,4 @@
-package promql
+package metricsql
 
 import (
 	"testing"

lib/metricsql/binaryop/funcs.go

@@ -0,0 +1,104 @@
+package binaryop
+
+import (
+	"math"
+)
+
+var nan = math.NaN()
+
+// Eq returns true of left == right.
+func Eq(left, right float64) bool {
+	// Special handling for nan == nan.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 .
+	if math.IsNaN(left) {
+		return math.IsNaN(right)
+	}
+	return left == right
+}
+
+// Neq returns true of left != right.
+func Neq(left, right float64) bool {
+	// Special handling for comparison with nan.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 .
+	if math.IsNaN(left) {
+		return !math.IsNaN(right)
+	}
+	if math.IsNaN(right) {
+		return true
+	}
+	return left != right
+}
+
+// Gt returns true of left > right
+func Gt(left, right float64) bool {
+	return left > right
+}
+
+// Lt returns true if left < right
+func Lt(left, right float64) bool {
+	return left < right
+}
+
+// Gte returns true if left >= right
+func Gte(left, right float64) bool {
+	return left >= right
+}
+
+// Lte returns true if left <= right
+func Lte(left, right float64) bool {
+	return left <= right
+}
+
+// Plus returns left + right
+func Plus(left, right float64) float64 {
+	return left + right
+}
+
+// Minus returns left - right
+func Minus(left, right float64) float64 {
+	return left - right
+}
+
+// Mul returns left * right
+func Mul(left, right float64) float64 {
+	return left * right
+}
+
+// Div returns left / right
+func Div(left, right float64) float64 {
+	return left / right
+}
+
+// Mod returns mod(left, right)
+func Mod(left, right float64) float64 {
+	return math.Mod(left, right)
+}
+
+// Pow returns pow(left, right)
+func Pow(left, right float64) float64 {
+	return math.Pow(left, right)
+}
+
+// Default returns left or right if left is NaN.
+func Default(left, right float64) float64 {
+	if math.IsNaN(left) {
+		return right
+	}
+	return left
+}
+
+// If returns left if right is not NaN. Otherwise NaN is returned.
+func If(left, right float64) float64 {
+	if math.IsNaN(right) {
+		return nan
+	}
+	return left
+}
+
+// Ifnot returns left if right is NaN. Otherwise NaN is returned.
+func Ifnot(left, right float64) float64 {
+	if math.IsNaN(right) {
+		return left
+	}
+	return nan
+}

lib/metricsql/doc.go

@@ -0,0 +1,15 @@
+// Package metricsql implements MetricsQL parser.
+//
+// This parser can parse PromQL. Additionally it can parse all the MetricsQL extensions.
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL for details about MetricsQL extensions.
+//
+// Usage:
+//
+//    expr, err := metricsql.Parse(`sum(rate(foo{bar="baz"}[5m])) by (job)`)
+//    if err != nil {
+//        // parse error
+//    }
+//    // Now expr contains parsed MetricsQL as `*Expr` structs.
+//    // See Parse examples for more details.
+//
+package metricsql

lib/metricsql/lexer.go

@@ -1,11 +1,9 @@
-package promql
+package metricsql
 
 import (
 	"fmt"
 	"strconv"
 	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 type lexer struct {
@@ -222,7 +220,7 @@ func scanIdent(s string) string {
 		}
 	}
 	if i == 0 {
-		logger.Panicf("BUG: scanIdent couldn't find a single ident char; make sure isIdentPrefix called before scanIdent")
+		panic("BUG: scanIdent couldn't find a single ident char; make sure isIdentPrefix called before scanIdent")
 	}
 	return s[:i]
 }
@@ -281,7 +279,7 @@ func toHex(n byte) byte {
 	return 'a' + (n - 10)
 }
 
-func appendEscapedIdent(dst, s []byte) []byte {
+func appendEscapedIdent(dst []byte, s string) []byte {
 	for i := 0; i < len(s); i++ {
 		ch := s[i]
 		if isIdentChar(ch) {

lib/metricsql/lexer_test.go

@@ -1,4 +1,4 @@
-package promql
+package metricsql
 
 import (
 	"reflect"
@@ -28,7 +28,7 @@ func TestUnescapeIdent(t *testing.T) {
 func TestAppendEscapedIdent(t *testing.T) {
 	f := func(s, resultExpected string) {
 		t.Helper()
-		result := appendEscapedIdent(nil, []byte(s))
+		result := appendEscapedIdent(nil, s)
 		if string(result) != resultExpected {
 			t.Fatalf("unexpected result for appendEscapedIdent(%q); got %q; want %q", s, result, resultExpected)
 		}

lib/metricsql/parser_example_test.go

@@ -0,0 +1,35 @@
+package metricsql_test
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
+)
+
+func ExampleParse() {
+	expr, err := metricsql.Parse(`sum(rate(foo{bar="baz"}[5m])) by (x,y)`)
+	if err != nil {
+		log.Fatalf("parse error: %s", err)
+	}
+	fmt.Printf("parsed expr: %s\n", expr.AppendString(nil))
+
+	ae := expr.(*metricsql.AggrFuncExpr)
+	fmt.Printf("aggr func: name=%s, arg=%s, modifier=%s\n", ae.Name, ae.Args[0].AppendString(nil), ae.Modifier.AppendString(nil))
+
+	fe := ae.Args[0].(*metricsql.FuncExpr)
+	fmt.Printf("func: name=%s, arg=%s\n", fe.Name, fe.Args[0].AppendString(nil))
+
+	re := fe.Args[0].(*metricsql.RollupExpr)
+	fmt.Printf("rollup: expr=%s, window=%s\n", re.Expr.AppendString(nil), re.Window)
+
+	me := re.Expr.(*metricsql.MetricExpr)
+	fmt.Printf("metric: labelFilter1=%s, labelFilter2=%s", me.LabelFilters[0].AppendString(nil), me.LabelFilters[1].AppendString(nil))
+
+	// Output:
+	// parsed expr: sum(rate(foo{bar="baz"}[5m])) by (x, y)
+	// aggr func: name=sum, arg=rate(foo{bar="baz"}[5m]), modifier=by (x, y)
+	// func: name=rate, arg=foo{bar="baz"}[5m]
+	// rollup: expr=foo{bar="baz"}, window=5m
+	// metric: labelFilter1=__name__="foo", labelFilter2=bar="baz"
+}

lib/metricsql/parser_test.go

@@ -0,0 +1,679 @@
+package metricsql
+
+import (
+	"testing"
+)
+
+func TestParseSuccess(t *testing.T) {
+	another := func(s string, sExpected string) {
+		t.Helper()
+
+		e, err := Parse(s)
+		if err != nil {
+			t.Fatalf("unexpected error when parsing %q: %s", s, err)
+		}
+		res := e.AppendString(nil)
+		if string(res) != sExpected {
+			t.Fatalf("unexpected string constructed;\ngot\n%q\nwant\n%q", res, sExpected)
+		}
+	}
+	same := func(s string) {
+		t.Helper()
+		another(s, s)
+	}
+
+	// metricExpr
+	same(`{}`)
+	same(`{}[5m]`)
+	same(`{}[5m:]`)
+	same(`{}[:]`)
+	another(`{}[: ]`, `{}[:]`)
+	same(`{}[:3s]`)
+	another(`{}[: 3s ]`, `{}[:3s]`)
+	same(`{}[5m:3s]`)
+	another(`{}[ 5m : 3s ]`, `{}[5m:3s]`)
+	same(`{} offset 5m`)
+	same(`{} offset -5m`)
+	same(`{}[5m] offset 10y`)
+	same(`{}[5.3m:3.4s] offset 10y`)
+	same(`{}[:3.4s] offset 10y`)
+	same(`{}[:3.4s] offset -10y`)
+	same(`{Foo="bAR"}`)
+	same(`{foo="bar"}`)
+	same(`{foo="bar"}[5m]`)
+	same(`{foo="bar"}[5m:]`)
+	same(`{foo="bar"}[5m:3s]`)
+	same(`{foo="bar"} offset 10y`)
+	same(`{foo="bar"} offset -10y`)
+	same(`{foo="bar"}[5m] offset 10y`)
+	same(`{foo="bar"}[5m:3s] offset 10y`)
+	another(`{foo="bar"}[5m] oFFSEt 10y`, `{foo="bar"}[5m] offset 10y`)
+	same("METRIC")
+	same("metric")
+	same("m_e:tri44:_c123")
+	another("-metric", "0 - metric")
+	same(`metric offset 10h`)
+	same("metric[5m]")
+	same("metric[5m:3s]")
+	same("metric[5m] offset 10h")
+	same("metric[5m:3s] offset 10h")
+	same("metric[5i:3i] offset 10i")
+	same(`metric{foo="bar"}`)
+	same(`metric{foo="bar"} offset 10h`)
+	same(`metric{foo!="bar"}[2d]`)
+	same(`metric{foo="bar"}[2d] offset 10h`)
+	same(`metric{foo="bar", b="sdfsdf"}[2d:3h] offset 10h`)
+	another(`  metric  {  foo  = "bar"  }  [  2d ]   offset   10h  `, `metric{foo="bar"}[2d] offset 10h`)
+	// metric name matching keywords
+	same("rate")
+	same("RATE")
+	same("by")
+	same("BY")
+	same("bool")
+	same("BOOL")
+	same("unless")
+	same("UNLESS")
+	same("Ignoring")
+	same("with")
+	same("WITH")
+	same("With")
+	same("alias")
+	same(`alias{foo="bar"}`)
+	same(`aLIas{alias="aa"}`)
+	another(`al\ias`, `alias`)
+	// identifiers with with escape chars
+	same(`foo\ bar`)
+	same(`foo\-bar\{{baz\+bar="aa"}`)
+	another(`\x2E\x2ef\oo{b\xEF\ar="aa"}`, `\x2e.foo{b\xefar="aa"}`)
+	// Duplicate filters
+	same(`foo{__name__="bar"}`)
+	same(`foo{a="b", a="c", __name__="aaa", b="d"}`)
+	// Metric filters ending with comma
+	another(`m{foo="bar",}`, `m{foo="bar"}`)
+	// String concat in tag value
+	another(`m{foo="bar" + "baz"}`, `m{foo="barbaz"}`)
+
+	// Valid regexp
+	same(`foo{bar=~"x"}`)
+	same(`foo{bar=~"^x"}`)
+	same(`foo{bar=~"^x$"}`)
+	same(`foo{bar=~"^(a[bc]|d)$"}`)
+	same(`foo{bar!~"x"}`)
+	same(`foo{bar!~"^x"}`)
+	same(`foo{bar!~"^x$"}`)
+	same(`foo{bar!~"^(a[bc]|d)$"}`)
+
+	// stringExpr
+	same(`""`)
+	same(`"\n\t\r 12:{}[]()44"`)
+	another(`''`, `""`)
+	another("``", `""`)
+	another("   `foo\"b'ar`  ", "\"foo\\\"b'ar\"")
+	another(`  'foo\'bar"BAZ'  `, `"foo'bar\"BAZ"`)
+	// string concat
+	another(`"foo"+'bar'`, `"foobar"`)
+
+	// numberExpr
+	same(`1`)
+	same(`1.23`)
+	same(`0.23`)
+	same(`1.2e+45`)
+	same(`1.2e-45`)
+	same(`-1`)
+	same(`-1.23`)
+	same(`-0.23`)
+	same(`-1.2e+45`)
+	same(`-1.2e-45`)
+	same(`-1.2e-45`)
+	another(`12.5E34`, `1.25e+35`)
+	another(`-.2`, `-0.2`)
+	another(`-.2E-2`, `-0.002`)
+	same(`NaN`)
+	another(`nan`, `NaN`)
+	another(`NAN`, `NaN`)
+	another(`nAN`, `NaN`)
+	another(`Inf`, `+Inf`)
+	another(`INF`, `+Inf`)
+	another(`inf`, `+Inf`)
+	another(`+Inf`, `+Inf`)
+	another(`-Inf`, `-Inf`)
+	another(`-inF`, `-Inf`)
+
+	// binaryOpExpr
+	another(`nan == nan`, `NaN`)
+	another(`nan ==bool nan`, `1`)
+	another(`nan !=bool nan`, `0`)
+	another(`nan !=bool 2`, `1`)
+	another(`2 !=bool nan`, `1`)
+	another(`nan >bool nan`, `0`)
+	another(`nan <bool nan`, `0`)
+	another(`1 ==bool nan`, `0`)
+	another(`NaN !=bool 1`, `1`)
+	another(`inf >=bool 2`, `1`)
+	another(`-1 >bool -inf`, `1`)
+	another(`-1 <bool -inf`, `0`)
+	another(`nan + 2 *3 * inf`, `NaN`)
+	another(`INF - Inf`, `NaN`)
+	another(`Inf + inf`, `+Inf`)
+	another(`1/0`, `+Inf`)
+	another(`0/0`, `NaN`)
+	another(`-m`, `0 - m`)
+	same(`m + ignoring () n[5m]`)
+	another(`M + IGNORING () N[5m]`, `M + ignoring () N[5m]`)
+	same(`m + on (foo) n[5m]`)
+	another(`m + ON (Foo) n[5m]`, `m + on (Foo) n[5m]`)
+	same(`m + ignoring (a, b) n[5m]`)
+	another(`1 or 2`, `1`)
+	another(`1 and 2`, `1`)
+	another(`1 unless 2`, `NaN`)
+	another(`1 default 2`, `1`)
+	another(`1 default NaN`, `1`)
+	another(`NaN default 2`, `2`)
+	another(`1 > 2`, `NaN`)
+	another(`1 > bool 2`, `0`)
+	another(`3 >= 2`, `3`)
+	another(`3 <= bool 2`, `0`)
+	another(`1 + -2 - 3`, `-4`)
+	another(`1 / 0 + 2`, `+Inf`)
+	another(`2 + -1 / 0`, `-Inf`)
+	another(`-1 ^ 0.5`, `NaN`)
+	another(`512.5 - (1 + 3) * (2 ^ 2) ^ 3`, `256.5`)
+	another(`1 == bool 1 != bool 24 < bool 4 > bool -1`, `1`)
+	another(`1 == bOOl 1 != BOOL 24 < Bool 4 > booL -1`, `1`)
+	another(`m1+on(foo)group_left m2`, `m1 + on (foo) group_left () m2`)
+	another(`M1+ON(FOO)GROUP_left M2`, `M1 + on (FOO) group_left () M2`)
+	same(`m1 + on (foo) group_right () m2`)
+	same(`m1 + on (foo, bar) group_right (x, y) m2`)
+	another(`m1 + on (foo, bar,) group_right (x, y,) m2`, `m1 + on (foo, bar) group_right (x, y) m2`)
+	same(`m1 == bool on (foo, bar) group_right (x, y) m2`)
+	another(`5 - 1 + 3 * 2 ^ 2 ^ 3 - 2  OR Metric {Bar= "Baz", aaa!="bb",cc=~"dd" ,zz !~"ff" } `,
+		`770 or Metric{Bar="Baz", aaa!="bb", cc=~"dd", zz!~"ff"}`)
+	same(`"foo" + bar()`)
+	same(`"foo" + bar{x="y"}`)
+	same(`("foo"[3s] + bar{x="y"})[5m:3s] offset 10s`)
+	same(`("foo"[3s] + bar{x="y"})[5i:3i] offset 10i`)
+	same(`bar + "foo" offset 3s`)
+	same(`bar + "foo" offset 3i`)
+	another(`1+2 if 2>3`, `NaN`)
+	another(`1+4 if 2<3`, `5`)
+	another(`2+6 default 3 if 2>3`, `8`)
+	another(`2+6 if 2>3 default NaN`, `NaN`)
+	another(`42 if 3>2 if 2+2<5`, `42`)
+	another(`42 if 3>2 if 2+2>=5`, `NaN`)
+	another(`1+2 ifnot 2>3`, `3`)
+	another(`1+4 ifnot 2<3`, `NaN`)
+	another(`2+6 default 3 ifnot 2>3`, `8`)
+	another(`2+6 ifnot 2>3 default NaN`, `8`)
+	another(`42 if 3>2 ifnot 2+2<5`, `NaN`)
+	another(`42 if 3>2 ifnot 2+2>=5`, `42`)
+
+	// parensExpr
+	another(`(-foo + ((bar) / (baz))) + ((23))`, `((0 - foo) + (bar / baz)) + 23`)
+	another(`(FOO + ((Bar) / (baZ))) + ((23))`, `(FOO + (Bar / baZ)) + 23`)
+	same(`(foo, bar)`)
+	another(`((foo, bar),(baz))`, `((foo, bar), baz)`)
+	same(`(foo, (bar, baz), ((x, y), (z, y), xx))`)
+	another(`1+(foo, bar,)`, `1 + (foo, bar)`)
+	another(`((foo(bar,baz)), (1+(2)+(3,4)+()))`, `(foo(bar, baz), (3 + (3, 4)) + ())`)
+	same(`()`)
+
+	// funcExpr
+	same(`f()`)
+	another(`f(x,)`, `f(x)`)
+	another(`-f()-Ff()`, `(0 - f()) - Ff()`)
+	same(`F()`)
+	another(`+F()`, `F()`)
+	another(`++F()`, `F()`)
+	another(`--F()`, `0 - (0 - F())`)
+	same(`f(http_server_request)`)
+	same(`f(http_server_request)[4s:5m] offset 10m`)
+	same(`f(http_server_request)[4i:5i] offset 10i`)
+	same(`F(HttpServerRequest)`)
+	same(`f(job, foo)`)
+	same(`F(Job, Foo)`)
+	another(` FOO (bar) + f  (  m  (  ),ff(1 + (  2.5)) ,M[5m ]  , "ff"  )`, `FOO(bar) + f(m(), ff(3.5), M[5m], "ff")`)
+	// funcName matching keywords
+	same(`by(2)`)
+	same(`BY(2)`)
+	same(`or(2)`)
+	same(`OR(2)`)
+	same(`bool(2)`)
+	same(`BOOL(2)`)
+	same(`rate(rate(m))`)
+	same(`rate(rate(m[5m]))`)
+	same(`rate(rate(m[5m])[1h:])`)
+	same(`rate(rate(m[5m])[1h:3s])`)
+	// funcName with escape chars
+	same(`foo\(ba\-r()`)
+
+	// aggrFuncExpr
+	same(`sum(http_server_request) by ()`)
+	same(`sum(http_server_request) by (job)`)
+	same(`sum(http_server_request) without (job, foo)`)
+	another(`sum(x,y,) without (a,b,)`, `sum(x, y) without (a, b)`)
+	another(`sum by () (xx)`, `sum(xx) by ()`)
+	another(`sum by (s) (xx)[5s]`, `(sum(xx) by (s))[5s]`)
+	another(`SUM BY (ZZ, aa) (XX)`, `sum(XX) by (ZZ, aa)`)
+	another(`sum without (a, b) (xx,2+2)`, `sum(xx, 4) without (a, b)`)
+	another(`Sum WIthout (a, B) (XX,2+2)`, `sum(XX, 4) without (a, B)`)
+	same(`sum(a) or sum(b)`)
+	same(`sum(a) by () or sum(b) without (x, y)`)
+	same(`sum(a) + sum(b)`)
+	same(`sum(x) * (1 + sum(a))`)
+
+	// All the above
+	another(`Sum(Ff(M) * M{X=""}[5m] Offset 7m - 123, 35) BY (X, y) * F2("Test")`,
+		`sum((Ff(M) * M{X=""}[5m] offset 7m) - 123, 35) by (X, y) * F2("Test")`)
+	another(`# comment
+		Sum(Ff(M) * M{X=""}[5m] Offset 7m - 123, 35) BY (X, y) # yet another comment
+		* F2("Test")`,
+		`sum((Ff(M) * M{X=""}[5m] offset 7m) - 123, 35) by (X, y) * F2("Test")`)
+
+	// withExpr
+	another(`with () x`, `x`)
+	another(`with (x=1,) x`, `1`)
+	another(`with (x = m offset 5h) x + x`, `m offset 5h + m offset 5h`)
+	another(`with (x = m offset 5i) x + x`, `m offset 5i + m offset 5i`)
+	another(`with (foo = bar{x="x"}) 1`, `1`)
+	another(`with (foo = bar{x="x"}) "x"`, `"x"`)
+	another(`with (f="x") f`, `"x"`)
+	another(`with (foo = bar{x="x"}) x{x="y"}`, `x{x="y"}`)
+	another(`with (foo = bar{x="x"}) 1+1`, `2`)
+	another(`with (foo = bar{x="x"}) f()`, `f()`)
+	another(`with (foo = bar{x="x"}) sum(x)`, `sum(x)`)
+	another(`with (foo = bar{x="x"}) baz{foo="bar"}`, `baz{foo="bar"}`)
+	another(`with (foo = bar) baz`, `baz`)
+	another(`with (foo = bar) foo + foo{a="b"}`, `bar + bar{a="b"}`)
+	another(`with (foo = bar, bar=baz + f()) test`, `test`)
+	another(`with (ct={job="test"}) a{ct} + ct() + f({ct="x"})`, `(a{job="test"} + {job="test"}) + f({ct="x"})`)
+	another(`with (ct={job="test", i="bar"}) ct + {ct, x="d"} + foo{ct, ct} + ctx(1)`,
+		`(({job="test", i="bar"} + {job="test", i="bar", x="d"}) + foo{job="test", i="bar"}) + ctx(1)`)
+	another(`with (foo = bar) {__name__=~"foo"}`, `{__name__=~"foo"}`)
+	another(`with (foo = bar) foo{__name__="foo"}`, `bar`)
+	another(`with (foo = bar) {__name__="foo", x="y"}`, `bar{x="y"}`)
+	another(`with (foo(bar) = {__name__!="bar"}) foo(x)`, `{__name__!="bar"}`)
+	another(`with (foo(bar) = bar{__name__="bar"}) foo(x)`, `x`)
+	another(`with (foo\-bar(baz) = baz + baz) foo\-bar((x,y))`, `(x, y) + (x, y)`)
+	another(`with (foo\-bar(baz) = baz + baz) foo\-bar(x*y)`, `(x * y) + (x * y)`)
+	another(`with (foo\-bar(baz) = baz + baz) foo\-bar(x\*y)`, `x\*y + x\*y`)
+	another(`with (foo\-bar(b\ az) = b\ az + b\ az) foo\-bar(x\*y)`, `x\*y + x\*y`)
+	// override ttf to something new.
+	another(`with (ttf = a) ttf + b`, `a + b`)
+	// override ttf to ru
+	another(`with (ttf = ru(m, n)) ttf`, `(clamp_min(n - clamp_min(m, 0), 0) / clamp_min(n, 0)) * 100`)
+
+	// Verify withExpr recursion and forward reference
+	another(`with (x = x+y, y = x+x) y ^ 2`, `((x + y) + (x + y)) ^ 2`)
+	another(`with (f1(x)=f2(x), f2(x)=f1(x)^2) f1(foobar)`, `f2(foobar)`)
+	another(`with (f1(x)=f2(x), f2(x)=f1(x)^2) f2(foobar)`, `f2(foobar) ^ 2`)
+
+	// Verify withExpr funcs
+	another(`with (x() = y+1) x`, `y + 1`)
+	another(`with (x(foo) = foo+1) x(a)`, `a + 1`)
+	another(`with (x(a, b) = a + b) x(foo, bar)`, `foo + bar`)
+	another(`with (x(a, b) = a + b) x(foo, x(1, 2))`, `foo + 3`)
+	another(`with (x(a) = sum(a) by (b)) x(xx) / x(y)`, `sum(xx) by (b) / sum(y) by (b)`)
+	another(`with (f(a,f,x)=ff(x,f,a)) f(f(x,y,z),1,2)`, `ff(2, 1, ff(z, y, x))`)
+	another(`with (f(x)=1+f(x)) f(foo{bar="baz"})`, `1 + f(foo{bar="baz"})`)
+	another(`with (a=foo, y=bar, f(a)= a+a+y) f(x)`, `(x + x) + bar`)
+	another(`with (f(a, b) = m{a, b}) f({a="x", b="y"}, {c="d"})`, `m{a="x", b="y", c="d"}`)
+	another(`with (xx={a="x"}, f(a, b) = m{a, b}) f({xx, b="y"}, {c="d"})`, `m{a="x", b="y", c="d"}`)
+	another(`with (x() = {b="c"}) foo{x}`, `foo{b="c"}`)
+	another(`with (f(x)=x{foo="bar"} offset 5m) f(m offset 10m)`, `(m{foo="bar"} offset 10m) offset 5m`)
+	another(`with (f(x)=x{foo="bar",bas="a"}[5m]) f(m[10m] offset 3s)`, `(m{foo="bar", bas="a"}[10m] offset 3s)[5m]`)
+	another(`with (f(x)=x{foo="bar"}[5m] offset 10m) f(m{x="y"})`, `m{x="y", foo="bar"}[5m] offset 10m`)
+	another(`with (f(x)=x{foo="bar"}[5m] offset 10m) f({x="y", foo="bar", foo="bar"})`, `{x="y", foo="bar"}[5m] offset 10m`)
+	another(`with (f(m, x)=m{x}[5m] offset 10m) f(foo, {})`, `foo[5m] offset 10m`)
+	another(`with (f(m, x)=m{x, bar="baz"}[5m] offset 10m) f(foo, {})`, `foo{bar="baz"}[5m] offset 10m`)
+	another(`with (f(x)=x[5m] offset 3s) f(foo[3m]+bar)`, `(foo[3m] + bar)[5m] offset 3s`)
+	another(`with (f(x)=x[5m:3s] oFFsEt 1.5m) f(sum(s) by (a,b))`, `(sum(s) by (a, b))[5m:3s] offset 1.5m`)
+	another(`with (x="a", y=x) y+"bc"`, `"abc"`)
+	another(`with (x="a", y="b"+x) "we"+y+"z"+f()`, `"webaz" + f()`)
+	another(`with (f(x) = m{foo=x+"y", bar="y"+x, baz=x} + x) f("qwe")`, `m{foo="qwey", bar="yqwe", baz="qwe"} + "qwe"`)
+	another(`with (f(a)=a) f`, `f`)
+	another(`with (f\q(a)=a) f\q`, `fq`)
+
+	// Verify withExpr for aggr func modifiers
+	another(`with (f(x) = x, y = sum(m) by (f)) y`, `sum(m) by (f)`)
+	another(`with (f(x) = sum(m) by (x)) f(foo)`, `sum(m) by (foo)`)
+	another(`with (f(x) = sum(m) by (x)) f((foo, bar, foo))`, `sum(m) by (foo, bar)`)
+	another(`with (f(x) = sum(m) without (x,y)) f((a, b))`, `sum(m) without (a, b, y)`)
+	another(`with (f(x) = sum(m) without (y,x)) f((a, y))`, `sum(m) without (y, a)`)
+	another(`with (f(x,y) = a + on (x,y) group_left (y,bar) b) f(foo,())`, `a + on (foo) group_left (bar) b`)
+	another(`with (f(x,y) = a + on (x,y) group_left (y,bar) b) f((foo),())`, `a + on (foo) group_left (bar) b`)
+	another(`with (f(x,y) = a + on (x,y) group_left (y,bar) b) f((foo,xx),())`, `a + on (foo, xx) group_left (bar) b`)
+
+	// Verify nested with exprs
+	another(`with (f(x) = (with(x=y) x) + x) f(z)`, `y + z`)
+	another(`with (x=foo) f(a, with (y=x) y)`, `f(a, foo)`)
+	another(`with (x=foo) a * x + (with (y=x) y) / y`, `(a * foo) + (foo / y)`)
+	another(`with (x = with (y = foo) y + x) x/x`, `(foo + x) / (foo + x)`)
+	another(`with (
+		x = {foo="bar"},
+		q = m{x, y="1"},
+		f(x) =
+			with (
+				z(y) = x + y * q
+			)
+			z(foo) / f(x)
+	)
+	f(a)`, `(a + (foo * m{foo="bar", y="1"})) / f(a)`)
+
+	// complex withExpr
+	another(`WITH (
+		treshold = (0.9),
+		commonFilters = {job="cacher", instance=~"1.2.3.4"},
+		hits = rate(cache{type="hit", commonFilters}[5m]),
+		miss = rate(cache{type="miss", commonFilters}[5m]),
+		sumByInstance(arg) = sum(arg) by (instance),
+		hitRatio = sumByInstance(hits) / sumByInstance(hits + miss)
+	)
+	hitRatio < treshold`,
+		`(sum(rate(cache{type="hit", job="cacher", instance=~"1.2.3.4"}[5m])) by (instance) / sum(rate(cache{type="hit", job="cacher", instance=~"1.2.3.4"}[5m]) + rate(cache{type="miss", job="cacher", instance=~"1.2.3.4"}[5m])) by (instance)) < 0.9`)
+	another(`WITH (
+		x2(x) = x^2,
+		f(x, y) = x2(x) + x*y + x2(y)
+	)
+	f(a, 3)
+	`, `((a ^ 2) + (a * 3)) + 9`)
+	another(`WITH (
+		x2(x) = x^2,
+		f(x, y) = x2(x) + x*y + x2(y)
+	)
+	f(2, 3)
+	`, `19`)
+	another(`WITH (
+		commonFilters = {instance="foo"},
+		timeToFuckup(currv, maxv) = (maxv - currv) / rate(currv)
+	)
+	timeToFuckup(diskUsage{commonFilters}, maxDiskSize{commonFilters})`,
+		`(maxDiskSize{instance="foo"} - diskUsage{instance="foo"}) / rate(diskUsage{instance="foo"})`)
+	another(`WITH (
+	       commonFilters = {job="foo", instance="bar"},
+	       sumRate(m, cf) = sum(rate(m{cf})) by (job, instance),
+	       hitRate(hits, misses) = sumRate(hits, commonFilters) / (sumRate(hits, commonFilters) + sumRate(misses, commonFilters))
+	   )
+	   hitRate(cacheHits, cacheMisses)`,
+		`sum(rate(cacheHits{job="foo", instance="bar"})) by (job, instance) / (sum(rate(cacheHits{job="foo", instance="bar"})) by (job, instance) + sum(rate(cacheMisses{job="foo", instance="bar"})) by (job, instance))`)
+	another(`with(y=123,z=5) union(with(y=3,f(x)=x*y) f(2) + f(3), with(x=5,y=2) x*y*z)`, `union(15, 50)`)
+}
+
+func TestParseError(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+
+		e, err := Parse(s)
+		if err == nil {
+			t.Fatalf("expecting non-nil error when parsing %q", s)
+		}
+		if e != nil {
+			t.Fatalf("expecting nil expr when parsing %q", s)
+		}
+	}
+
+	// an empty string
+	f("")
+	f("  \t\b\r\n  ")
+
+	// invalid metricExpr
+	f(`{__name__="ff"} offset 55`)
+	f(`foo[55]`)
+	f(`m[-5m]`)
+	f(`{`)
+	f(`foo{`)
+	f(`foo{bar`)
+	f(`foo{bar=`)
+	f(`foo{bar="baz"`)
+	f(`foo{bar="baz",  `)
+	f(`foo{123="23"}`)
+	f(`foo{foo}`)
+	f(`foo{,}`)
+	f(`foo{,foo="bar"}`)
+	f(`foo{foo=}`)
+	f(`foo{foo="ba}`)
+	f(`foo{"foo"="bar"}`)
+	f(`foo{$`)
+	f(`foo{a $`)
+	f(`foo{a="b",$`)
+	f(`foo{a="b"}$`)
+	f(`[`)
+	f(`[]`)
+	f(`f[5m]$`)
+	f(`[5m]`)
+	f(`[5m] offset 4h`)
+	f(`m[5m] offset $`)
+	f(`m[5m] offset 5h $`)
+	f(`m[]`)
+	f(`m[-5m]`)
+	f(`m[5m:`)
+	f(`m[5m:-`)
+	f(`m[5m:-1`)
+	f(`m[5m:-1]`)
+	f(`m[5m:-1s]`)
+	f(`m[-5m:1s]`)
+	f(`m[-5m:-1s]`)
+	f(`m[:`)
+	f(`m[:-`)
+	f(`m[:1]`)
+	f(`m[:-1m]`)
+	f(`m[5]`)
+	f(`m[[5m]]`)
+	f(`m[foo]`)
+	f(`m["ff"]`)
+	f(`m[10m`)
+	f(`m[123`)
+	f(`m["ff`)
+	f(`m[(f`)
+	f(`fd}`)
+	f(`]`)
+	f(`m $`)
+	f(`m{,}`)
+	f(`m{x=y}`)
+	f(`m{x=y/5}`)
+	f(`m{x=y+5}`)
+
+	// Invalid regexp
+	f(`foo{bar=~"x["}`)
+	f(`foo{bar=~"x("}`)
+	f(`foo{bar=~"x)"}`)
+	f(`foo{bar!~"x["}`)
+	f(`foo{bar!~"x("}`)
+	f(`foo{bar!~"x)"}`)
+
+	// invalid stringExpr
+	f(`'`)
+	f(`"`)
+	f("`")
+	f(`"foo`)
+	f(`'foo`)
+	f("`foo")
+	f(`"foo\"bar`)
+	f(`'foo\'bar`)
+	f("`foo\\`bar")
+	f(`"" $`)
+	f(`"foo" +`)
+	f(`n{"foo" + m`)
+
+	// invalid numberExpr
+	f(`12.`)
+	f(`1.2e`)
+	f(`23e-`)
+	f(`23E+`)
+	f(`.`)
+	f(`-12.`)
+	f(`-1.2e`)
+	f(`-23e-`)
+	f(`-23E+`)
+	f(`-.`)
+	f(`-1$$`)
+	f(`-$$`)
+	f(`+$$`)
+	f(`23 $$`)
+
+	// invalid binaryOpExpr
+	f(`+`)
+	f(`1 +`)
+	f(`1 + 2.`)
+	f(`3 unless`)
+	f(`23 + on (foo)`)
+	f(`m + on (,) m`)
+	f(`3 * ignoring`)
+	f(`m * on (`)
+	f(`m * on (foo`)
+	f(`m * on (foo,`)
+	f(`m * on (foo,)`)
+	f(`m * on (,foo)`)
+	f(`m * on (,)`)
+	f(`m == bool (bar) baz`)
+	f(`m == bool () baz`)
+	f(`m * by (baz) n`)
+	f(`m + bool group_left m2`)
+	f(`m + on () group_left (`)
+	f(`m + on () group_left (,`)
+	f(`m + on () group_left (,foo`)
+	f(`m + on () group_left (foo,)`)
+	f(`m + on () group_left (,foo)`)
+	f(`m + on () group_left (foo)`)
+	f(`m + on () group_right (foo) (m`)
+	f(`m or ignoring () group_left () n`)
+	f(`1 + bool 2`)
+	f(`m % bool n`)
+	f(`m * bool baz`)
+	f(`M * BOoL BaZ`)
+	f(`foo unless ignoring (bar) group_left xxx`)
+	f(`foo or bool bar`)
+	f(`foo == bool $$`)
+	f(`"foo" + bar`)
+
+	// invalid parensExpr
+	f(`(`)
+	f(`($`)
+	f(`(+`)
+	f(`(1`)
+	f(`(m+`)
+	f(`1)`)
+	f(`(,)`)
+	f(`(1)$`)
+
+	// invalid funcExpr
+	f(`f $`)
+	f(`f($)`)
+	f(`f[`)
+	f(`f()$`)
+	f(`f(`)
+	f(`f(foo`)
+	f(`f(f,`)
+	f(`f(,`)
+	f(`f(,)`)
+	f(`f(,foo)`)
+	f(`f(,foo`)
+	f(`f(foo,$`)
+	f(`f() by (a)`)
+	f(`f without (x) (y)`)
+	f(`f() foo (a)`)
+	f(`f bar (x) (b)`)
+	f(`f bar (x)`)
+
+	// invalid aggrFuncExpr
+	f(`sum(`)
+	f(`sum $`)
+	f(`sum [`)
+	f(`sum($)`)
+	f(`sum()$`)
+	f(`sum(foo) ba`)
+	f(`sum(foo) ba()`)
+	f(`sum(foo) by`)
+	f(`sum(foo) without x`)
+	f(`sum(foo) aaa`)
+	f(`sum(foo) aaa x`)
+	f(`sum() by $`)
+	f(`sum() by (`)
+	f(`sum() by ($`)
+	f(`sum() by (a`)
+	f(`sum() by (a $`)
+	f(`sum() by (a ]`)
+	f(`sum() by (a)$`)
+	f(`sum() by (,`)
+	f(`sum() by (a,$`)
+	f(`sum() by (,)`)
+	f(`sum() by (,a`)
+	f(`sum() by (,a)`)
+	f(`sum() on (b)`)
+	f(`sum() bool`)
+	f(`sum() group_left`)
+	f(`sum() group_right(x)`)
+	f(`sum ba`)
+	f(`sum ba ()`)
+	f(`sum by (`)
+	f(`sum by (a`)
+	f(`sum by (,`)
+	f(`sum by (,)`)
+	f(`sum by (,a`)
+	f(`sum by (,a)`)
+	f(`sum by (a)`)
+	f(`sum by (a) (`)
+	f(`sum by (a) [`)
+	f(`sum by (a) {`)
+	f(`sum by (a) (b`)
+	f(`sum by (a) (b,`)
+	f(`sum by (a) (,)`)
+	f(`avg by (a) (,b)`)
+	f(`sum by (x) (y) by (z)`)
+	f(`sum(m) by (1)`)
+
+	// invalid withExpr
+	f(`with $`)
+	f(`with a`)
+	f(`with a=b c`)
+	f(`with (`)
+	f(`with (x=b)$`)
+	f(`with ($`)
+	f(`with (foo`)
+	f(`with (foo $`)
+	f(`with (x y`)
+	f(`with (x =`)
+	f(`with (x = $`)
+	f(`with (x= y`)
+	f(`with (x= y $`)
+	f(`with (x= y)`)
+	f(`with (x=(`)
+	f(`with (x=[)`)
+	f(`with (x=() x)`)
+	f(`with ($$)`)
+	f(`with (x $$`)
+	f(`with (x = $$)`)
+	f(`with (x = foo) bar{x}`)
+	f(`with (x = {foo="bar"}[5m]) bar{x}`)
+	f(`with (x = {foo="bar"} offset 5m) bar{x}`)
+	f(`with (x = a, x = b) c`)
+	f(`with (x(a, a) = b) c`)
+	f(`with (x=m{f="x"}) foo{x}`)
+	f(`with (sum = x) y`)
+	f(`with (rate(a) = b) c`)
+	f(`with (clamp_min=x) y`)
+	f(`with (f()`)
+	f(`with (a=b c=d) e`)
+	f(`with (f(x)=x^2) m{x}`)
+	f(`with (f(x)=ff()) m{x}`)
+	f(`with (f(x`)
+	f(`with (x=m) a{x} + b`)
+	f(`with (x=m) b + a{x}`)
+	f(`with (x=m) f(b, a{x})`)
+	f(`with (x=m) sum(a{x})`)
+	f(`with (x=m) (a{x})`)
+	f(`with (f(a)=a) f(1, 2)`)
+	f(`with (f(x)=x{foo="bar"}) f(1)`)
+	f(`with (f(x)=x{foo="bar"}) f(m + n)`)
+	f(`with (f = with`)
+	f(`with (,)`)
+	f(`with (1) 2`)
+	f(`with (f(1)=2) 3`)
+	f(`with (f(,)=x) x`)
+	f(`with (x(a) = {b="c"}) foo{x}`)
+	f(`with (f(x) = m{foo=xx}) f("qwe")`)
+	f(`a + with(f(x)=x) f(1,2)`)
+	f(`with (f(x) = sum(m) by (x)) f({foo="bar"})`)
+	f(`with (f(x) = sum(m) by (x)) f((xx(), {foo="bar"}))`)
+	f(`with (f(x) = m + on (x) n) f(xx())`)
+	f(`with (f(x) = m + on (a) group_right (x) n) f(xx())`)
+}

lib/metricsql/regexp_cache.go

@@ -1,4 +1,4 @@
-package promql
+package metricsql
 
 import (
 	"regexp"
@@ -8,12 +8,14 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-func compileRegexpAnchored(re string) (*regexp.Regexp, error) {
+// CompileRegexpAnchored returns compiled regexp `^re$`.
+func CompileRegexpAnchored(re string) (*regexp.Regexp, error) {
 	reAnchored := "^(?:" + re + ")$"
-	return compileRegexp(reAnchored)
+	return CompileRegexp(reAnchored)
 }
 
-func compileRegexp(re string) (*regexp.Regexp, error) {
+// CompileRegexp returns compile regexp re.
+func CompileRegexp(re string) (*regexp.Regexp, error) {
 	rcv := regexpCacheV.Get(re)
 	if rcv != nil {
 		return rcv.r, rcv.err

lib/metricsql/rollup.go

@@ -0,0 +1,66 @@
+package metricsql
+
+import (
+	"strings"
+)
+
+var rollupFuncs = map[string]bool{
+	// Standard rollup funcs from PromQL.
+	// See funcs accepting range-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
+	"changes":            true,
+	"delta":              true,
+	"deriv":              true,
+	"deriv_fast":         true,
+	"holt_winters":       true,
+	"idelta":             true,
+	"increase":           true,
+	"irate":              true,
+	"predict_linear":     true,
+	"rate":               true,
+	"resets":             true,
+	"avg_over_time":      true,
+	"min_over_time":      true,
+	"max_over_time":      true,
+	"sum_over_time":      true,
+	"count_over_time":    true,
+	"quantile_over_time": true,
+	"stddev_over_time":   true,
+	"stdvar_over_time":   true,
+	"absent_over_time":   true,
+
+	// Additional rollup funcs.
+	"default_rollup":        true,
+	"range_over_time":       true,
+	"sum2_over_time":        true,
+	"geomean_over_time":     true,
+	"first_over_time":       true,
+	"last_over_time":        true,
+	"distinct_over_time":    true,
+	"increases_over_time":   true,
+	"decreases_over_time":   true,
+	"integrate":             true,
+	"ideriv":                true,
+	"lifetime":              true,
+	"lag":                   true,
+	"scrape_interval":       true,
+	"tmin_over_time":        true,
+	"tmax_over_time":        true,
+	"share_le_over_time":    true,
+	"share_gt_over_time":    true,
+	"histogram_over_time":   true,
+	"rollup":                true,
+	"rollup_rate":           true,
+	"rollup_deriv":          true,
+	"rollup_delta":          true,
+	"rollup_increase":       true,
+	"rollup_candlestick":    true,
+	"aggr_over_time":        true,
+	"hoeffding_bound_upper": true,
+	"hoeffding_bound_lower": true,
+}
+
+// IsRollupFunc returns whether funcName is known rollup function.
+func IsRollupFunc(funcName string) bool {
+	s := strings.ToLower(funcName)
+	return rollupFuncs[s]
+}

lib/metricsql/transform.go

@@ -0,0 +1,85 @@
+package metricsql
+
+import (
+	"strings"
+)
+
+var transformFuncs = map[string]bool{
+	// Standard promql funcs
+	// See funcs accepting instant-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
+	"abs":                true,
+	"absent":             true,
+	"ceil":               true,
+	"clamp_max":          true,
+	"clamp_min":          true,
+	"day_of_month":       true,
+	"day_of_week":        true,
+	"days_in_month":      true,
+	"exp":                true,
+	"floor":              true,
+	"histogram_quantile": true,
+	"hour":               true,
+	"label_join":         true,
+	"label_replace":      true,
+	"ln":                 true,
+	"log2":               true,
+	"log10":              true,
+	"minute":             true,
+	"month":              true,
+	"round":              true,
+	"scalar":             true,
+	"sort":               true,
+	"sort_desc":          true,
+	"sqrt":               true,
+	"time":               true,
+	"timestamp":          true,
+	"vector":             true,
+	"year":               true,
+
+	// New funcs from MetricsQL
+	"label_set":          true,
+	"label_del":          true,
+	"label_keep":         true,
+	"label_copy":         true,
+	"label_move":         true,
+	"label_transform":    true,
+	"label_value":        true,
+	"label_match":        true,
+	"label_mismatch":     true,
+	"union":              true,
+	"":                   true, // empty func is a synonim to union
+	"keep_last_value":    true,
+	"start":              true,
+	"end":                true,
+	"step":               true,
+	"running_sum":        true,
+	"running_max":        true,
+	"running_min":        true,
+	"running_avg":        true,
+	"range_sum":          true,
+	"range_max":          true,
+	"range_min":          true,
+	"range_avg":          true,
+	"range_first":        true,
+	"range_last":         true,
+	"range_quantile":     true,
+	"smooth_exponential": true,
+	"remove_resets":      true,
+	"rand":               true,
+	"rand_normal":        true,
+	"rand_exponential":   true,
+	"pi":                 true,
+	"sin":                true,
+	"cos":                true,
+	"asin":               true,
+	"acos":               true,
+	"prometheus_buckets": true,
+	"histogram_share":    true,
+}
+
+// IsTransformFunc returns whether funcName is known transform function.
+func IsTransformFunc(funcName string) bool {
+	s := strings.ToLower(funcName)
+	return transformFuncs[s]
+
+}

lib/metricsql/utils.go

@@ -0,0 +1,12 @@
+package metricsql
+
+// ExpandWithExprs expands WITH expressions inside q and returns the resulting
+// PromQL without WITH expressions.
+func ExpandWithExprs(q string) (string, error) {
+	e, err := Parse(q)
+	if err != nil {
+		return "", err
+	}
+	buf := e.AppendString(nil)
+	return string(buf), nil
+}

lib/metricsql/utils_example_test.go

@@ -0,0 +1,28 @@
+package metricsql_test
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
+)
+
+func ExampleExpandWithExprs() {
+	// mql can contain arbitrary MetricsQL extensions - see https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/ExtendedPromQL
+	mql := `WITH (
+		commonFilters = {job="$job", instance="$instance"},
+		f(a, b) = 100*(a/b),
+	)
+	f(disk_free_bytes{commonFilters}, disk_total_bytes{commonFilters})`
+
+	// Convert mql to PromQL
+	pql, err := metricsql.ExpandWithExprs(mql)
+	if err != nil {
+		log.Fatalf("cannot expand with expressions: %s", err)
+	}
+
+	fmt.Printf("%s\n", pql)
+
+	// Output:
+	// 100 * (disk_free_bytes{job="$job", instance="$instance"} / disk_total_bytes{job="$job", instance="$instance"})
+}

lib/metricsql/utils_test.go

@@ -0,0 +1,43 @@
+package metricsql
+
+import (
+	"testing"
+)
+
+func TestExpandWithExprsSuccess(t *testing.T) {
+	f := func(q, qExpected string) {
+		t.Helper()
+		for i := 0; i < 3; i++ {
+			qExpanded, err := ExpandWithExprs(q)
+			if err != nil {
+				t.Fatalf("unexpected error when expanding %q: %s", q, err)
+			}
+			if qExpanded != qExpected {
+				t.Fatalf("unexpected expanded expression for %q;\ngot\n%q\nwant\n%q", q, qExpanded, qExpected)
+			}
+		}
+	}
+
+	f(`1`, `1`)
+	f(`foobar`, `foobar`)
+	f(`with (x = 1) x+x`, `2`)
+	f(`with (f(x) = x*x) 3+f(2)+2`, `9`)
+}
+
+func TestExpandWithExprsError(t *testing.T) {
+	f := func(q string) {
+		t.Helper()
+		for i := 0; i < 3; i++ {
+			qExpanded, err := ExpandWithExprs(q)
+			if err == nil {
+				t.Fatalf("expecting non-nil error when expanding %q", q)
+			}
+			if qExpanded != "" {
+				t.Fatalf("unexpected non-empty qExpanded=%q", qExpanded)
+			}
+		}
+	}
+
+	f(``)
+	f(`  with (`)
+}

lib/prompb/util.go

@@ -1,6 +1,7 @@
 package prompb
 
 import (
+	"flag"
 	"fmt"
 	"io"
 
@@ -8,18 +9,20 @@ import (
 	"github.com/golang/snappy"
 )
 
+var maxInsertRequestSize = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size in bytes of a single Prometheus remote_write API request")
+
 // ReadSnappy reads r, unpacks it using snappy, appends it to dst
 // and returns the result.
-func ReadSnappy(dst []byte, r io.Reader, maxSize int64) ([]byte, error) {
-	lr := io.LimitReader(r, maxSize+1)
+func ReadSnappy(dst []byte, r io.Reader) ([]byte, error) {
+	lr := io.LimitReader(r, int64(*maxInsertRequestSize)+1)
 	bb := bodyBufferPool.Get()
 	reqLen, err := bb.ReadFrom(lr)
 	if err != nil {
 		bodyBufferPool.Put(bb)
 		return dst, fmt.Errorf("cannot read compressed request: %s", err)
 	}
-	if reqLen > maxSize {
-		return dst, fmt.Errorf("too big packed request; mustn't exceed %d bytes", maxSize)
+	if reqLen > int64(*maxInsertRequestSize) {
+		return dst, fmt.Errorf("too big packed request; mustn't exceed `-maxInsertRequestSize=%d` bytes", *maxInsertRequestSize)
 	}
 
 	buf := dst[len(dst):cap(dst)]
@@ -29,8 +32,8 @@ func ReadSnappy(dst []byte, r io.Reader, maxSize int64) ([]byte, error) {
 		err = fmt.Errorf("cannot decompress request with length %d: %s", reqLen, err)
 		return dst, err
 	}
-	if int64(len(buf)) > maxSize {
-		return dst, fmt.Errorf("too big unpacked request; musn't exceed %d bytes", maxSize)
+	if len(buf) > *maxInsertRequestSize {
+		return dst, fmt.Errorf("too big unpacked request; mustn't exceed `-maxInsertRequestSize=%d` bytes", *maxInsertRequestSize)
 	}
 	if len(buf) > 0 && len(dst) < cap(dst) && &buf[0] == &dst[len(dst):cap(dst)][0] {
 		dst = dst[:len(dst)+len(buf)]

lib/protoparser/opentsdbhttp/parser_pool.go

@@ -0,0 +1,21 @@
+package opentsdbhttp
+
+import (
+	"github.com/valyala/fastjson"
+)
+
+// GetParser returns JSON parser.
+//
+// The parser must be returned to the pool via PutParser when no longer needed.
+func GetParser() *fastjson.Parser {
+	return parserPool.Get()
+}
+
+// PutParser returns p to the pool.
+//
+// p cannot be used after returning to the pool.
+func PutParser(p *fastjson.Parser) {
+	parserPool.Put(p)
+}
+
+var parserPool fastjson.ParserPool

lib/protoparser/opentsdbhttp/parser_test.go

@@ -9,8 +9,8 @@ func TestRowsUnmarshalFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		var rows Rows
-		p := parserPool.Get()
-		defer parserPool.Put(p)
+		p := GetParser()
+		defer PutParser(p)
 		v, err := p.Parse(s)
 		if err != nil {
 			// Expected JSON parser error
@@ -84,8 +84,8 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		t.Helper()
 		var rows Rows
 
-		p := parserPool.Get()
-		defer parserPool.Put(p)
+		p := GetParser()
+		defer PutParser(p)
 		v, err := p.Parse(s)
 		if err != nil {
 			t.Fatalf("cannot parse json %s: %s", s, err)

lib/protoparser/prometheus/parser.go

@@ -0,0 +1,292 @@
+package prometheus
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson/fastfloat"
+)
+
+// Rows contains parsed Prometheus rows.
+type Rows struct {
+	Rows []Row
+
+	tagsPool []Tag
+}
+
+// Reset resets rs.
+func (rs *Rows) Reset() {
+	// Reset items, so they can be GC'ed
+
+	for i := range rs.Rows {
+		rs.Rows[i].reset()
+	}
+	rs.Rows = rs.Rows[:0]
+
+	for i := range rs.tagsPool {
+		rs.tagsPool[i].reset()
+	}
+	rs.tagsPool = rs.tagsPool[:0]
+}
+
+// Unmarshal unmarshals Prometheus exposition text rows from s.
+//
+// See https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-format-details
+//
+// s must be unchanged until rs is in use.
+func (rs *Rows) Unmarshal(s string) {
+	noEscapes := strings.IndexByte(s, '\\') < 0
+	rs.Rows, rs.tagsPool = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0], noEscapes)
+}
+
+// Row is a single Prometheus row.
+type Row struct {
+	Metric    string
+	Tags      []Tag
+	Value     float64
+	Timestamp int64
+}
+
+func (r *Row) reset() {
+	r.Metric = ""
+	r.Tags = nil
+	r.Value = 0
+	r.Timestamp = 0
+}
+
+func skipLeadingWhitespace(s string) string {
+	// Prometheus treats ' ' and '\t' as whitespace
+	// according to https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-format-details
+	for len(s) > 0 && (s[0] == ' ' || s[0] == '\t') {
+		s = s[1:]
+	}
+	return s
+}
+
+func skipTrailingWhitespace(s string) string {
+	// Prometheus treats ' ' and '\t' as whitespace
+	// according to https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-format-details
+	for len(s) > 0 && (s[len(s)-1] == ' ' || s[len(s)-1] == '\t') {
+		s = s[:len(s)-1]
+	}
+	return s
+}
+
+func nextWhitespace(s string) int {
+	n := strings.IndexByte(s, ' ')
+	if n < 0 {
+		return strings.IndexByte(s, '\t')
+	}
+	n1 := strings.IndexByte(s, '\t')
+	if n1 < 0 || n1 > n {
+		return n
+	}
+	return n1
+}
+
+func (r *Row) unmarshal(s string, tagsPool []Tag, noEscapes bool) ([]Tag, error) {
+	r.reset()
+	s = skipLeadingWhitespace(s)
+	n := strings.IndexByte(s, '{')
+	if n >= 0 {
+		// Tags found. Parse them.
+		r.Metric = skipTrailingWhitespace(s[:n])
+		s = s[n+1:]
+		tagsStart := len(tagsPool)
+		var err error
+		s, tagsPool, err = unmarshalTags(tagsPool, s, noEscapes)
+		if err != nil {
+			return tagsPool, fmt.Errorf("cannot unmarshal tags: %s", err)
+		}
+		if len(s) > 0 && s[0] == ' ' {
+			// Fast path - skip whitespace.
+			s = s[1:]
+		}
+		tags := tagsPool[tagsStart:]
+		r.Tags = tags[:len(tags):len(tags)]
+	} else {
+		// Tags weren't found. Search for value after whitespace
+		n = nextWhitespace(s)
+		if n < 0 {
+			return tagsPool, fmt.Errorf("missing value")
+		}
+		r.Metric = s[:n]
+		s = s[n+1:]
+	}
+	if len(r.Metric) == 0 {
+		return tagsPool, fmt.Errorf("metric cannot be empty")
+	}
+	s = skipLeadingWhitespace(s)
+	if len(s) == 0 {
+		return tagsPool, fmt.Errorf("value cannot be empty")
+	}
+	n = nextWhitespace(s)
+	if n < 0 {
+		// There is no timestamp.
+		r.Value = fastfloat.ParseBestEffort(s)
+		return tagsPool, nil
+	}
+	// There is timestamp.
+	r.Value = fastfloat.ParseBestEffort(s[:n])
+	s = skipLeadingWhitespace(s[n+1:])
+	r.Timestamp = fastfloat.ParseInt64BestEffort(s)
+	return tagsPool, nil
+}
+
+func unmarshalRows(dst []Row, s string, tagsPool []Tag, noEscapes bool) ([]Row, []Tag) {
+	for len(s) > 0 {
+		n := strings.IndexByte(s, '\n')
+		if n < 0 {
+			// The last line.
+			return unmarshalRow(dst, s, tagsPool, noEscapes)
+		}
+		dst, tagsPool = unmarshalRow(dst, s[:n], tagsPool, noEscapes)
+		s = s[n+1:]
+	}
+	return dst, tagsPool
+}
+
+func unmarshalRow(dst []Row, s string, tagsPool []Tag, noEscapes bool) ([]Row, []Tag) {
+	if len(s) > 0 && s[len(s)-1] == '\r' {
+		s = s[:len(s)-1]
+	}
+	s = skipLeadingWhitespace(s)
+	if len(s) == 0 {
+		// Skip empty line
+		return dst, tagsPool
+	}
+	if s[0] == '#' {
+		// Skip comment
+		return dst, tagsPool
+	}
+	if cap(dst) > len(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, Row{})
+	}
+	r := &dst[len(dst)-1]
+	var err error
+	tagsPool, err = r.unmarshal(s, tagsPool, noEscapes)
+	if err != nil {
+		dst = dst[:len(dst)-1]
+		logger.Errorf("cannot unmarshal Prometheus line %q: %s", s, err)
+		invalidLines.Inc()
+	}
+	return dst, tagsPool
+}
+
+var invalidLines = metrics.NewCounter(`vm_rows_invalid_total{type="prometheus"}`)
+
+func unmarshalTags(dst []Tag, s string, noEscapes bool) (string, []Tag, error) {
+	s = skipLeadingWhitespace(s)
+	if len(s) > 0 && s[0] == '}' {
+		// End of tags found.
+		return s[1:], dst, nil
+	}
+	for {
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return s, dst, fmt.Errorf("missing value for tag %q", s)
+		}
+		key := skipTrailingWhitespace(s[:n])
+		s = skipLeadingWhitespace(s[n+1:])
+		if len(s) == 0 || s[0] != '"' {
+			return s, dst, fmt.Errorf("expecting quoted value for tag %q; got %q", key, s)
+		}
+		value := s[1:]
+		if noEscapes {
+			// Fast path - the line has no escape chars
+			n = strings.IndexByte(value, '"')
+			if n < 0 {
+				return s, dst, fmt.Errorf("missing closing quote for tag value %q", s)
+			}
+			s = value[n+1:]
+			value = value[:n]
+		} else {
+			// Slow path - the line contains escape chars
+			n = findClosingQuote(s)
+			if n < 0 {
+				return s, dst, fmt.Errorf("missing closing quote for tag value %q", s)
+			}
+			var err error
+			value, err = unescapeValue(s[:n+1])
+			if err != nil {
+				return s, dst, fmt.Errorf("cannot unescape value %q for tag %q: %s", s[:n+1], key, err)
+			}
+			s = s[n+1:]
+		}
+		if len(key) > 0 && len(value) > 0 {
+			if cap(dst) > len(dst) {
+				dst = dst[:len(dst)+1]
+			} else {
+				dst = append(dst, Tag{})
+			}
+			tag := &dst[len(dst)-1]
+			tag.Key = key
+			tag.Value = value
+		}
+		s = skipLeadingWhitespace(s)
+		if len(s) > 0 && s[0] == '}' {
+			// End of tags found.
+			return s[1:], dst, nil
+		}
+		if len(s) == 0 || s[0] != ',' {
+			return s, dst, fmt.Errorf("missing comma after tag %s=%q", key, value)
+		}
+		s = s[1:]
+	}
+}
+
+// Tag is a Prometheus tag.
+type Tag struct {
+	Key   string
+	Value string
+}
+
+func (t *Tag) reset() {
+	t.Key = ""
+	t.Value = ""
+}
+
+func findClosingQuote(s string) int {
+	if len(s) == 0 || s[0] != '"' {
+		return -1
+	}
+	off := 1
+	s = s[1:]
+	for {
+		n := strings.IndexByte(s, '"')
+		if n < 0 {
+			return -1
+		}
+		if prevBackslashesCount(s[:n])%2 == 0 {
+			return off + n
+		}
+		off += n + 1
+		s = s[n+1:]
+	}
+}
+
+func unescapeValue(s string) (string, error) {
+	if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
+		return "", fmt.Errorf("unexpected tag value: %q", s)
+	}
+	n := strings.IndexByte(s, '\\')
+	if n < 0 {
+		// Fast path - nothing to unescape
+		return s[1 : len(s)-1], nil
+	}
+	return strconv.Unquote(s)
+}
+
+func prevBackslashesCount(s string) int {
+	n := 0
+	for len(s) > 0 && s[len(s)-1] == '\\' {
+		n++
+		s = s[:len(s)-1]
+	}
+	return n
+}

lib/protoparser/prometheus/parser_test.go

@@ -0,0 +1,270 @@
+package prometheus
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestPrevBackslashesCount(t *testing.T) {
+	f := func(s string, nExpected int) {
+		t.Helper()
+		n := prevBackslashesCount(s)
+		if n != nExpected {
+			t.Fatalf("unexpected value returned from prevBackslashesCount(%q); got %d; want %d", s, n, nExpected)
+		}
+	}
+	f(``, 0)
+	f(`foo`, 0)
+	f(`\`, 1)
+	f(`\\`, 2)
+	f(`\\\`, 3)
+	f(`\\\a`, 0)
+	f(`foo\bar`, 0)
+	f(`foo\\`, 2)
+	f(`\\foo\`, 1)
+	f(`\\foo\\\\`, 4)
+}
+
+func TestFindClosingQuote(t *testing.T) {
+	f := func(s string, nExpected int) {
+		t.Helper()
+		n := findClosingQuote(s)
+		if n != nExpected {
+			t.Fatalf("unexpected value returned from findClosingQuote(%q); got %d; want %d", s, n, nExpected)
+		}
+	}
+	f(``, -1)
+	f(`x`, -1)
+	f(`"`, -1)
+	f(`""`, 1)
+	f(`foobar"`, -1)
+	f(`"foo"`, 4)
+	f(`"\""`, 3)
+	f(`"\\"`, 3)
+	f(`"\"`, -1)
+	f(`"foo\"bar\"baz"`, 14)
+}
+
+func TestUnescapeValueFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		ss, err := unescapeValue(s)
+		if err == nil {
+			t.Fatalf("expecting error")
+		}
+		if ss != "" {
+			t.Fatalf("expecting empty string; got %q", ss)
+		}
+	}
+	f(``)
+	f(`foobar`)
+	f(`"foobar`)
+	f(`foobar"`)
+	f(`"foobar\"`)
+	f(` "foobar"`)
+	f(`"foobar" `)
+}
+
+func TestUnescapeValueSuccess(t *testing.T) {
+	f := func(s, resultExpected string) {
+		t.Helper()
+		result, err := unescapeValue(s)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
+		}
+	}
+	f(`""`, "")
+	f(`"f"`, "f")
+	f(`"foobar"`, "foobar")
+	f(`"\"\n\t"`, "\"\n\t")
+}
+
+func TestRowsUnmarshalFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		var rows Rows
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0;\nrows:%#v", len(rows.Rows), rows.Rows)
+		}
+
+		// Try again
+		rows.Unmarshal(s)
+		if len(rows.Rows) != 0 {
+			t.Fatalf("unexpected number of rows parsed; got %d; want 0;\nrows:%#v", len(rows.Rows), rows.Rows)
+		}
+	}
+
+	// Empty lines and comments
+	f("")
+	f(" ")
+	f("\t")
+	f("\t  \r")
+	f("\t\t  \n\n  # foobar")
+	f("#foobar")
+	f("#foobar\n")
+
+	// invalid tags
+	f("a{")
+	f("a { ")
+	f("a {foo")
+	f("a {foo}")
+	f("a {foo  =")
+	f(`a {foo  ="bar`)
+	f(`a {foo  ="b\ar`)
+	f(`a {foo  = "bar"`)
+	f(`a {foo  ="bar",`)
+	f(`a {foo  ="bar" , `)
+	f(`a {foo  ="bar" , }`)
+	f(`a {foo  ="bar" , baz }`)
+
+	// empty metric name
+	f(`{foo="bar"}`)
+
+	// Missing value
+	f("aaa")
+	f(" aaa")
+	f(" aaa ")
+	f(" aaa   \n")
+	f(` aa{foo="bar"}   ` + "\n")
+}
+
+func TestRowsUnmarshalSuccess(t *testing.T) {
+	f := func(s string, rowsExpected *Rows) {
+		t.Helper()
+		var rows Rows
+		rows.Unmarshal(s)
+		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
+			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
+		}
+
+		// Try unmarshaling again
+		rows.Unmarshal(s)
+		if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
+			t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
+		}
+
+		rows.Reset()
+		if len(rows.Rows) != 0 {
+			t.Fatalf("non-empty rows after reset: %+v", rows.Rows)
+		}
+	}
+
+	// Empty line or comment
+	f("", &Rows{})
+	f("\r", &Rows{})
+	f("\n\n", &Rows{})
+	f("\n\r\n", &Rows{})
+	f("\t  \t\n\r\n#foobar\n  # baz", &Rows{})
+
+	// Single line
+	f("foobar 78.9", &Rows{
+		Rows: []Row{{
+			Metric: "foobar",
+			Value:  78.9,
+		}},
+	})
+	f("foobar 123.456 789\n", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     123.456,
+			Timestamp: 789,
+		}},
+	})
+	f("foobar{} 123.456 789\n", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     123.456,
+			Timestamp: 789,
+		}},
+	})
+
+	// Timestamp bigger than 1<<31
+	f("aaa 1123 429496729600", &Rows{
+		Rows: []Row{{
+			Metric:    "aaa",
+			Value:     1123,
+			Timestamp: 429496729600,
+		}},
+	})
+
+	// Tags
+	f(`foo{bar="baz"} 1 2`, &Rows{
+		Rows: []Row{{
+			Metric: "foo",
+			Tags: []Tag{{
+				Key:   "bar",
+				Value: "baz",
+			}},
+			Value:     1,
+			Timestamp: 2,
+		}},
+	})
+	f(`foo{bar="b\"a\\z"} -1.2`, &Rows{
+		Rows: []Row{{
+			Metric: "foo",
+			Tags: []Tag{{
+				Key:   "bar",
+				Value: "b\"a\\z",
+			}},
+			Value: -1.2,
+		}},
+	})
+	// Empty tags
+	f(`foo {bar="baz",aa="",x="y",="z"} 1 2`, &Rows{
+		Rows: []Row{{
+			Metric: "foo",
+			Tags: []Tag{
+				{
+					Key:   "bar",
+					Value: "baz",
+				},
+				{
+					Key:   "x",
+					Value: "y",
+				},
+			},
+			Value:     1,
+			Timestamp: 2,
+		}},
+	})
+
+	// Multi lines
+	f("# foo\n # bar ba zzz\nfoo 0.3 2\naaa 3\nbar.baz 0.34 43\n", &Rows{
+		Rows: []Row{
+			{
+				Metric:    "foo",
+				Value:     0.3,
+				Timestamp: 2,
+			},
+			{
+				Metric: "aaa",
+				Value:  3,
+			},
+			{
+				Metric:    "bar.baz",
+				Value:     0.34,
+				Timestamp: 43,
+			},
+		},
+	})
+
+	// Multi lines with invalid line
+	f("\t foo\t {} 0.3\t 2\naaa\n  bar.baz 0.34 43\n", &Rows{
+		Rows: []Row{
+			{
+				Metric:    "foo",
+				Value:     0.3,
+				Timestamp: 2,
+			},
+			{
+				Metric:    "bar.baz",
+				Value:     0.34,
+				Timestamp: 43,
+			},
+		},
+	})
+}

lib/protoparser/prometheus/parser_timing_test.go

@@ -0,0 +1,25 @@
+package prometheus
+
+import (
+	"fmt"
+	"testing"
+)
+
+func BenchmarkRowsUnmarshal(b *testing.B) {
+	s := `cpu_usage{mode="user"} 1.23
+cpu_usage{mode="system"} 23.344
+cpu_usage{mode="iowait"} 3.3443
+cpu_usage{mode="irq"} 0.34432
+`
+	b.SetBytes(int64(len(s)))
+	b.ReportAllocs()
+	b.RunParallel(func(pb *testing.PB) {
+		var rows Rows
+		for pb.Next() {
+			rows.Unmarshal(s)
+			if len(rows.Rows) != 4 {
+				panic(fmt.Errorf("unexpected number of rows unmarshaled: got %d; want 4", len(rows.Rows)))
+			}
+		}
+	})
+}

lib/storage/block.go

@@ -244,7 +244,10 @@ func (b *Block) UnmarshalData() error {
 	if err != nil {
 		return err
 	}
-	encoding.EnsureNonDecreasingSequence(b.timestamps, b.bh.MinTimestamp, b.bh.MaxTimestamp)
+	if b.bh.PrecisionBits < 64 {
+		// Recover timestamps order after lossy compression.
+		encoding.EnsureNonDecreasingSequence(b.timestamps, b.bh.MinTimestamp, b.bh.MaxTimestamp)
+	}
 	b.timestampsData = b.timestampsData[:0]
 
 	b.values, err = encoding.UnmarshalValues(b.values[:0], b.valuesData, b.bh.ValuesMarshalType, b.bh.FirstValue, int(b.bh.RowsCount))

lib/storage/index_db.go

@@ -1749,7 +1749,7 @@ func (is *indexSearch) updateMetricIDsForTagFilters(metricIDs *uint64set.Set, tf
 		}
 		minMetricIDs = mIDs
 	}
-	metricIDs.Union(minMetricIDs)
+	metricIDs.UnionMayOwn(minMetricIDs)
 	return nil
 }
 
@@ -2068,7 +2068,7 @@ func (is *indexSearch) getMetricIDsForTimeRange(tr TimeRange, maxMetrics int) (*
 			err := isLocal.getMetricIDsForDate(date, &result, maxMetrics)
 			mu.Lock()
 			if metricIDs.Len() < maxMetrics {
-				metricIDs.Union(&result)
+				metricIDs.UnionMayOwn(&result)
 			}
 			if err != nil {
 				errGlobal = err
@@ -2114,7 +2114,7 @@ func (is *indexSearch) tryUpdatingMetricIDsForDateRange(metricIDs *uint64set.Set
 			ok, err := isLocal.tryUpdatingMetricIDsForDate(date, &result, tfs, maxMetrics)
 			mu.Lock()
 			if metricIDs.Len() < maxMetrics {
-				metricIDs.Union(&result)
+				metricIDs.UnionMayOwn(&result)
 			}
 			if !ok {
 				okGlobal = ok
@@ -2203,7 +2203,7 @@ func (is *indexSearch) tryUpdatingMetricIDsForDate(date uint64, metricIDs *uint6
 			return true, nil
 		}
 	}
-	metricIDs.Union(result)
+	metricIDs.UnionMayOwn(result)
 	return true, nil
 }
 

lib/storage/part.go

@@ -5,7 +5,7 @@ import (
 	"path/filepath"
 	"sync"
 	"sync/atomic"
-	"unsafe"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/filestream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -28,7 +28,8 @@ var (
 	maxCachedIndexBlocksPerPartOnce sync.Once
 )
 
-type partInternals struct {
+// part represents a searchable part containing time series data.
+type part struct {
 	ph partHeader
 
 	// Filesystem path to the part.
@@ -44,16 +45,8 @@ type partInternals struct {
 	indexFile      fs.ReadAtCloser
 
 	metaindex []metaindexRow
-}
 
-// part represents a searchable part containing time series data.
-type part struct {
-	partInternals
-
-	// Align ibCache to 8 bytes in order to align internal counters on 32-bit architectures.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
-	_       [(8 - (unsafe.Sizeof(partInternals{}) % 8)) % 8]byte
-	ibCache indexBlockCache
+	ibCache *indexBlockCache
 }
 
 // openFilePart opens file-based part from the given path.
@@ -132,7 +125,7 @@ func newPart(ph *partHeader, path string, size uint64, metaindexReader filestrea
 		return nil, err
 	}
 
-	p.ibCache.Init()
+	p.ibCache = newIndexBlockCache()
 
 	return &p, nil
 }
@@ -152,7 +145,7 @@ func (p *part) MustClose() {
 	p.indexFile.MustClose()
 
 	isBig := p.ph.RowsCount > maxRowsPerSmallPart()
-	p.ibCache.Reset(isBig)
+	p.ibCache.MustClose(isBig)
 }
 
 type indexBlock struct {
@@ -180,19 +173,39 @@ type indexBlockCache struct {
 	requests uint64
 	misses   uint64
 
-	m         map[uint64]*indexBlock
-	missesMap map[uint64]uint64
-	mu        sync.RWMutex
+	m  map[uint64]*indexBlockCacheEntry
+	mu sync.RWMutex
+
+	cleanerStopCh chan struct{}
+	cleanerWG     sync.WaitGroup
 }
 
-func (ibc *indexBlockCache) Init() {
-	ibc.m = make(map[uint64]*indexBlock)
-	ibc.missesMap = make(map[uint64]uint64)
-	ibc.requests = 0
-	ibc.misses = 0
+type indexBlockCacheEntry struct {
+	// Atomically updated counters must go first in the struct, so they are properly
+	// aligned to 8 bytes on 32-bit architectures.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/212
+	lastAccessTime uint64
+
+	ib *indexBlock
 }
 
-func (ibc *indexBlockCache) Reset(isBig bool) {
+func newIndexBlockCache() *indexBlockCache {
+	var ibc indexBlockCache
+	ibc.m = make(map[uint64]*indexBlockCacheEntry)
+
+	ibc.cleanerStopCh = make(chan struct{})
+	ibc.cleanerWG.Add(1)
+	go func() {
+		defer ibc.cleanerWG.Done()
+		ibc.cleaner()
+	}()
+	return &ibc
+}
+
+func (ibc *indexBlockCache) MustClose(isBig bool) {
+	close(ibc.cleanerStopCh)
+	ibc.cleanerWG.Wait()
+
 	if isBig {
 		atomic.AddUint64(&bigIndexBlockCacheRequests, ibc.requests)
 		atomic.AddUint64(&bigIndexBlockCacheMisses, ibc.misses)
@@ -202,10 +215,36 @@ func (ibc *indexBlockCache) Reset(isBig bool) {
 	}
 	// It is safe returning ibc.m itemst to the pool, since Reset must
 	// be called only when no other goroutines access ibc entries.
-	for _, ib := range ibc.m {
-		putIndexBlock(ib)
+	for _, ibe := range ibc.m {
+		putIndexBlock(ibe.ib)
 	}
-	ibc.Init()
+	ibc.m = nil
+}
+
+// cleaner periodically cleans least recently used items.
+func (ibc *indexBlockCache) cleaner() {
+	t := time.NewTimer(5 * time.Second)
+	for {
+		select {
+		case <-t.C:
+			ibc.cleanByTimeout()
+		case <-ibc.cleanerStopCh:
+			t.Stop()
+			return
+		}
+	}
+}
+
+func (ibc *indexBlockCache) cleanByTimeout() {
+	currentTime := atomic.LoadUint64(&currentTimestamp)
+	ibc.mu.Lock()
+	for k, ibe := range ibc.m {
+		// Delete items accessed more than 10 minutes ago.
+		if currentTime-atomic.LoadUint64(&ibe.lastAccessTime) > 10*60 {
+			delete(ibc.m, k)
+		}
+	}
+	ibc.mu.Unlock()
 }
 
 var (
@@ -220,29 +259,23 @@ func (ibc *indexBlockCache) Get(k uint64) *indexBlock {
 	atomic.AddUint64(&ibc.requests, 1)
 
 	ibc.mu.RLock()
-	ib := ibc.m[k]
+	ibe := ibc.m[k]
 	ibc.mu.RUnlock()
 
-	if ib != nil {
-		return ib
+	if ibe != nil {
+		currentTime := atomic.LoadUint64(&currentTimestamp)
+		if atomic.LoadUint64(&ibe.lastAccessTime) != currentTime {
+			atomic.StoreUint64(&ibe.lastAccessTime, currentTime)
+		}
+		return ibe.ib
 	}
 	atomic.AddUint64(&ibc.misses, 1)
-	ibc.mu.Lock()
-	ibc.missesMap[k]++
-	ibc.mu.Unlock()
 	return nil
 }
 
 func (ibc *indexBlockCache) Put(k uint64, ib *indexBlock) bool {
 	ibc.mu.Lock()
 
-	if ibc.missesMap[k] < 2 {
-		// Do not store infrequently accessed ib in the cache,
-		// so it don't evict frequently accessed items.
-		ibc.mu.Unlock()
-		return false
-	}
-
 	// Clean superflouos cache entries.
 	if overflow := len(ibc.m) - getMaxCachedIndexBlocksPerPart(); overflow > 0 {
 		// Remove 10% of items from the cache.
@@ -256,21 +289,13 @@ func (ibc *indexBlockCache) Put(k uint64, ib *indexBlock) bool {
 			}
 		}
 	}
-	if overflow := len(ibc.missesMap) - 8*getMaxCachedIndexBlocksPerPart(); overflow > 0 {
-		// Remove 10% of items from the cache.
-		overflow = int(float64(len(ibc.missesMap)) * 0.1)
-		for k := range ibc.missesMap {
-			delete(ibc.missesMap, k)
-			overflow--
-			if overflow == 0 {
-				break
-			}
-		}
-	}
 
 	// Store frequently requested ib in the cache.
-	delete(ibc.missesMap, k)
-	ibc.m[k] = ib
+	ibe := &indexBlockCacheEntry{
+		lastAccessTime: atomic.LoadUint64(&currentTimestamp),
+		ib:             ib,
+	}
+	ibc.m[k] = ibe
 	ibc.mu.Unlock()
 	return true
 }
@@ -289,3 +314,15 @@ func (ibc *indexBlockCache) Len() uint64 {
 	ibc.mu.Unlock()
 	return n
 }
+
+func init() {
+	go func() {
+		t := time.NewTimer(time.Second)
+		for tm := range t.C {
+			t := uint64(tm.Unix())
+			atomic.StoreUint64(&currentTimestamp, t)
+		}
+	}()
+}
+
+var currentTimestamp uint64

lib/storage/part_search.go

@@ -88,7 +88,7 @@ func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange, fetchData bool)
 	ps.tr = tr
 	ps.fetchData = fetchData
 	ps.metaindex = p.metaindex
-	ps.ibCache = &p.ibCache
+	ps.ibCache = p.ibCache
 
 	// Advance to the first tsid. There is no need in checking
 	// the returned result, since it will be checked in NextBlock.