CHANGELOG.md: cut v1.107.0 release

Comment broken tests for remote_read integration test.
 Prometheus broke library compatibility and it's required to rewrite tests.
 Also, test structure and format should be revisited and improved according to our test code style.

Signed-off-by: f41gh7 <nik@victoriametrics.com>

.github/workflows/main.yml

@@ -88,6 +88,35 @@ jobs:
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           file: ./coverage.txt
+
+  integration-test:
+    name: integration-test
+    needs: [lint, test]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: stable
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
+
+      - name: Run integration tests
+        run: make integration-test

app/victoria-metrics/self_scraper.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -48,7 +48,7 @@ func selfScraper(scrapeInterval time.Duration) {
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
 	var mrs []storage.MetricRow
-	var labels []prompb.Label
+	var labels []prompbmarshal.Label
 	t := time.NewTicker(scrapeInterval)
 	f := func(currentTime time.Time, sendStaleMarkers bool) {
 		currentTimestamp := currentTime.UnixNano() / 1e6
@@ -99,11 +99,11 @@ func selfScraper(scrapeInterval time.Duration) {
 	}
 }
 
-func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
+func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label {
 	if len(dst) < cap(dst) {
 		dst = dst[:len(dst)+1]
 	} else {
-		dst = append(dst, prompb.Label{})
+		dst = append(dst, prompbmarshal.Label{})
 	}
 	lb := &dst[len(dst)-1]
 	lb.Name = key

app/vmagent/main.go

@@ -498,7 +498,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
 		return true
 	}
-	at, err := auth.NewToken(p.AuthToken)
+	at, err := auth.NewTokenPossibleMultitenant(p.AuthToken)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
@@ -510,7 +510,13 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.WriteHeader(http.StatusNoContent)
+		statusCode := http.StatusNoContent
+		if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus/metrics/job/") {
+			// Return 200 status code for pushgateway requests.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
+			statusCode = http.StatusOK
+		}
+		w.WriteHeader(statusCode)
 		return true
 	}
 	if strings.HasPrefix(p.Suffix, "datadog/") {

app/vmagent/remotewrite/remotewrite.go

@@ -99,9 +99,6 @@ var (
 	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
 	rwctxsGlobal []*remoteWriteCtx
 
-	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
-	defaultAuthToken = &auth.Token{}
-
 	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
 	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
 		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
@@ -209,7 +206,7 @@ func Init() {
 
 	initStreamAggrConfigGlobal()
 
-	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	rwctxsGlobal = newRemoteWriteCtxs(*remoteWriteURLs)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -294,7 +291,7 @@ var (
 	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
-func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
+func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
@@ -316,11 +313,6 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
 		}
 		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
-		if at != nil {
-			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
-			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
-		}
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
@@ -411,11 +403,6 @@ func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
 	tss := wr.Timeseries
 
-	if at == nil && MultitenancyEnabled() {
-		// Write data to default tenant if at isn't set when multitenancy is enabled.
-		at = defaultAuthToken
-	}
-
 	var tenantRctx *relabelCtx
 	if at != nil {
 		// Convert at to (vm_account_id, vm_project_id) labels.

app/vmalert-tool/unittest/recording.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/metricsql"
 )
@@ -48,7 +49,7 @@ Outer:
 		}
 		var expSamples []parsedSample
 		for _, s := range mt.ExpSamples {
-			expLb := datasource.Labels{}
+			expLb := []prompbmarshal.Label{}
 			if s.Labels != "" {
 				metricsqlExpr, err := metricsql.Parse(s.Labels)
 				if err != nil {
@@ -64,7 +65,7 @@ Outer:
 				}
 				if len(metricsqlMetricExpr.LabelFilterss) > 0 {
 					for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
-						expLb = append(expLb, datasource.Label{
+						expLb = append(expLb, prompbmarshal.Label{
 							Name:  l.Label,
 							Value: l.Value,
 						})

app/vmalert-tool/unittest/testdata/failed-test-with-missing-rulefile.yaml

@@ -0,0 +1,4 @@
+rule_files:
+  - non-existing-file.yaml
+
+tests: []

app/vmalert-tool/unittest/unittest.go

@@ -74,8 +74,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 		logger.Fatalf("failed to load test files %q: %v", files, err)
 	}
 	if len(testfiles) == 0 {
-		fmt.Println("no test file found")
-		return false
+		logger.Fatalf("no test file found")
 	}
 
 	labels := make(map[string]string)
@@ -97,8 +96,8 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	var failed bool
 	for fileName, file := range testfiles {
 		if err := ruleUnitTest(fileName, file, labels); err != nil {
-			fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", fileName, err)
+			fmt.Println("FAILED")
+			fmt.Printf("failed to run unit test for file %q: \n%v", fileName, err)
 			failed = true
 		} else {
 			fmt.Println("  SUCCESS")
@@ -109,7 +108,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 }
 
 func ruleUnitTest(filename string, content []byte, externalLabels map[string]string) []error {
-	fmt.Println("\nUnit Testing: ", filename)
+	fmt.Println("\n\nUnit Testing: ", filename)
 	var unitTestInp unitTestFile
 	if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
 		return []error{fmt.Errorf("failed to unmarshal file: %w", err)}
@@ -139,6 +138,9 @@ func ruleUnitTest(filename string, content []byte, externalLabels map[string]str
 	if err != nil {
 		return []error{fmt.Errorf("failed to parse `rule_files`: %w", err)}
 	}
+	if len(testGroups) == 0 {
+		return []error{fmt.Errorf("found no rule group in %v", unitTestInp.RuleFiles)}
+	}
 
 	var errs []error
 	for _, t := range unitTestInp.Tests {

app/vmalert-tool/unittest/unittest_test.go

@@ -24,7 +24,8 @@ func TestUnitTest_Failure(t *testing.T) {
 		}
 	}
 
-	// failing test
+	f([]string{"./testdata/failed-test-with-missing-rulefile.yaml"})
+
 	f([]string{"./testdata/failed-test.yaml"})
 }
 

app/vmalert/datasource/client_graphite.go

@@ -46,8 +46,8 @@ const (
 	graphitePrefix = "/graphite"
 )
 
-func (s *Client) setGraphiteReqParams(r *http.Request, query string) {
-	if s.appendTypePrefix {
+func (c *Client) setGraphiteReqParams(r *http.Request, query string) {
+	if c.appendTypePrefix {
 		r.URL.Path += graphitePrefix
 	}
 	r.URL.Path += graphitePath
@@ -58,7 +58,7 @@ func (s *Client) setGraphiteReqParams(r *http.Request, query string) {
 	q.Set("target", query)
 	q.Set("until", "now")
 
-	for k, vs := range s.extraParams {
+	for k, vs := range c.extraParams {
 		if q.Has(k) { // extraParams are prior to params in URL
 			q.Del(k)
 		}

app/vmalert/datasource/client_prom.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 
 	"github.com/valyala/fastjson"
 )
@@ -16,7 +17,8 @@ import (
 var (
 	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' or '/select/logsql/stats_query' path "+
 		"to the configured -datasource.url and -remoteRead.url")
-	disableStepParam = flag.Bool("datasource.disableStepParam", false, "Whether to disable adding 'step' param to the issued instant queries. "+
+	disableStepParam = flag.Bool("datasource.disableStepParam", false, "Whether to disable adding 'step' param in instant queries to the configured -datasource.url and -remoteRead.url. "+
+		"Only valid for prometheus datasource. "+
 		"This might be useful when using vmalert with datasources that do not support 'step' param for instant queries, like Google Managed Prometheus. "+
 		"It is not recommended to enable this flag if you use vmalert with VictoriaMetrics.")
 )
@@ -81,14 +83,14 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 		labels := metric.GetObject()
 
 		r := &pi.ms[i]
-		r.Labels = make([]Label, 0, labels.Len())
+		r.Labels = make([]prompbmarshal.Label, 0, labels.Len())
 		labels.Visit(func(key []byte, v *fastjson.Value) {
 			lv, errLocal := v.StringBytes()
 			if errLocal != nil {
 				err = fmt.Errorf("error when parsing label value %q: %s", v, errLocal)
 				return
 			}
-			r.Labels = append(r.Labels, Label{
+			r.Labels = append(r.Labels, prompbmarshal.Label{
 				Name:  string(key),
 				Value: string(lv),
 			})
@@ -218,8 +220,8 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result
 	return res, nil
 }
 
-func (s *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
-	if s.appendTypePrefix {
+func (c *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+	if c.appendTypePrefix {
 		r.URL.Path += "/prometheus"
 	}
 	if !*disablePathAppend {
@@ -227,22 +229,22 @@ func (s *Client) setPrometheusInstantReqParams(r *http.Request, query string, ti
 	}
 	q := r.URL.Query()
 	q.Set("time", timestamp.Format(time.RFC3339))
-	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
+	if !*disableStepParam && c.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
 	}
-	if !*disableStepParam && s.queryStep > 0 { // override step with user-specified value
+	if !*disableStepParam && c.queryStep > 0 { // override step with user-specified value
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(c.queryStep.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }
 
-func (s *Client) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
-	if s.appendTypePrefix {
+func (c *Client) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
+	if c.appendTypePrefix {
 		r.URL.Path += "/prometheus"
 	}
 	if !*disablePathAppend {
@@ -251,11 +253,11 @@ func (s *Client) setPrometheusRangeReqParams(r *http.Request, query string, star
 	q := r.URL.Query()
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
-	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+	if c.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }

app/vmalert/datasource/client_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var (
@@ -144,12 +145,12 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected := []Metric{
 		{
-			Labels:     []Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
@@ -214,7 +215,7 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	exp := []Metric{
 		{
-			Labels:     []Label{{Value: "constantLine(10)", Name: "name"}},
+			Labels:     []prompbmarshal.Label{{Value: "constantLine(10)", Name: "name"}},
 			Timestamps: []int64{1611758403},
 			Values:     []float64{10},
 		},
@@ -236,12 +237,12 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected = []Metric{
 		{
-			Labels:     []Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
@@ -444,7 +445,7 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected := Metric{
-		Labels:     []Label{{Value: "vm_rows", Name: "__name__"}},
+		Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{13763},
 	}
@@ -475,7 +476,7 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected = Metric{
-		Labels:     []Label{{Value: "total", Name: "stats_result"}},
+		Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{10},
 	}

app/vmalert/datasource/client_vlogs.go

@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-func (s *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+func (c *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
 	if !*disablePathAppend {
 		r.URL.Path += "/select/logsql/stats_query"
@@ -16,15 +16,15 @@ func (s *Client) setVLogsInstantReqParams(r *http.Request, query string, timesta
 	q.Set("time", timestamp.Format(time.RFC3339))
 	// set the `start` and `end` params if applyIntervalAsTimeFilter is enabled(time filter is missing in the rule expr),
 	// so the query will be executed in time range [timestamp - evaluationInterval, timestamp].
-	if s.applyIntervalAsTimeFilter && s.evaluationInterval > 0 {
-		q.Set("start", timestamp.Add(-s.evaluationInterval).Format(time.RFC3339))
+	if c.applyIntervalAsTimeFilter && c.evaluationInterval > 0 {
+		q.Set("start", timestamp.Add(-c.evaluationInterval).Format(time.RFC3339))
 		q.Set("end", timestamp.Format(time.RFC3339))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }
 
-func (s *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) {
+func (c *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) {
 	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
 	if !*disablePathAppend {
 		r.URL.Path += "/select/logsql/stats_query_range"
@@ -33,11 +33,11 @@ func (s *Client) setVLogsRangeReqParams(r *http.Request, query string, start, en
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
 	// set step as evaluationInterval by default
-	if s.evaluationInterval > 0 {
-		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+	if c.evaluationInterval > 0 {
+		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }
 
 func parseVLogsResponse(req *http.Request, resp *http.Response) (res Result, err error) {

app/vmalert/datasource/datasource.go

@@ -8,6 +8,8 @@ import (
 	"sort"
 	"strconv"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Querier interface wraps Query and QueryRange methods
@@ -55,7 +57,7 @@ type QuerierParams struct {
 
 // Metric is the basic entity which should be return by datasource
 type Metric struct {
-	Labels     []Label
+	Labels     []prompbmarshal.Label
 	Timestamps []int64
 	Values     []float64
 }
@@ -72,22 +74,9 @@ func (m *Metric) SetLabel(key, value string) {
 	m.AddLabel(key, value)
 }
 
-// SetLabels sets the given map as Metric labels
-func (m *Metric) SetLabels(ls map[string]string) {
-	var i int
-	m.Labels = make([]Label, len(ls))
-	for k, v := range ls {
-		m.Labels[i] = Label{
-			Name:  k,
-			Value: v,
-		}
-		i++
-	}
-}
-
 // AddLabel appends the given label to the label set
 func (m *Metric) AddLabel(key, value string) {
-	m.Labels = append(m.Labels, Label{Name: key, Value: value})
+	m.Labels = append(m.Labels, prompbmarshal.Label{Name: key, Value: value})
 }
 
 // DelLabel deletes the given label from the label set
@@ -110,14 +99,8 @@ func (m *Metric) Label(key string) string {
 	return ""
 }
 
-// Label represents metric's label
-type Label struct {
-	Name  string
-	Value string
-}
-
 // Labels is collection of Label
-type Labels []Label
+type Labels []prompbmarshal.Label
 
 func (ls Labels) Len() int           { return len(ls) }
 func (ls Labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
@@ -172,7 +155,7 @@ func LabelCompare(a, b Labels) int {
 // ConvertToLabels convert map to Labels
 func ConvertToLabels(m map[string]string) (labelset Labels) {
 	for k, v := range m {
-		labelset = append(labelset, Label{
+		labelset = append(labelset, prompbmarshal.Label{
 			Name:  k,
 			Value: v,
 		})

app/vmalert/datasource/init.go

@@ -11,7 +11,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
@@ -48,22 +47,15 @@ var (
 	oauth2TokenURL = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url")
 	oauth2Scopes   = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
 
-	lookBack = flag.Duration("datasource.lookback", 0, `Deprecated: please adjust "-search.latencyOffset" at datasource side `+
-		`or specify "latency_offset" in rule group's params. Lookback defines how far into the past to look when evaluating queries. `+
-		`For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
-	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
+	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries to the configured -datasource.url and -remoteRead.url. Only valid for prometheus datasource. "+
 		"For example, if -datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query. "+
 		"If set to 0, rule's evaluation interval will be used instead.")
-	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Deprecated: please use "eval_alignment" in rule group instead. `+
-		`Whether to align "time" parameter with evaluation interval. `+
-		"Alignment supposed to produce deterministic results despite number of vmalert replicas or time they were started. "+
-		"See more details at https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections    = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 	idleConnectionTimeout = flag.Duration("datasource.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
 	disableKeepAlive      = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
 		`If true, disables HTTP keep-alive and will only use the connection to the server for a single HTTP request.`)
-	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
-		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
+	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests which limits the number of digits after the decimal point in response values. `+
+		`Only valid for VictoriaMetrics as the datasource.`)
 )
 
 // InitSecretFlags must be called after flag.Parse and before any logging
@@ -90,12 +82,6 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
-	if !*queryTimeAlignment {
-		logger.Warnf("flag `-datasource.queryTimeAlignment` is deprecated and will be removed in next releases. Please use `eval_alignment` in rule group instead.")
-	}
-	if *lookBack != 0 {
-		logger.Warnf("flag `-datasource.lookback` is deprecated and will be removed in next releases. Please adjust `-search.latencyOffset` at datasource side or specify `latency_offset` in rule group's params. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
-	}
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {

app/vmalert/datasource/vm_prom_api_test.go

@@ -3,6 +3,8 @@ package datasource
 import (
 	"reflect"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestPromInstant_UnmarshalPositive(t *testing.T) {
@@ -21,7 +23,7 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 
 	f(`[{"metric":{"__name__":"up"},"value":[1583780000,"42"]}]`, []Metric{
 		{
-			Labels:     []Label{{Name: "__name__", Value: "up"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
@@ -31,17 +33,17 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 {"metric":{"__name__":"foo"},"value":[1583780001,"7"]},
 {"metric":{"__name__":"baz", "instance":"bar"},"value":[1583780002,"8"]}]`, []Metric{
 		{
-			Labels:     []Label{{Name: "__name__", Value: "up"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
 		{
-			Labels:     []Label{{Name: "__name__", Value: "foo"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "foo"}},
 			Timestamps: []int64{1583780001},
 			Values:     []float64{7},
 		},
 		{
-			Labels:     []Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
 			Timestamps: []int64{1583780002},
 			Values:     []float64{8},
 		},

app/vmalert/main.go

@@ -78,8 +78,6 @@ absolute path to all .tpl files in root.
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
 		"In case of conflicts, original labels are kept with prefix `exported_`.")
 
-	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases.")
-
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
@@ -97,10 +95,6 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	if !*remoteReadIgnoreRestoreErrors {
-		logger.Warnf("flag `remoteRead.ignoreRestoreErrors` is deprecated and will be removed in next releases.")
-	}
-
 	err := templates.Load(*ruleTemplatesPath, true)
 	if err != nil {
 		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)

app/vmalert/notifier/alert.go

@@ -167,14 +167,8 @@ type tplData struct {
 	ExternalURL    string
 }
 
-func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.Template, execute bool) error {
-	tpl, err := tmpl.Clone()
-	if err != nil {
-		return fmt.Errorf("error cloning template before parse annotation: %w", err)
-	}
-	// Clone() doesn't copy tpl Options, so we set them manually
-	tpl = tpl.Option("missingkey=zero")
-	tpl, err = tpl.Parse(text)
+func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
+	tpl, err := tpl.Parse(text)
 	if err != nil {
 		return fmt.Errorf("error parsing annotation template: %w", err)
 	}

app/vmalert/notifier/alert_test.go

@@ -33,7 +33,7 @@ func TestAlertExecTemplate(t *testing.T) {
 		qFn := func(_ string) ([]datasource.Metric, error) {
 			return []datasource.Metric{
 				{
-					Labels: []datasource.Label{
+					Labels: []prompbmarshal.Label{
 						{Name: "foo", Value: "bar"},
 						{Name: "baz", Value: "qux"},
 					},
@@ -41,7 +41,7 @@ func TestAlertExecTemplate(t *testing.T) {
 					Timestamps: []int64{1},
 				},
 				{
-					Labels: []datasource.Label{
+					Labels: []prompbmarshal.Label{
 						{Name: "foo", Value: "garply"},
 						{Name: "baz", Value: "fred"},
 					},

app/vmalert/remoteread/init.go

@@ -14,7 +14,7 @@ import (
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect."+
 		"Remote read is used to restore alerts state."+
 		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
 		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+

app/vmalert/remotewrite/client.go

@@ -27,7 +27,7 @@ var defaultConcurrency = cgroup.AvailableCPUs() * 2
 
 const (
 	defaultMaxBatchSize  = 1e4
-	defaultMaxQueueSize  = 1e6
+	defaultMaxQueueSize  = 1e5
 	defaultFlushInterval = 2 * time.Second
 	defaultWriteTimeout  = 30 * time.Second
 )

app/vmalert/rule/alerting.go

@@ -14,8 +14,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertingRule is basic alert entity
@@ -454,13 +456,16 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
 	var numActivePending int
+	var tss []prompbmarshal.TimeSeries
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
 		if _, ok := updated[h]; !ok {
 			if a.State == notifier.StatePending {
-				// alert was in Pending state - it is not
-				// active anymore
+				// alert was in Pending state - it is not active anymore
+				// add stale time series
+				tss = append(tss, pendingAlertStaleTimeSeries(a.Labels, ts.Unix(), true)...)
+
 				delete(ar.alerts, h)
 				ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round")
 				continue
@@ -478,6 +483,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
+					// add stale time series
+					tss = append(tss, firingAlertStaleTimeSeries(a.Labels, ts.Unix())...)
+
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
 					continue
 				}
@@ -489,6 +497,10 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.State = notifier.StateFiring
 			a.Start = ts
 			alertsFired.Inc()
+			if ar.For > 0 {
+				// add stale time series
+				tss = append(tss, pendingAlertStaleTimeSeries(a.Labels, ts.Unix(), false)...)
+			}
 			ar.logDebugf(ts, a, "PENDING => FIRING: %s since becoming active at %v", ts.Sub(a.ActiveAt), a.ActiveAt)
 		}
 	}
@@ -497,7 +509,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		curState.Err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
 		return nil, curState.Err
 	}
-	return ar.toTimeSeries(ts.Unix()), nil
+	return append(tss, ar.toTimeSeries(ts.Unix())...), nil
 }
 
 func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
@@ -522,6 +534,7 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 	return ls, as, nil
 }
 
+// toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts
 func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	for _, a := range ar.alerts {
@@ -601,26 +614,83 @@ func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
+	var labels []prompbmarshal.Label
 	for k, v := range a.Labels {
-		labels[k] = v
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	// __name__ already been dropped, no need to check duplication
+	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	if ol := promrelabel.GetLabelByName(labels, alertStateLabel); ol != nil {
+		ol.Value = a.State.String()
+	} else {
+		labels = append(labels, prompbmarshal.Label{Name: alertStateLabel, Value: a.State.String()})
 	}
-	labels["__name__"] = alertMetricName
-	labels[alertStateLabel] = a.State.String()
 	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
-// alertForToTimeSeries returns a timeseries that represents
+// alertForToTimeSeries returns a time series that represents
 // state of active alerts, where value is time when alert become active
 func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
+	var labels []prompbmarshal.Label
 	for k, v := range a.Labels {
-		labels[k] = v
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
 	}
-	labels["__name__"] = alertForStateMetricName
+	// __name__ already been dropped, no need to check duplication
+	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
 	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
+// pendingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
+// for alerts which changed their state from Pending to Inactive or Firing.
+func pendingAlertStaleTimeSeries(ls map[string]string, timestamp int64, includeAlertForState bool) []prompbmarshal.TimeSeries {
+	var result []prompbmarshal.TimeSeries
+	var baseLabels []prompbmarshal.Label
+	for k, v := range ls {
+		baseLabels = append(baseLabels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	// __name__ already been dropped, no need to check duplication
+	alertsLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StatePending.String()})
+	result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels))
+
+	if includeAlertForState {
+		alertsForStateLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
+		result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsForStateLabels))
+	}
+	return result
+}
+
+// firingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
+// for alerts which changed their state from Firing to Inactive.
+func firingAlertStaleTimeSeries(ls map[string]string, timestamp int64) []prompbmarshal.TimeSeries {
+	var baseLabels []prompbmarshal.Label
+	for k, v := range ls {
+		baseLabels = append(baseLabels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	// __name__ already been dropped, no need to check duplication
+	alertsLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StateFiring.String()})
+
+	alertsForStateLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
+
+	return []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels),
+		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsForStateLabels),
+	}
+}
+
 // restore restores the value of ActiveAt field for active alerts,
 // based on previously written time series `alertForStateMetricName`.
 // Only rules with For > 0 can be restored.
@@ -641,7 +711,8 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 	for k, v := range ar.Labels {
 		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
 	}
-	expr := fmt.Sprintf("last_over_time(%s{%s%s}[%ds])",
+	// use `default_rollup()` instead of `last_over_time()` here to accounts for possible staleness markers
+	expr := fmt.Sprintf("default_rollup(%s{%s%s}[%ds])",
 		alertForStateMetricName, nameStr, labelsFilter, int(lookback.Seconds()))
 
 	res, _, err := q.Query(ctx, expr, ts)

app/vmalert/rule/alerting_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -28,7 +29,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		rule.alerts[alert.ID] = alert
 		tss := rule.toTimeSeries(timestamp.Unix())
 		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries mismatch: %s", err)
+			t.Fatalf("timeseries mismatch for rule %q: %s", rule.Name, err)
 		}
 	}
 
@@ -36,14 +37,23 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":      alertMetricName,
-			alertStateLabel: notifier.StateFiring.String(),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StateFiring.String(),
+			},
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
-				"__name__": alertForStateMetricName,
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: alertForStateMetricName,
+				},
 			}),
 	})
 
@@ -54,18 +64,40 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 			"instance": "bar",
 		},
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":      alertMetricName,
-			alertStateLabel: notifier.StateFiring.String(),
-			"job":           "foo",
-			"instance":      "bar",
-		}),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()},
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: alertMetricName,
+				},
+				{
+					Name:  alertStateLabel,
+					Value: notifier.StateFiring.String(),
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+				{
+					Name:  "instance",
+					Value: "bar",
+				},
+			}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
-				"__name__": alertForStateMetricName,
-				"job":      "foo",
-				"instance": "bar",
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: alertForStateMetricName,
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+				{
+					Name:  "instance",
+					Value: "bar",
+				},
 			}),
 	})
 
@@ -73,18 +105,29 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
 		Labels: map[string]string{
 			alertStateLabel: "foo",
-			"__name__":      "bar",
 		},
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":      alertMetricName,
-			alertStateLabel: notifier.StateFiring.String(),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StateFiring.String(),
+			},
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
-				"__name__":      alertForStateMetricName,
-				alertStateLabel: "foo",
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: alertForStateMetricName,
+				},
+				{
+					Name:  alertStateLabel,
+					Value: "foo",
+				},
 			}),
 	})
 
@@ -92,14 +135,23 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":      alertMetricName,
-			alertStateLabel: notifier.StateFiring.String(),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StateFiring.String(),
+			},
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
-				"__name__": alertForStateMetricName,
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: alertForStateMetricName,
+				},
 			}),
 	})
 
@@ -107,12 +159,21 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StatePending,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":      alertMetricName,
-			alertStateLabel: notifier.StatePending.String(),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StatePending.String(),
+			},
 		}),
-		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": alertForStateMetricName,
+		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: alertForStateMetricName,
+			},
 		}),
 	})
 }
@@ -124,7 +185,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 		alert  *notifier.Alert
 	}
 
-	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert) {
+	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
+
+	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert, tssExpected map[int][]prompbmarshal.TimeSeries) {
 		t.Helper()
 
 		fq := &datasource.FakeQuerier{}
@@ -134,13 +197,19 @@ func TestAlertingRule_Exec(t *testing.T) {
 			Name: "TestRule_Exec",
 		}
 		rule.GroupID = fakeGroup.ID()
-		ts := time.Now()
 		for i, step := range steps {
 			fq.Reset()
 			fq.Add(step...)
-			if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
+			tss, err := rule.exec(context.TODO(), ts, 0)
+			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
+			// check generate time series
+			if _, ok := tssExpected[i]; ok {
+				if err := compareTimeSeries(t, tssExpected[i], tss); err != nil {
+					t.Fatalf("generated time series mismatch for rule %q in step %d: %s", rule.Name, i, err)
+				}
+			}
 
 			// shift the execution timestamp before the next iteration
 			ts = ts.Add(defaultStep)
@@ -174,13 +243,21 @@ func TestAlertingRule_Exec(t *testing.T) {
 		}
 	}
 
-	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil)
+	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil, nil)
 
-	f(newTestAlertingRule("empty labels", 0), [][]datasource.Metric{
+	f(newTestAlertingRule("empty_labels", 0), [][]datasource.Metric{
 		{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 	}, map[int][]testAlert{
 		0: {{alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	},
+		map[int][]prompbmarshal.TimeSeries{
+			0: {
+				{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "empty_labels"}, {Name: "alertstate", Value: "firing"}},
+					Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+				{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "empty_labels"}},
+					Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+			},
+		})
 
 	f(newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -194,6 +271,25 @@ func TestAlertingRule_Exec(t *testing.T) {
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
+		2: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+		},
 	})
 
 	f(newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive=>firing", 0), [][]datasource.Metric{
@@ -210,7 +306,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRule("multiple-firing", 0), [][]datasource.Metric{
 		{
@@ -224,7 +320,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 		},
-	})
+	}, nil)
 
 	// 1: fire first alert
 	// 2: fire second alert, set first inactive
@@ -233,27 +329,57 @@ func TestAlertingRule_Exec(t *testing.T) {
 		{metricWithLabels(t, "name", "foo")},
 		{metricWithLabels(t, "name", "foo1")},
 		{metricWithLabels(t, "name", "foo2")},
-	},
-		map[int][]testAlert{
-			0: {
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-			},
-			1: {
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-			},
-			2: {
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-			},
-		})
+	}, map[int][]testAlert{
+		0: {
+			{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+		},
+		1: {
+			{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+		},
+		2: {
+			{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+		},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			// stale time series for foo, `firing -> inactive`
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			// new time series for foo1
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
+		2: {
+			// stale time series for foo1
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			// new time series for foo2
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo2"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo2"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+		},
+	})
 
 	f(newTestAlertingRule("for-pending", time.Minute), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
 	}, map[int][]testAlert{
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRule("for-fired", defaultStep), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -261,6 +387,22 @@ func TestAlertingRule_Exec(t *testing.T) {
 	}, map[int][]testAlert{
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			// stale time series for `pending -> firing`
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
 	})
 
 	f(newTestAlertingRule("for-pending=>empty", time.Second), [][]datasource.Metric{
@@ -272,6 +414,26 @@ func TestAlertingRule_Exec(t *testing.T) {
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		2: {},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
+		// stale time series for `pending -> inactive`
+		2: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+		},
 	})
 
 	f(newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep), [][]datasource.Metric{
@@ -287,7 +449,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>firing", defaultStep, 0, defaultStep, nil), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -300,7 +462,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>keepfiring=>inactive=>pending=>firing", defaultStep, 0, 2*defaultStep, nil), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -321,7 +483,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		6: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 }
 
 func TestAlertingRuleExecRange(t *testing.T) {
@@ -477,7 +639,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
@@ -523,7 +685,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{
@@ -629,7 +791,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert with state restore
 	ts := time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
 		stateMetric("foo", ts))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
@@ -642,7 +804,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// two rules, two active alerts, one with state restored
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
 		stateMetric("bar", ts))
 	fn(
 		[]config.Rule{
@@ -662,9 +824,9 @@ func TestGroup_Restore(t *testing.T) {
 
 	// two rules, two active alerts, two with state restored
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
 		stateMetric("foo", ts))
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
 		stateMetric("bar", ts))
 	fn(
 		[]config.Rule{
@@ -684,7 +846,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert but wrong state restore
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
 		stateMetric("wrong alert", ts))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
@@ -697,7 +859,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert with labels
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
 		stateMetric("foo", ts, "env", "dev"))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
@@ -710,7 +872,7 @@ func TestGroup_Restore(t *testing.T) {
 
 	// one active alert with restore labels missmatch
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
 		stateMetric("foo", ts, "env", "dev", "team", "foo"))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
@@ -1047,7 +1209,7 @@ func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, kee
 
 func TestAlertingRule_ToLabels(t *testing.T) {
 	metric := datasource.Metric{
-		Labels: []datasource.Label{
+		Labels: []prompbmarshal.Label{
 			{Name: "instance", Value: "0.0.0.0:8800"},
 			{Name: "group", Value: "vmalert"},
 			{Name: "alertname", Value: "ConfigurationReloadFailure"},

app/vmalert/rule/group.go

@@ -8,12 +8,9 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net/url"
-	"strconv"
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-
 	"github.com/cheggaaa/pb/v3"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -21,7 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
@@ -350,10 +346,9 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 	}
 
 	e := &executor{
-		Rw:                       rw,
-		Notifiers:                nts,
-		notifierHeaders:          g.NotifierHeaders,
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		Rw:              rw,
+		Notifiers:       nts,
+		notifierHeaders: g.NotifierHeaders,
 	}
 
 	g.infof("started")
@@ -426,8 +421,6 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 				continue
 			}
 
-			// ensure that staleness is tracked for existing rules only
-			e.purgeStaleSeries(g.Rules)
 			e.notifierHeaders = g.NotifierHeaders
 			g.mu.Unlock()
 
@@ -539,10 +532,9 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 // ExecOnce evaluates all the rules under group for once with given timestamp.
 func (g *Group) ExecOnce(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, evalTS time.Time) chan error {
 	e := &executor{
-		Rw:                       rw,
-		Notifiers:                nts,
-		notifierHeaders:          g.NotifierHeaders,
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		Rw:              rw,
+		Notifiers:       nts,
+		notifierHeaders: g.NotifierHeaders,
 	}
 	if len(g.Rules) < 1 {
 		return nil
@@ -633,13 +625,6 @@ type executor struct {
 	notifierHeaders map[string]string
 
 	Rw remotewrite.RWClient
-
-	previouslySentSeriesToRWMu sync.Mutex
-	// previouslySentSeriesToRW stores series sent to RW on previous iteration
-	// map[ruleID]map[ruleLabels][]prompb.Label
-	// where `ruleID` is ID of the Rule within a Group
-	// and `ruleLabels` is []prompb.Label marshalled to a string
-	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }
 
 // execConcurrently executes rules concurrently if concurrency>1
@@ -706,11 +691,6 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		if err := pushToRW(tss); err != nil {
 			return err
 		}
-
-		staleSeries := e.getStaleSeries(r, tss, ts)
-		if err := pushToRW(staleSeries); err != nil {
-			return err
-		}
 	}
 
 	ar, ok := r.(*AlertingRule)
@@ -737,79 +717,3 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 	wg.Wait()
 	return errGr.Err()
 }
-
-var bbPool bytesutil.ByteBufferPool
-
-// getStaleSeries checks whether there are stale series from previously sent ones.
-func (e *executor) getStaleSeries(r Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
-	bb := bbPool.Get()
-	defer bbPool.Put(bb)
-
-	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
-	for _, ts := range tss {
-		// convert labels to strings, so we can compare with previously sent series
-		bb.B = labelsToString(bb.B, ts.Labels)
-		ruleLabels[string(bb.B)] = ts.Labels
-		bb.Reset()
-	}
-
-	rID := r.ID()
-	var staleS []prompbmarshal.TimeSeries
-	// check whether there are series which disappeared and need to be marked as stale
-	e.previouslySentSeriesToRWMu.Lock()
-	for key, labels := range e.previouslySentSeriesToRW[rID] {
-		if _, ok := ruleLabels[key]; ok {
-			continue
-		}
-		// previously sent series are missing in current series, so we mark them as stale
-		ss := newTimeSeriesPB([]float64{decimal.StaleNaN}, []int64{timestamp.Unix()}, labels)
-		staleS = append(staleS, ss)
-	}
-	// set previous series to current
-	e.previouslySentSeriesToRW[rID] = ruleLabels
-	e.previouslySentSeriesToRWMu.Unlock()
-
-	return staleS
-}
-
-// purgeStaleSeries deletes references in tracked
-// previouslySentSeriesToRW list to Rules which aren't present
-// in the given activeRules list. The method is used when the list
-// of loaded rules has changed and executor has to remove
-// references to non-existing rules.
-func (e *executor) purgeStaleSeries(activeRules []Rule) {
-	newPreviouslySentSeriesToRW := make(map[uint64]map[string][]prompbmarshal.Label)
-
-	e.previouslySentSeriesToRWMu.Lock()
-
-	for _, rule := range activeRules {
-		id := rule.ID()
-		prev, ok := e.previouslySentSeriesToRW[id]
-		if ok {
-			// keep previous series for staleness detection
-			newPreviouslySentSeriesToRW[id] = prev
-		}
-	}
-	e.previouslySentSeriesToRW = nil
-	e.previouslySentSeriesToRW = newPreviouslySentSeriesToRW
-
-	e.previouslySentSeriesToRWMu.Unlock()
-}
-
-func labelsToString(dst []byte, labels []prompbmarshal.Label) []byte {
-	dst = append(dst, '{')
-	for i, label := range labels {
-		if len(label.Name) == 0 {
-			dst = append(dst, "__name__"...)
-		} else {
-			dst = append(dst, label.Name...)
-		}
-		dst = append(dst, '=')
-		dst = strconv.AppendQuote(dst, label.Value)
-		if i < len(labels)-1 {
-			dst = append(dst, ',')
-		}
-	}
-	dst = append(dst, '}')
-	return dst
-}

app/vmalert/rule/group_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"reflect"
 	"sort"
 	"testing"
 	"time"
@@ -17,8 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -383,153 +380,6 @@ func TestGetResolveDuration(t *testing.T) {
 	f(2*time.Minute, 0, 1*time.Minute, 8*time.Minute)
 }
 
-func TestGetStaleSeries(t *testing.T) {
-	ts := time.Now()
-	e := &executor{
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-	}
-	f := func(r Rule, labels, expLabels [][]prompbmarshal.Label) {
-		t.Helper()
-
-		var tss []prompbmarshal.TimeSeries
-		for _, l := range labels {
-			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
-		}
-		staleS := e.getStaleSeries(r, tss, ts)
-		if staleS == nil && expLabels == nil {
-			return
-		}
-		if len(staleS) != len(expLabels) {
-			t.Fatalf("expected to get %d stale series, got %d",
-				len(expLabels), len(staleS))
-		}
-		for i, exp := range expLabels {
-			got := staleS[i]
-			if !reflect.DeepEqual(exp, got.Labels) {
-				t.Fatalf("expected to get labels: \n%v;\ngot instead: \n%v",
-					exp, got.Labels)
-			}
-			if len(got.Samples) != 1 {
-				t.Fatalf("expected to have 1 sample; got %d", len(got.Samples))
-			}
-			if !decimal.IsStaleNaN(got.Samples[0].Value) {
-				t.Fatalf("expected sample value to be %v; got %v", decimal.StaleNaN, got.Samples[0].Value)
-			}
-		}
-	}
-
-	// warn: keep in mind, that executor holds the state, so sequence of f calls matters
-
-	// single series
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		nil,
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
-	f(&AlertingRule{RuleID: 1},
-		nil,
-		nil)
-
-	// multiple series
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{
-			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
-			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
-		},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		nil,
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")})
-
-	// multiple rules and series
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{
-			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
-			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
-		},
-		nil)
-	f(&AlertingRule{RuleID: 2},
-		[][]prompbmarshal.Label{
-			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
-			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
-		},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		nil)
-}
-
-func TestPurgeStaleSeries(t *testing.T) {
-	ts := time.Now()
-	labels := toPromLabels(t, "__name__", "job:foo", "job", "foo")
-	tss := []prompbmarshal.TimeSeries{newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels)}
-
-	f := func(curRules, newRules, expStaleRules []Rule) {
-		t.Helper()
-		e := &executor{
-			previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-		}
-		// seed executor with series for
-		// current rules
-		for _, rule := range curRules {
-			e.getStaleSeries(rule, tss, ts)
-		}
-
-		e.purgeStaleSeries(newRules)
-
-		if len(e.previouslySentSeriesToRW) != len(expStaleRules) {
-			t.Fatalf("expected to get %d stale series, got %d",
-				len(expStaleRules), len(e.previouslySentSeriesToRW))
-		}
-
-		for _, exp := range expStaleRules {
-			if _, ok := e.previouslySentSeriesToRW[exp.ID()]; !ok {
-				t.Fatalf("expected to have rule %d; got nil instead", exp.ID())
-			}
-		}
-	}
-
-	f(nil, nil, nil)
-	f(
-		nil,
-		[]Rule{&AlertingRule{RuleID: 1}},
-		nil,
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}},
-		nil,
-		nil,
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}},
-		[]Rule{&AlertingRule{RuleID: 2}},
-		nil,
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 2}},
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-	)
-}
-
 func TestFaultyNotifier(t *testing.T) {
 	fq := &datasource.FakeQuerier{}
 	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
@@ -580,8 +430,7 @@ func TestFaultyRW(t *testing.T) {
 	}
 
 	e := &executor{
-		Rw:                       &remotewrite.Client{},
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		Rw: &remotewrite.Client{},
 	}
 
 	err := e.exec(context.Background(), r, time.Now(), 0, 10)

app/vmalert/rule/group_timing_test.go

@@ -1,36 +0,0 @@
-package rule
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-)
-
-func BenchmarkGetStaleSeries(b *testing.B) {
-	ts := time.Now()
-	n := 100
-	payload := make([]prompbmarshal.TimeSeries, 0, n)
-	for i := 0; i < n; i++ {
-		s := fmt.Sprintf("%d", i)
-		labels := toPromLabels(b,
-			"__name__", "foo", ""+
-				"instance", s,
-			"job", s,
-			"state", s,
-		)
-		payload = append(payload, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels))
-	}
-
-	e := &executor{
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-	}
-	ar := &AlertingRule{RuleID: 1}
-
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		e.getStaleSeries(ar, payload, ts)
-	}
-}

app/vmalert/rule/recording.go

@@ -3,16 +3,17 @@ package rule
 import (
 	"context"
 	"fmt"
-	"sort"
 	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // RecordingRule is a Rule that supposed
@@ -34,6 +35,8 @@ type RecordingRule struct {
 	// during evaluations
 	state *ruleState
 
+	lastEvaluation map[string]struct{}
+
 	metrics *recordingRuleMetrics
 }
 
@@ -113,7 +116,7 @@ func (rr *RecordingRule) execRange(ctx context.Context, start, end time.Time) ([
 	var tss []prompbmarshal.TimeSeries
 	for _, s := range res.Data {
 		ts := rr.toTimeSeries(s)
-		key := stringifyLabels(ts)
+		key := stringifyLabels(ts.Labels)
 		if _, ok := duplicates[key]; ok {
 			return nil, fmt.Errorf("original metric %v; resulting labels %q: %w", s.Labels, key, errDuplicate)
 		}
@@ -155,28 +158,47 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 		return nil, curState.Err
 	}
 
-	duplicates := make(map[string]struct{}, len(qMetrics))
+	curEvaluation := make(map[string]struct{}, len(qMetrics))
+	lastEvaluation := rr.lastEvaluation
 	var tss []prompbmarshal.TimeSeries
 	for _, r := range qMetrics {
 		ts := rr.toTimeSeries(r)
-		key := stringifyLabels(ts)
-		if _, ok := duplicates[key]; ok {
+		key := stringifyLabels(ts.Labels)
+		if _, ok := curEvaluation[key]; ok {
 			curState.Err = fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
 			return nil, curState.Err
 		}
-		duplicates[key] = struct{}{}
+		curEvaluation[key] = struct{}{}
+		delete(lastEvaluation, key)
 		tss = append(tss, ts)
 	}
+	// check for stale time series
+	for k := range lastEvaluation {
+		tss = append(tss, prompbmarshal.TimeSeries{
+			Labels: stringToLabels(k),
+			Samples: []prompbmarshal.Sample{
+				{Value: decimal.StaleNaN, Timestamp: ts.UnixNano() / 1e6},
+			}})
+	}
+	rr.lastEvaluation = curEvaluation
 	return tss, nil
 }
 
-func stringifyLabels(ts prompbmarshal.TimeSeries) string {
-	labels := ts.Labels
-	if len(labels) > 1 {
-		sort.Slice(labels, func(i, j int) bool {
-			return labels[i].Name < labels[j].Name
-		})
+func stringToLabels(s string) []prompbmarshal.Label {
+	labels := strings.Split(s, ",")
+	rLabels := make([]prompbmarshal.Label, 0, len(labels))
+	for i := range labels {
+		if label := strings.Split(labels[i], "="); len(label) == 2 {
+			rLabels = append(rLabels, prompbmarshal.Label{
+				Name:  label[0],
+				Value: label[1],
+			})
+		}
 	}
+	return rLabels
+}
+
+func stringifyLabels(labels []prompbmarshal.Label) string {
 	b := strings.Builder{}
 	for i, l := range labels {
 		b.WriteString(l.Name)
@@ -190,19 +212,27 @@ func stringifyLabels(ts prompbmarshal.TimeSeries) string {
 }
 
 func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
-	for _, l := range m.Labels {
-		labels[l.Name] = l.Value
+	if preN := promrelabel.GetLabelByName(m.Labels, "__name__"); preN != nil {
+		preN.Value = rr.Name
+	} else {
+		m.Labels = append(m.Labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: rr.Name,
+		})
 	}
-	labels["__name__"] = rr.Name
-	// override existing labels with configured ones
-	for k, v := range rr.Labels {
-		if _, ok := labels[k]; ok && labels[k] != v {
-			labels[fmt.Sprintf("exported_%s", k)] = labels[k]
+	for k := range rr.Labels {
+		prevLabel := promrelabel.GetLabelByName(m.Labels, k)
+		if prevLabel != nil && prevLabel.Value != rr.Labels[k] {
+			// Rename the prevLabel to "exported_" + label.Name
+			prevLabel.Name = fmt.Sprintf("exported_%s", prevLabel.Name)
 		}
-		labels[k] = v
+		m.Labels = append(m.Labels, prompbmarshal.Label{
+			Name:  k,
+			Value: rr.Labels[k],
+		})
 	}
-	return newTimeSeries(m.Values, m.Timestamps, labels)
+	ts := newTimeSeries(m.Values, m.Timestamps, m.Labels)
+	return ts
 }
 
 // updateWith copies all significant fields.

app/vmalert/rule/recording_test.go

@@ -9,59 +9,131 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestRecordingRule_Exec(t *testing.T) {
-	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
+	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
+	const defaultStep = 5 * time.Millisecond
+
+	f := func(rule *RecordingRule, steps [][]datasource.Metric, tssExpected [][]prompbmarshal.TimeSeries) {
 		t.Helper()
 
 		fq := &datasource.FakeQuerier{}
-		fq.Add(metrics...)
-		rule.q = fq
-		rule.state = &ruleState{
-			entries: make([]StateEntry, 10),
-		}
-		tss, err := rule.exec(context.TODO(), time.Now(), 0)
-		if err != nil {
-			t.Fatalf("unexpected RecordingRule.exec error: %s", err)
-		}
-		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries missmatch: %s", err)
+		for i, step := range steps {
+			fq.Reset()
+			fq.Add(step...)
+			rule.q = fq
+			rule.state = &ruleState{
+				entries: make([]StateEntry, 10),
+			}
+			tss, err := rule.exec(context.TODO(), ts, 0)
+			if err != nil {
+				t.Fatalf("fail to test rule %s: unexpected error: %s", rule.Name, err)
+			}
+			if err := compareTimeSeries(t, tssExpected[i], tss); err != nil {
+				t.Fatalf("fail to test rule %s: time series mismatch on step %d: %s", rule.Name, i, err)
+			}
+
+			ts = ts.Add(defaultStep)
 		}
 	}
 
-	timestamp := time.Now()
-
 	f(&RecordingRule{
 		Name: "foo",
-	}, []datasource.Metric{
+	}, [][]datasource.Metric{{
 		metricWithValueAndLabels(t, 10, "__name__", "bar"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foo",
+	}}, [][]prompbmarshal.TimeSeries{{
+		newTimeSeries([]float64{10}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "foo",
+			},
 		}),
-	})
+	}})
 
 	f(&RecordingRule{
 		Name: "foobarbaz",
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
-		metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
-		metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "foo",
-		}),
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "bar",
-		}),
-		newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "baz",
-		}),
+	}, [][]datasource.Metric{
+		{
+			metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
+			metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
+		},
+		{
+			metricWithValueAndLabels(t, 10, "__name__", "foo", "job", "foo"),
+		},
+		{
+			metricWithValueAndLabels(t, 10, "__name__", "foo", "job", "bar"),
+		},
+	}, [][]prompbmarshal.TimeSeries{
+		{
+			newTimeSeries([]float64{1}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+			}),
+			newTimeSeries([]float64{2}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+			}),
+		},
+		{
+			newTimeSeries([]float64{10}, []int64{ts.Add(defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+			}),
+			// stale time series
+			newTimeSeries([]float64{decimal.StaleNaN}, []int64{ts.Add(defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+			}),
+		},
+		{
+			newTimeSeries([]float64{10}, []int64{ts.Add(2 * defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+			}),
+			newTimeSeries([]float64{decimal.StaleNaN}, []int64{ts.Add(2 * defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+			}),
+		},
 	})
 
 	f(&RecordingRule{
@@ -69,22 +141,44 @@ func TestRecordingRule_Exec(t *testing.T) {
 		Labels: map[string]string{
 			"source": "test",
 		},
-	}, []datasource.Metric{
+	}, [][]datasource.Metric{{
 		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "foo",
-			"source":   "test",
+	}}, [][]prompbmarshal.TimeSeries{{
+		newTimeSeries([]float64{2}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "job:foo",
+			},
+			{
+				Name:  "job",
+				Value: "foo",
+			},
+			{
+				Name:  "source",
+				Value: "test",
+			},
 		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":        "job:foo",
-			"job":             "bar",
-			"source":          "test",
-			"exported_source": "origin",
-		}),
-	})
+		newTimeSeries([]float64{1}, []int64{ts.UnixNano()},
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "job:foo",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+				{
+					Name:  "source",
+					Value: "test",
+				},
+				{
+					Name:  "exported_source",
+					Value: "origin",
+				},
+			}),
+	}})
 }
 
 func TestRecordingRule_ExecRange(t *testing.T) {
@@ -110,9 +204,13 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValuesAndLabels(t, []float64{10, 20, 30}, "__name__", "bar"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-			"__name__": "foo",
-		}),
+		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foo",
+				},
+			}),
 	})
 
 	f(&RecordingRule{
@@ -122,18 +220,36 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
 		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "foo",
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "foobarbaz",
+			},
+			{
+				Name:  "job",
+				Value: "foo",
+			},
 		}),
-		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "bar",
+		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "foobarbaz",
+			},
+			{
+				Name:  "job",
+				Value: "bar",
+			},
 		}),
 		newTimeSeries([]float64{4, 5, 6},
-			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-				"__name__": "foobarbaz",
-				"job":      "baz",
+			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "baz",
+				},
 			}),
 	})
 
@@ -146,16 +262,35 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "foo",
-			"source":   "test",
-		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "bar",
-			"source":   "test",
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
+			{
+				Name:  "__name__",
+				Value: "job:foo",
+			},
+			{
+				Name:  "job",
+				Value: "foo",
+			},
+			{
+				Name:  "source",
+				Value: "test",
+			},
 		}),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()},
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "job:foo",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+				{
+					Name:  "source",
+					Value: "test",
+				},
+			}),
 	})
 }
 

app/vmalert/rule/test_helpers.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -87,7 +88,7 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	}
 	m := datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}
 	for i := 0; i < len(labels); i += 2 {
-		m.Labels = append(m.Labels, datasource.Label{
+		m.Labels = append(m.Labels, prompbmarshal.Label{
 			Name:  labels[i],
 			Value: labels[i+1],
 		})
@@ -95,21 +96,6 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
-func toPromLabels(t testing.TB, labels ...string) []prompbmarshal.Label {
-	t.Helper()
-	if len(labels) == 0 || len(labels)%2 != 0 {
-		t.Fatalf("expected to get even number of labels")
-	}
-	var ls []prompbmarshal.Label
-	for i := 0; i < len(labels); i += 2 {
-		ls = append(ls, prompbmarshal.Label{
-			Name:  labels[i],
-			Value: labels[i+1],
-		})
-	}
-	return ls
-}
-
 func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 	t.Helper()
 	if len(a) != len(b) {
@@ -122,7 +108,7 @@ func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 		}
 		for i, exp := range expTS.Samples {
 			got := gotTS.Samples[i]
-			if got.Value != exp.Value {
+			if got.Value != exp.Value && (!decimal.IsStaleNaN(got.Value) || !decimal.IsStaleNaN(exp.Value)) {
 				return fmt.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
 			}
 			// timestamp validation isn't always correct for now.

app/vmalert/rule/utils.go

@@ -9,10 +9,14 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-func newTimeSeries(values []float64, timestamps []int64, labels map[string]string) prompbmarshal.TimeSeries {
+// newTimeSeries first sorts given labels, then returns new time series.
+func newTimeSeries(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
+	promrelabel.SortLabels(labels)
 	ts := prompbmarshal.TimeSeries{
+		Labels:  labels,
 		Samples: make([]prompbmarshal.Sample, len(values)),
 	}
 	for i := range values {
@@ -21,34 +25,6 @@ func newTimeSeries(values []float64, timestamps []int64, labels map[string]strin
 			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
 		}
 	}
-	keys := make([]string, 0, len(labels))
-	for k := range labels {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys) // make order deterministic
-	for _, key := range keys {
-		ts.Labels = append(ts.Labels, prompbmarshal.Label{
-			Name:  key,
-			Value: labels[key],
-		})
-	}
-	return ts
-}
-
-// newTimeSeriesPB creates prompbmarshal.TimeSeries with given
-// values, timestamps and labels.
-// It expects that labels are already sorted.
-func newTimeSeriesPB(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
-	ts := prompbmarshal.TimeSeries{
-		Samples: make([]prompbmarshal.Sample, len(values)),
-	}
-	for i := range values {
-		ts.Samples[i] = prompbmarshal.Sample{
-			Value:     values[i],
-			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
-		}
-	}
-	ts.Labels = labels
 	return ts
 }
 

app/vmalert/templates/template.go

@@ -169,6 +169,8 @@ func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Clone() doesn't copy tpl Options, so we set them manually
+	tmpl = tmpl.Option("missingkey=zero")
 	return tmpl.Funcs(funcs), nil
 }
 

app/vmauth/auth_config.go

@@ -67,6 +67,7 @@ type UserInfo struct {
 	URLPrefix              *URLPrefix  `yaml:"url_prefix,omitempty"`
 	DiscoverBackendIPs     *bool       `yaml:"discover_backend_ips,omitempty"`
 	URLMaps                []URLMap    `yaml:"url_map,omitempty"`
+	DumpRequestOnErrors    bool        `yaml:"dump_request_on_errors,omitempty"`
 	HeadersConf            HeadersConf `yaml:",inline"`
 	MaxConcurrentRequests  int         `yaml:"max_concurrent_requests,omitempty"`
 	DefaultURL             *URLPrefix  `yaml:"default_url,omitempty"`
@@ -462,17 +463,12 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 
 	// Slow path - select other backend urls.
 	n := atomicCounter.Add(1) - 1
-	buMin := bus[n%uint32(len(bus))]
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
 		bu := bus[idx]
 		if bu.isBroken() {
 			continue
 		}
-		if buMin.isBroken() {
-			// verify that buMin isn't set as broken
-			buMin = bu
-		}
 		if bu.concurrentRequests.Load() == 0 {
 			// Fast path - return the backend with zero concurrently executed requests.
 			// Do not use CompareAndSwap() instead of Load(), since it is much slower on systems with many CPU cores.
@@ -482,12 +478,13 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	}
 
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
+	buMin := bus[n%uint32(len(bus))]
 	minRequests := buMin.concurrentRequests.Load()
 	for _, bu := range bus {
 		if bu.isBroken() {
 			continue
 		}
-		if n := bu.concurrentRequests.Load(); n < minRequests {
+		if n := bu.concurrentRequests.Load(); n < minRequests || buMin.isBroken() {
 			buMin = bu
 			minRequests = n
 		}
@@ -864,22 +861,23 @@ func (ui *UserInfo) initURLs() error {
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
 	dropSrcPathPrefixParts := 0
 	discoverBackendIPs := *discoverBackendIPsGlobal
+	if ui.RetryStatusCodes != nil {
+		retryStatusCodes = ui.RetryStatusCodes
+	}
+	if ui.LoadBalancingPolicy != "" {
+		loadBalancingPolicy = ui.LoadBalancingPolicy
+	}
+	if ui.DropSrcPathPrefixParts != nil {
+		dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts
+	}
+	if ui.DiscoverBackendIPs != nil {
+		discoverBackendIPs = *ui.DiscoverBackendIPs
+	}
+
 	if ui.URLPrefix != nil {
 		if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil {
 			return err
 		}
-		if ui.RetryStatusCodes != nil {
-			retryStatusCodes = ui.RetryStatusCodes
-		}
-		if ui.LoadBalancingPolicy != "" {
-			loadBalancingPolicy = ui.LoadBalancingPolicy
-		}
-		if ui.DropSrcPathPrefixParts != nil {
-			dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts
-		}
-		if ui.DiscoverBackendIPs != nil {
-			discoverBackendIPs = *ui.DiscoverBackendIPs
-		}
 		ui.URLPrefix.retryStatusCodes = retryStatusCodes
 		ui.URLPrefix.dropSrcPathPrefixParts = dropSrcPathPrefixParts
 		ui.URLPrefix.discoverBackendIPs = discoverBackendIPs

app/vmauth/example_config_ent.yml

@@ -22,26 +22,26 @@ users:
   #  - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path
   #
   # Regular expressions are allowed in `src_paths` entries.
-- username: "foobar"
-  url_map:
-  - src_paths:
-    - "/api/v1/query"
-    - "/api/v1/query_range"
-    - "/api/v1/label/[^/]+/values"
-    url_prefix:
-    - "http://vmselect1:8481/select/42/prometheus"
-    - "http://vmselect2:8481/select/42/prometheus"
-  - src_paths: ["/api/v1/write"]
-    url_prefix: "http://vminsert:8480/insert/42/prometheus"
-    headers:
-    - "X-Scope-OrgID: abc"
+  - username: "foobar"
     ip_filters:
       deny_list: [127.0.0.1]
-  default_url:
-  - "http://default1:8888/unsupported_url_handler"
-  - "http://default2:8888/unsupported_url_handler"
+    url_map:
+      - src_paths:
+          - "/api/v1/query"
+          - "/api/v1/query_range"
+          - "/api/v1/label/[^/]+/values"
+        url_prefix:
+          - "http://vmselect1:8481/select/42/prometheus"
+          - "http://vmselect2:8481/select/42/prometheus"
+      - src_paths: ["/api/v1/write"]
+        url_prefix: "http://vminsert:8480/insert/42/prometheus"
+        headers:
+          - "X-Scope-OrgID: abc"
+    default_url:
+      - "http://default1:8888/unsupported_url_handler"
+      - "http://default2:8888/unsupported_url_handler"
 
 ip_filters:
   allow_list: ["1.2.3.0/24", "127.0.0.1"]
   deny_list:
-  - 10.1.0.1
+    - 10.1.0.1

app/vmauth/main.go

@@ -61,6 +61,9 @@ var (
 		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
 	backendTLSServerName = flag.String("backend.TLSServerName", "", "Optional TLS ServerName, which must be sent to HTTPS backend. "+
 		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
+	dryRun                   = flag.Bool("dryRun", false, "Whether to check only config files without running vmauth. The auth configuration file is validated. The -auth.config flag must be specified.")
+	removeXFFHTTPHeaderValue = flag.Bool(`removeXFFHTTPHeaderValue`, false, "Whether to remove the X-Forwarded-For HTTP header value from client requests before forwarding them to the backend. "+
+		"Recommended when vmauth is exposed to the internet.")
 )
 
 func main() {
@@ -71,6 +74,16 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
+	if *dryRun {
+		if len(*authConfigPath) == 0 {
+			logger.Fatalf("missing required `-auth.config` command-line flag")
+		}
+		if _, err := reloadAuthConfig(); err != nil {
+			logger.Fatalf("failed to parse %q: %s", *authConfigPath, err)
+		}
+		return
+	}
+
 	listenAddrs := *httpListenAddrs
 	if len(listenAddrs) == 0 {
 		listenAddrs = []string{":8427"}
@@ -123,6 +136,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	ui := getUserInfoByAuthTokens(ats)
 	if ui == nil {
+		uu := authConfig.Load().UnauthorizedUser
+		if uu != nil {
+			processUserRequest(w, r, uu)
+			return true
+		}
+
 		invalidAuthTokenRequests.Inc()
 		if *logInvalidAuthTokens {
 			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
@@ -192,7 +211,11 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 				return
 			}
 			missingRouteRequests.Inc()
-			httpserver.Errorf(w, r, "missing route for %s", u.String())
+			var di string
+			if ui.DumpRequestOnErrors {
+				di = debugInfo(u, r.Header)
+			}
+			httpserver.Errorf(w, r, "missing route for %q%s", u.String(), di)
 			return
 		}
 		up, hc = ui.DefaultURL, ui.HeadersConf
@@ -371,7 +394,7 @@ func sanitizeRequestHeaders(r *http.Request) *http.Request {
 		// X-Forwarded-For information as a comma+space
 		// separated list and fold multiple headers into one.
 		prior := req.Header["X-Forwarded-For"]
-		if len(prior) > 0 {
+		if len(prior) > 0 && !*removeXFFHTTPHeaderValue {
 			clientIP = strings.Join(prior, ", ") + ", " + clientIP
 		}
 		req.Header.Set("X-Forwarded-For", clientIP)
@@ -644,3 +667,14 @@ func (rtb *readTrackingBody) Close() error {
 
 	return nil
 }
+
+func debugInfo(u *url.URL, h http.Header) string {
+	s := &strings.Builder{}
+	fmt.Fprintf(s, " (host: %q; ", u.Host)
+	fmt.Fprintf(s, "path: %q; ", u.Path)
+	fmt.Fprintf(s, "args: %q; ", u.Query().Encode())
+	fmt.Fprint(s, "headers:")
+	_ = h.WriteSubset(s, nil)
+	fmt.Fprint(s, ")")
+	return s.String()
+}

app/vmauth/main_test.go

@@ -90,6 +90,20 @@ User-Agent: vmauth
 X-Forwarded-For: 12.34.56.78, 42.2.3.84`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
+	// routing of all failed to authorize requests to unauthorized_user (issue #7543)
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo"
+  keep_original_host: true`
+	requestURL = "http://foo:invalid-secret@some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url=http://some-host.com/foo/abc/def`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
 	// keep_original_host
 	cfgStr = `
 unauthorized_user:
@@ -346,7 +360,27 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=400
-remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for http://some-host.com/abc?de=fg`
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for "http://some-host.com/abc?de=fg"`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// missing default_url and default url_prefix for unauthorized user with dump_request_on_errors enabled
+	cfgStr = `
+unauthorized_user:
+  dump_request_on_errors: true
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=400
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
+Pass-Header: abc
+Some-Header: foobar
+X-Forwarded-For: 12.34.56.78
+)`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
 	// missing default_url and default url_prefix for unauthorized user when there are configs for authorized users

app/vmauth/target_url_test.go

@@ -187,6 +187,10 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 				RetryStatusCodes:       []int{},
 				DropSrcPathPrefixParts: intp(0),
 			},
+			{
+				SrcPaths:  getRegexs([]string{"/metrics"}),
+				URLPrefix: mustParseURL("http://metrics-server"),
+			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
 		HeadersConf: HeadersConf{
@@ -206,6 +210,35 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		"bb: aaa", "x: y", []int{502}, "least_loaded", 2)
 	f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
 	f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", "bb: aaa", "x: y", []int{502}, "least_loaded", 2)
+	f(ui, "https://foo-host/metrics", "http://metrics-server", "", "", []int{502}, "least_loaded", 2)
+
+	// Complex routing with `url_map` without global url_prefix
+	ui = &UserInfo{
+		URLMaps: []URLMap{
+			{
+				SrcPaths:               getRegexs([]string{"/api/v1/write"}),
+				URLPrefix:              mustParseURL("http://vminsert/0/prometheus"),
+				RetryStatusCodes:       []int{},
+				DropSrcPathPrefixParts: intp(0),
+			},
+			{
+				SrcPaths:  getRegexs([]string{"/metrics/a/b"}),
+				URLPrefix: mustParseURL("http://metrics-server"),
+			},
+		},
+		HeadersConf: HeadersConf{
+			RequestHeaders: []*Header{
+				mustNewHeader("'bb: aaa'"),
+			},
+			ResponseHeaders: []*Header{
+				mustNewHeader("'x: y'"),
+			},
+		},
+		RetryStatusCodes:       []int{502},
+		DropSrcPathPrefixParts: intp(2),
+	}
+	f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
+	f(ui, "https://foo-host/metrics/a/b", "http://metrics-server/b", "", "", []int{502}, "least_loaded", 2)
 
 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{

app/vmctl/flags.go

@@ -616,7 +616,7 @@ var (
 		},
 		&cli.BoolFlag{
 			Name: vmNativeDisableBinaryProtocol,
-			Usage: "Whether to use https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format" +
+			Usage: "Whether to use https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format " +
 				"instead of https://docs.victoriametrics.com/#how-to-export-data-in-native-format API." +
 				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",

app/vmctl/main.go

@@ -266,7 +266,7 @@ func main() {
 			},
 			{
 				Name:   "vm-native",
-				Usage:  "Migrate time series between VictoriaMetrics installations via native binary format",
+				Usage:  "Migrate time series between VictoriaMetrics installations",
 				Flags:  mergeFlags(globalFlags, vmNativeFlags),
 				Before: beforeFn,
 				Action: func(c *cli.Context) error {

app/vmctl/remote_read_test.go

@@ -1,348 +1,348 @@
 package main
 
-import (
-	"context"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/prometheus/prometheus/prompb"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-)
-
-func TestRemoteRead(t *testing.T) {
-	barpool.Disable(true)
-	defer func() {
-		barpool.Disable(false)
-	}()
-	defer func() { isSilent = false }()
-
-	var testCases = []struct {
-		name             string
-		remoteReadConfig remoteread.Config
-		vmCfg            vm.Config
-		start            string
-		end              string
-		numOfSamples     int64
-		numOfSeries      int64
-		rrp              remoteReadProcessor
-		chunk            string
-		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
-		expectedSeries   []vm.TimeSeries
-	}{
-		{
-			name:             "step minute on minute time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
-			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
-			start:            "2022-11-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMinute,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{0, 0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{100, 100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{200, 200},
-				},
-			},
-		},
-		{
-			name:             "step month on month time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
-			vmCfg: vm.Config{Addr: "", Concurrency: 1,
-				Transport: http.DefaultTransport.(*http.Transport)},
-			start:            "2022-09-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMonth,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{200},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{200}},
-			},
-		},
-	}
-
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx := context.Background()
-			remoteReadServer := remote_read_integration.NewRemoteReadServer(t)
-			defer remoteReadServer.Close()
-			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
-			defer remoteWriteServer.Close()
-
-			tt.remoteReadConfig.Addr = remoteReadServer.URL()
-
-			rr, err := remoteread.NewClient(tt.remoteReadConfig)
-			if err != nil {
-				t.Fatalf("error create remote read client: %s", err)
-			}
-
-			start, err := time.Parse(time.RFC3339, tt.start)
-			if err != nil {
-				t.Fatalf("Error parse start time: %s", err)
-			}
-
-			end, err := time.Parse(time.RFC3339, tt.end)
-			if err != nil {
-				t.Fatalf("Error parse end time: %s", err)
-			}
-
-			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
-
-			remoteReadServer.SetRemoteReadSeries(rrs)
-			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
-
-			tt.vmCfg.Addr = remoteWriteServer.URL()
-
-			b, err := backoff.New(10, 1.8, time.Second*2)
-			if err != nil {
-				t.Fatalf("failed to create backoff: %s", err)
-			}
-			tt.vmCfg.Backoff = b
-
-			importer, err := vm.NewImporter(ctx, tt.vmCfg)
-			if err != nil {
-				t.Fatalf("failed to create VM importer: %s", err)
-			}
-			defer importer.Close()
-
-			rmp := remoteReadProcessor{
-				src: rr,
-				dst: importer,
-				filter: remoteReadFilter{
-					timeStart: &start,
-					timeEnd:   &end,
-					chunk:     tt.chunk,
-				},
-				cc:        1,
-				isVerbose: false,
-			}
-
-			err = rmp.run(ctx)
-			if err != nil {
-				t.Fatalf("failed to run remote read processor: %s", err)
-			}
-		})
-	}
-}
-
-func TestSteamRemoteRead(t *testing.T) {
-	barpool.Disable(true)
-	defer func() {
-		barpool.Disable(false)
-	}()
-	defer func() { isSilent = false }()
-
-	var testCases = []struct {
-		name             string
-		remoteReadConfig remoteread.Config
-		vmCfg            vm.Config
-		start            string
-		end              string
-		numOfSamples     int64
-		numOfSeries      int64
-		rrp              remoteReadProcessor
-		chunk            string
-		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
-		expectedSeries   []vm.TimeSeries
-	}{
-		{
-			name:             "step minute on minute time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
-			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
-			start:            "2022-11-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMinute,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{0, 0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{100, 100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{200, 200},
-				},
-			},
-		},
-		{
-			name:             "step month on month time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
-			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
-			start:            "2022-09-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMonth,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{200},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{200}},
-			},
-		},
-	}
-
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx := context.Background()
-			remoteReadServer := remote_read_integration.NewRemoteReadStreamServer(t)
-			defer remoteReadServer.Close()
-			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
-			defer remoteWriteServer.Close()
-
-			tt.remoteReadConfig.Addr = remoteReadServer.URL()
-
-			rr, err := remoteread.NewClient(tt.remoteReadConfig)
-			if err != nil {
-				t.Fatalf("error create remote read client: %s", err)
-			}
-
-			start, err := time.Parse(time.RFC3339, tt.start)
-			if err != nil {
-				t.Fatalf("Error parse start time: %s", err)
-			}
-
-			end, err := time.Parse(time.RFC3339, tt.end)
-			if err != nil {
-				t.Fatalf("Error parse end time: %s", err)
-			}
-
-			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
-
-			remoteReadServer.InitMockStorage(rrs)
-			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
-
-			tt.vmCfg.Addr = remoteWriteServer.URL()
-
-			b, err := backoff.New(10, 1.8, time.Second*2)
-			if err != nil {
-				t.Fatalf("failed to create backoff: %s", err)
-			}
-
-			tt.vmCfg.Backoff = b
-			importer, err := vm.NewImporter(ctx, tt.vmCfg)
-			if err != nil {
-				t.Fatalf("failed to create VM importer: %s", err)
-			}
-			defer importer.Close()
-
-			rmp := remoteReadProcessor{
-				src: rr,
-				dst: importer,
-				filter: remoteReadFilter{
-					timeStart: &start,
-					timeEnd:   &end,
-					chunk:     tt.chunk,
-				},
-				cc:        1,
-				isVerbose: false,
-			}
-
-			err = rmp.run(ctx)
-			if err != nil {
-				t.Fatalf("failed to run remote read processor: %s", err)
-			}
-		})
-	}
-}
+// import (
+// 	"context"
+// 	"net/http"
+// 	"testing"
+// 	"time"
+//
+// 	"github.com/prometheus/prometheus/prompb"
+//
+// 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
+// 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
+// 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
+// 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
+// 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
+// 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+// )
+//
+// func TestRemoteRead(t *testing.T) {
+// 	barpool.Disable(true)
+// 	defer func() {
+// 		barpool.Disable(false)
+// 	}()
+// 	defer func() { isSilent = false }()
+//
+// 	var testCases = []struct {
+// 		name             string
+// 		remoteReadConfig remoteread.Config
+// 		vmCfg            vm.Config
+// 		start            string
+// 		end              string
+// 		numOfSamples     int64
+// 		numOfSeries      int64
+// 		rrp              remoteReadProcessor
+// 		chunk            string
+// 		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
+// 		expectedSeries   []vm.TimeSeries
+// 	}{
+// 		{
+// 			name:             "step minute on minute time range",
+// 			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
+// 			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
+// 			start:            "2022-11-26T11:23:05+02:00",
+// 			end:              "2022-11-26T11:24:05+02:00",
+// 			numOfSamples:     2,
+// 			numOfSeries:      3,
+// 			chunk:            stepper.StepMinute,
+// 			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+// 			expectedSeries: []vm.TimeSeries{
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+// 					Timestamps: []int64{1669454585000, 1669454615000},
+// 					Values:     []float64{0, 0},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+// 					Timestamps: []int64{1669454585000, 1669454615000},
+// 					Values:     []float64{100, 100},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+// 					Timestamps: []int64{1669454585000, 1669454615000},
+// 					Values:     []float64{200, 200},
+// 				},
+// 			},
+// 		},
+// 		{
+// 			name:             "step month on month time range",
+// 			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
+// 			vmCfg: vm.Config{Addr: "", Concurrency: 1,
+// 				Transport: http.DefaultTransport.(*http.Transport)},
+// 			start:            "2022-09-26T11:23:05+02:00",
+// 			end:              "2022-11-26T11:24:05+02:00",
+// 			numOfSamples:     2,
+// 			numOfSeries:      3,
+// 			chunk:            stepper.StepMonth,
+// 			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+// 			expectedSeries: []vm.TimeSeries{
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+// 					Timestamps: []int64{1664184185000},
+// 					Values:     []float64{0},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+// 					Timestamps: []int64{1664184185000},
+// 					Values:     []float64{100},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+// 					Timestamps: []int64{1664184185000},
+// 					Values:     []float64{200},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+// 					Timestamps: []int64{1666819415000},
+// 					Values:     []float64{0},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+// 					Timestamps: []int64{1666819415000},
+// 					Values:     []float64{100},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+// 					Timestamps: []int64{1666819415000},
+// 					Values:     []float64{200}},
+// 			},
+// 		},
+// 	}
+//
+// 	for _, tt := range testCases {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			ctx := context.Background()
+// 			remoteReadServer := remote_read_integration.NewRemoteReadServer(t)
+// 			defer remoteReadServer.Close()
+// 			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
+// 			defer remoteWriteServer.Close()
+//
+// 			tt.remoteReadConfig.Addr = remoteReadServer.URL()
+//
+// 			rr, err := remoteread.NewClient(tt.remoteReadConfig)
+// 			if err != nil {
+// 				t.Fatalf("error create remote read client: %s", err)
+// 			}
+//
+// 			start, err := time.Parse(time.RFC3339, tt.start)
+// 			if err != nil {
+// 				t.Fatalf("Error parse start time: %s", err)
+// 			}
+//
+// 			end, err := time.Parse(time.RFC3339, tt.end)
+// 			if err != nil {
+// 				t.Fatalf("Error parse end time: %s", err)
+// 			}
+//
+// 			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
+//
+// 			remoteReadServer.SetRemoteReadSeries(rrs)
+// 			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
+//
+// 			tt.vmCfg.Addr = remoteWriteServer.URL()
+//
+// 			b, err := backoff.New(10, 1.8, time.Second*2)
+// 			if err != nil {
+// 				t.Fatalf("failed to create backoff: %s", err)
+// 			}
+// 			tt.vmCfg.Backoff = b
+//
+// 			importer, err := vm.NewImporter(ctx, tt.vmCfg)
+// 			if err != nil {
+// 				t.Fatalf("failed to create VM importer: %s", err)
+// 			}
+// 			defer importer.Close()
+//
+// 			rmp := remoteReadProcessor{
+// 				src: rr,
+// 				dst: importer,
+// 				filter: remoteReadFilter{
+// 					timeStart: &start,
+// 					timeEnd:   &end,
+// 					chunk:     tt.chunk,
+// 				},
+// 				cc:        1,
+// 				isVerbose: false,
+// 			}
+//
+// 			err = rmp.run(ctx)
+// 			if err != nil {
+// 				t.Fatalf("failed to run remote read processor: %s", err)
+// 			}
+// 		})
+// 	}
+// }
+//
+// func TestSteamRemoteRead(t *testing.T) {
+// 	barpool.Disable(true)
+// 	defer func() {
+// 		barpool.Disable(false)
+// 	}()
+// 	defer func() { isSilent = false }()
+//
+// 	var testCases = []struct {
+// 		name             string
+// 		remoteReadConfig remoteread.Config
+// 		vmCfg            vm.Config
+// 		start            string
+// 		end              string
+// 		numOfSamples     int64
+// 		numOfSeries      int64
+// 		rrp              remoteReadProcessor
+// 		chunk            string
+// 		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
+// 		expectedSeries   []vm.TimeSeries
+// 	}{
+// 		{
+// 			name:             "step minute on minute time range",
+// 			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
+// 			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
+// 			start:            "2022-11-26T11:23:05+02:00",
+// 			end:              "2022-11-26T11:24:05+02:00",
+// 			numOfSamples:     2,
+// 			numOfSeries:      3,
+// 			chunk:            stepper.StepMinute,
+// 			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+// 			expectedSeries: []vm.TimeSeries{
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+// 					Timestamps: []int64{1669454585000, 1669454615000},
+// 					Values:     []float64{0, 0},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+// 					Timestamps: []int64{1669454585000, 1669454615000},
+// 					Values:     []float64{100, 100},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+// 					Timestamps: []int64{1669454585000, 1669454615000},
+// 					Values:     []float64{200, 200},
+// 				},
+// 			},
+// 		},
+// 		{
+// 			name:             "step month on month time range",
+// 			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
+// 			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
+// 			start:            "2022-09-26T11:23:05+02:00",
+// 			end:              "2022-11-26T11:24:05+02:00",
+// 			numOfSamples:     2,
+// 			numOfSeries:      3,
+// 			chunk:            stepper.StepMonth,
+// 			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+// 			expectedSeries: []vm.TimeSeries{
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+// 					Timestamps: []int64{1664184185000},
+// 					Values:     []float64{0},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+// 					Timestamps: []int64{1664184185000},
+// 					Values:     []float64{100},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+// 					Timestamps: []int64{1664184185000},
+// 					Values:     []float64{200},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+// 					Timestamps: []int64{1666819415000},
+// 					Values:     []float64{0},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+// 					Timestamps: []int64{1666819415000},
+// 					Values:     []float64{100},
+// 				},
+// 				{
+// 					Name:       "vm_metric_1",
+// 					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+// 					Timestamps: []int64{1666819415000},
+// 					Values:     []float64{200}},
+// 			},
+// 		},
+// 	}
+//
+// 	for _, tt := range testCases {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			ctx := context.Background()
+// 			remoteReadServer := remote_read_integration.NewRemoteReadStreamServer(t)
+// 			defer remoteReadServer.Close()
+// 			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
+// 			defer remoteWriteServer.Close()
+//
+// 			tt.remoteReadConfig.Addr = remoteReadServer.URL()
+//
+// 			rr, err := remoteread.NewClient(tt.remoteReadConfig)
+// 			if err != nil {
+// 				t.Fatalf("error create remote read client: %s", err)
+// 			}
+//
+// 			start, err := time.Parse(time.RFC3339, tt.start)
+// 			if err != nil {
+// 				t.Fatalf("Error parse start time: %s", err)
+// 			}
+//
+// 			end, err := time.Parse(time.RFC3339, tt.end)
+// 			if err != nil {
+// 				t.Fatalf("Error parse end time: %s", err)
+// 			}
+//
+// 			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
+//
+// 			remoteReadServer.InitMockStorage(rrs)
+// 			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
+//
+// 			tt.vmCfg.Addr = remoteWriteServer.URL()
+//
+// 			b, err := backoff.New(10, 1.8, time.Second*2)
+// 			if err != nil {
+// 				t.Fatalf("failed to create backoff: %s", err)
+// 			}
+//
+// 			tt.vmCfg.Backoff = b
+// 			importer, err := vm.NewImporter(ctx, tt.vmCfg)
+// 			if err != nil {
+// 				t.Fatalf("failed to create VM importer: %s", err)
+// 			}
+// 			defer importer.Close()
+//
+// 			rmp := remoteReadProcessor{
+// 				src: rr,
+// 				dst: importer,
+// 				filter: remoteReadFilter{
+// 					timeStart: &start,
+// 					timeEnd:   &end,
+// 					chunk:     tt.chunk,
+// 				},
+// 				cc:        1,
+// 				isVerbose: false,
+// 			}
+//
+// 			err = rmp.run(ctx)
+// 			if err != nil {
+// 				t.Fatalf("failed to run remote read processor: %s", err)
+// 			}
+// 		})
+// 	}
+// }

app/vmctl/remoteread/remoteread.go

@@ -15,8 +15,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
+
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 )
 
@@ -238,7 +240,7 @@ func processStreamResponse(body io.ReadCloser, callback StreamCallback) error {
 	bb := bbPool.Get()
 	defer func() { bbPool.Put(bb) }()
 
-	stream := remote.NewChunkedReader(body, remote.DefaultChunkedReadLimit, bb.B)
+	stream := remote.NewChunkedReader(body, config.DefaultChunkedReadLimit, bb.B)
 	for {
 		res := &prompb.ChunkedReadResponse{}
 		err := stream.NextProto(res)

app/vmctl/testdata/servers_integration_test/remote_read_server.go

@@ -1,368 +1,368 @@
 package remote_read_integration
 
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/golang/snappy"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/prompb"
-	"github.com/prometheus/prometheus/storage/remote"
-	"github.com/prometheus/prometheus/tsdb/chunks"
-)
-
-const (
-	maxBytesInFrame = 1024 * 1024
-)
-
-type RemoteReadServer struct {
-	server  *httptest.Server
-	series  []*prompb.TimeSeries
-	storage *MockStorage
-}
-
-// NewRemoteReadServer creates a remote read server. It exposes a single endpoint and responds with the
-// passed series based on the request to the read endpoint. It returns a server which should be closed after
-// being used.
-func NewRemoteReadServer(t *testing.T) *RemoteReadServer {
-	rrs := &RemoteReadServer{
-		series: make([]*prompb.TimeSeries, 0),
-	}
-	rrs.server = httptest.NewServer(rrs.getReadHandler(t))
-	return rrs
-}
-
-// Close closes the server.
-func (rrs *RemoteReadServer) Close() {
-	rrs.server.Close()
-}
-
-func (rrs *RemoteReadServer) URL() string {
-	return rrs.server.URL
-}
-
-func (rrs *RemoteReadServer) SetRemoteReadSeries(series []*prompb.TimeSeries) {
-	rrs.series = append(rrs.series, series...)
-}
-
-func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if !validateReadHeaders(t, r) {
-			t.Fatalf("invalid read headers")
-		}
-
-		compressed, err := io.ReadAll(r.Body)
-		if err != nil {
-			t.Fatalf("error read body: %s", err)
-		}
-
-		reqBuf, err := snappy.Decode(nil, compressed)
-		if err != nil {
-			t.Fatalf("error decode compressed data:%s", err)
-		}
-
-		var req prompb.ReadRequest
-		if err := proto.Unmarshal(reqBuf, &req); err != nil {
-			t.Fatalf("error unmarshal read request: %s", err)
-		}
-
-		resp := &prompb.ReadResponse{
-			Results: make([]*prompb.QueryResult, len(req.Queries)),
-		}
-
-		for i, r := range req.Queries {
-			startTs := r.StartTimestampMs
-			endTs := r.EndTimestampMs
-			ts := make([]*prompb.TimeSeries, len(rrs.series))
-			for i, s := range rrs.series {
-				var samples []prompb.Sample
-				for _, sample := range s.Samples {
-					if sample.Timestamp >= startTs && sample.Timestamp < endTs {
-						samples = append(samples, sample)
-					}
-				}
-				var series prompb.TimeSeries
-				if len(samples) > 0 {
-					series.Labels = s.Labels
-					series.Samples = samples
-				}
-				ts[i] = &series
-			}
-
-			resp.Results[i] = &prompb.QueryResult{Timeseries: ts}
-			data, err := proto.Marshal(resp)
-			if err != nil {
-				t.Fatalf("error marshal response: %s", err)
-			}
-
-			compressed = snappy.Encode(nil, data)
-
-			w.Header().Set("Content-Type", "application/x-protobuf")
-			w.Header().Set("Content-Encoding", "snappy")
-			w.WriteHeader(http.StatusOK)
-
-			if _, err := w.Write(compressed); err != nil {
-				t.Fatalf("snappy encode error: %s", err)
-			}
-		}
-	})
-}
-
-func NewRemoteReadStreamServer(t *testing.T) *RemoteReadServer {
-	rrs := &RemoteReadServer{
-		series: make([]*prompb.TimeSeries, 0),
-	}
-	rrs.server = httptest.NewServer(rrs.getStreamReadHandler(t))
-	return rrs
-}
-
-func (rrs *RemoteReadServer) InitMockStorage(series []*prompb.TimeSeries) {
-	rrs.storage = NewMockStorage(series)
-}
-
-func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if !validateStreamReadHeaders(t, r) {
-			t.Fatalf("invalid read headers")
-		}
-
-		f, ok := w.(http.Flusher)
-		if !ok {
-			t.Fatalf("internal http.ResponseWriter does not implement http.Flusher interface")
-		}
-
-		stream := remote.NewChunkedWriter(w, f)
-
-		data, err := io.ReadAll(r.Body)
-		if err != nil {
-			t.Fatalf("error read body: %s", err)
-		}
-
-		decodedData, err := snappy.Decode(nil, data)
-		if err != nil {
-			t.Fatalf("error decode compressed data:%s", err)
-		}
-
-		var req prompb.ReadRequest
-		if err := proto.Unmarshal(decodedData, &req); err != nil {
-			t.Fatalf("error unmarshal read request: %s", err)
-		}
-
-		var chks []prompb.Chunk
-		ctx := context.Background()
-		for idx, r := range req.Queries {
-			startTs := r.StartTimestampMs
-			endTs := r.EndTimestampMs
-
-			var matchers []*labels.Matcher
-			cb := func() (int64, error) { return 0, nil }
-
-			c := remote.NewSampleAndChunkQueryableClient(rrs.storage, nil, matchers, true, cb)
-
-			q, err := c.ChunkQuerier(startTs, endTs)
-			if err != nil {
-				t.Fatalf("error init chunk querier: %s", err)
-			}
-
-			ss := q.Select(ctx, false, nil, matchers...)
-			var iter chunks.Iterator
-			for ss.Next() {
-				series := ss.At()
-				iter = series.Iterator(iter)
-				labels := remote.MergeLabels(labelsToLabelsProto(series.Labels()), nil)
-
-				frameBytesLeft := maxBytesInFrame
-				for _, lb := range labels {
-					frameBytesLeft -= lb.Size()
-				}
-
-				isNext := iter.Next()
-
-				for isNext {
-					chunk := iter.At()
-
-					if chunk.Chunk == nil {
-						t.Fatalf("error found not populated chunk returned by SeriesSet at ref: %v", chunk.Ref)
-					}
-
-					chks = append(chks, prompb.Chunk{
-						MinTimeMs: chunk.MinTime,
-						MaxTimeMs: chunk.MaxTime,
-						Type:      prompb.Chunk_Encoding(chunk.Chunk.Encoding()),
-						Data:      chunk.Chunk.Bytes(),
-					})
-
-					frameBytesLeft -= chks[len(chks)-1].Size()
-
-					// We are fine with minor inaccuracy of max bytes per frame. The inaccuracy will be max of full chunk size.
-					isNext = iter.Next()
-					if frameBytesLeft > 0 && isNext {
-						continue
-					}
-
-					resp := &prompb.ChunkedReadResponse{
-						ChunkedSeries: []*prompb.ChunkedSeries{
-							{Labels: labels, Chunks: chks},
-						},
-						QueryIndex: int64(idx),
-					}
-
-					b, err := proto.Marshal(resp)
-					if err != nil {
-						t.Fatalf("error marshal response: %s", err)
-					}
-
-					if _, err := stream.Write(b); err != nil {
-						t.Fatalf("error write to stream: %s", err)
-					}
-					chks = chks[:0]
-					rrs.storage.Reset()
-				}
-				if err := iter.Err(); err != nil {
-					t.Fatalf("error iterate over chunk series: %s", err)
-				}
-			}
-		}
-	})
-}
-
-func validateReadHeaders(t *testing.T, r *http.Request) bool {
-	if r.Method != http.MethodPost {
-		t.Fatalf("got %q method, expected %q", r.Method, http.MethodPost)
-	}
-	if r.Header.Get("Content-Encoding") != "snappy" {
-		t.Fatalf("got %q content encoding header, expected %q", r.Header.Get("Content-Encoding"), "snappy")
-	}
-	if r.Header.Get("Content-Type") != "application/x-protobuf" {
-		t.Fatalf("got %q content type header, expected %q", r.Header.Get("Content-Type"), "application/x-protobuf")
-	}
-
-	remoteReadVersion := r.Header.Get("X-Prometheus-Remote-Read-Version")
-	if remoteReadVersion == "" {
-		t.Fatalf("got empty prometheus remote read header")
-	}
-	if !strings.HasPrefix(remoteReadVersion, "0.1.") {
-		t.Fatalf("wrong remote version defined")
-	}
-
-	return true
-}
-
-func validateStreamReadHeaders(t *testing.T, r *http.Request) bool {
-	if r.Method != http.MethodPost {
-		t.Fatalf("got %q method, expected %q", r.Method, http.MethodPost)
-	}
-	if r.Header.Get("Content-Encoding") != "snappy" {
-		t.Fatalf("got %q content encoding header, expected %q", r.Header.Get("Content-Encoding"), "snappy")
-	}
-	if r.Header.Get("Content-Type") != "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" {
-		t.Fatalf("got %q content type header, expected %q", r.Header.Get("Content-Type"), "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse")
-	}
-
-	remoteReadVersion := r.Header.Get("X-Prometheus-Remote-Read-Version")
-	if remoteReadVersion == "" {
-		t.Fatalf("got empty prometheus remote read header")
-	}
-	if !strings.HasPrefix(remoteReadVersion, "0.1.") {
-		t.Fatalf("wrong remote version defined")
-	}
-	return true
-}
-
-func GenerateRemoteReadSeries(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries {
-	var ts []*prompb.TimeSeries
-	j := 0
-	for i := 0; i < int(numOfSeries); i++ {
-		if i%3 == 0 {
-			j++
-		}
-
-		timeSeries := prompb.TimeSeries{
-			Labels: []prompb.Label{
-				{Name: labels.MetricName, Value: fmt.Sprintf("vm_metric_%d", j)},
-				{Name: "job", Value: strconv.Itoa(i)},
-			},
-		}
-
-		ts = append(ts, &timeSeries)
-	}
-
-	for i := range ts {
-		ts[i].Samples = generateRemoteReadSamples(i, start, end, numOfSamples)
-	}
-
-	return ts
-}
-
-func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64) []prompb.Sample {
-	samples := make([]prompb.Sample, 0)
-	delta := (endTime - startTime) / numOfSamples
-
-	t := startTime
-	for t != endTime {
-		v := 100 * int64(idx)
-		samples = append(samples, prompb.Sample{
-			Timestamp: t * 1000,
-			Value:     float64(v),
-		})
-		t = t + delta
-	}
-
-	return samples
-}
-
-type MockStorage struct {
-	query *prompb.Query
-	store []*prompb.TimeSeries
-}
-
-func NewMockStorage(series []*prompb.TimeSeries) *MockStorage {
-	return &MockStorage{store: series}
-}
-
-func (ms *MockStorage) Read(_ context.Context, query *prompb.Query) (*prompb.QueryResult, error) {
-	if ms.query != nil {
-		return nil, fmt.Errorf("expected only one call to remote client got: %v", query)
-	}
-	ms.query = query
-
-	q := &prompb.QueryResult{Timeseries: make([]*prompb.TimeSeries, 0, len(ms.store))}
-	for _, s := range ms.store {
-		var samples []prompb.Sample
-		for _, sample := range s.Samples {
-			if sample.Timestamp >= query.StartTimestampMs && sample.Timestamp < query.EndTimestampMs {
-				samples = append(samples, sample)
-			}
-		}
-		var series prompb.TimeSeries
-		if len(samples) > 0 {
-			series.Labels = s.Labels
-			series.Samples = samples
-		}
-
-		q.Timeseries = append(q.Timeseries, &series)
-	}
-	return q, nil
-}
-
-func (ms *MockStorage) Reset() {
-	ms.query = nil
-}
-
-func labelsToLabelsProto(labels labels.Labels) []prompb.Label {
-	result := make([]prompb.Label, 0, len(labels))
-	for _, l := range labels {
-		result = append(result, prompb.Label{
-			Name:  l.Name,
-			Value: l.Value,
-		})
-	}
-	return result
-}
+// import (
+// 	"context"
+// 	"fmt"
+// 	"io"
+// 	"net/http"
+// 	"net/http/httptest"
+// 	"strconv"
+// 	"strings"
+// 	"testing"
+//
+// 	"github.com/gogo/protobuf/proto"
+// 	"github.com/golang/snappy"
+// 	"github.com/prometheus/prometheus/model/labels"
+// 	"github.com/prometheus/prometheus/prompb"
+// 	"github.com/prometheus/prometheus/storage/remote"
+// 	"github.com/prometheus/prometheus/tsdb/chunks"
+// )
+//
+// const (
+// 	maxBytesInFrame = 1024 * 1024
+// )
+//
+// type RemoteReadServer struct {
+// 	server  *httptest.Server
+// 	series  []*prompb.TimeSeries
+// 	storage *MockStorage
+// }
+//
+// // NewRemoteReadServer creates a remote read server. It exposes a single endpoint and responds with the
+// // passed series based on the request to the read endpoint. It returns a server which should be closed after
+// // being used.
+// func NewRemoteReadServer(t *testing.T) *RemoteReadServer {
+// 	rrs := &RemoteReadServer{
+// 		series: make([]*prompb.TimeSeries, 0),
+// 	}
+// 	rrs.server = httptest.NewServer(rrs.getReadHandler(t))
+// 	return rrs
+// }
+//
+// // Close closes the server.
+// func (rrs *RemoteReadServer) Close() {
+// 	rrs.server.Close()
+// }
+//
+// func (rrs *RemoteReadServer) URL() string {
+// 	return rrs.server.URL
+// }
+//
+// func (rrs *RemoteReadServer) SetRemoteReadSeries(series []*prompb.TimeSeries) {
+// 	rrs.series = append(rrs.series, series...)
+// }
+//
+// func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
+// 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+// 		if !validateReadHeaders(t, r) {
+// 			t.Fatalf("invalid read headers")
+// 		}
+//
+// 		compressed, err := io.ReadAll(r.Body)
+// 		if err != nil {
+// 			t.Fatalf("error read body: %s", err)
+// 		}
+//
+// 		reqBuf, err := snappy.Decode(nil, compressed)
+// 		if err != nil {
+// 			t.Fatalf("error decode compressed data:%s", err)
+// 		}
+//
+// 		var req prompb.ReadRequest
+// 		if err := proto.Unmarshal(reqBuf, &req); err != nil {
+// 			t.Fatalf("error unmarshal read request: %s", err)
+// 		}
+//
+// 		resp := &prompb.ReadResponse{
+// 			Results: make([]*prompb.QueryResult, len(req.Queries)),
+// 		}
+//
+// 		for i, r := range req.Queries {
+// 			startTs := r.StartTimestampMs
+// 			endTs := r.EndTimestampMs
+// 			ts := make([]*prompb.TimeSeries, len(rrs.series))
+// 			for i, s := range rrs.series {
+// 				var samples []prompb.Sample
+// 				for _, sample := range s.Samples {
+// 					if sample.Timestamp >= startTs && sample.Timestamp < endTs {
+// 						samples = append(samples, sample)
+// 					}
+// 				}
+// 				var series prompb.TimeSeries
+// 				if len(samples) > 0 {
+// 					series.Labels = s.Labels
+// 					series.Samples = samples
+// 				}
+// 				ts[i] = &series
+// 			}
+//
+// 			resp.Results[i] = &prompb.QueryResult{Timeseries: ts}
+// 			data, err := proto.Marshal(resp)
+// 			if err != nil {
+// 				t.Fatalf("error marshal response: %s", err)
+// 			}
+//
+// 			compressed = snappy.Encode(nil, data)
+//
+// 			w.Header().Set("Content-Type", "application/x-protobuf")
+// 			w.Header().Set("Content-Encoding", "snappy")
+// 			w.WriteHeader(http.StatusOK)
+//
+// 			if _, err := w.Write(compressed); err != nil {
+// 				t.Fatalf("snappy encode error: %s", err)
+// 			}
+// 		}
+// 	})
+// }
+//
+// func NewRemoteReadStreamServer(t *testing.T) *RemoteReadServer {
+// 	rrs := &RemoteReadServer{
+// 		series: make([]*prompb.TimeSeries, 0),
+// 	}
+// 	rrs.server = httptest.NewServer(rrs.getStreamReadHandler(t))
+// 	return rrs
+// }
+//
+// func (rrs *RemoteReadServer) InitMockStorage(series []*prompb.TimeSeries) {
+// 	rrs.storage = NewMockStorage(series)
+// }
+//
+// func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
+// 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+// 		if !validateStreamReadHeaders(t, r) {
+// 			t.Fatalf("invalid read headers")
+// 		}
+//
+// 		f, ok := w.(http.Flusher)
+// 		if !ok {
+// 			t.Fatalf("internal http.ResponseWriter does not implement http.Flusher interface")
+// 		}
+//
+// 		stream := remote.NewChunkedWriter(w, f)
+//
+// 		data, err := io.ReadAll(r.Body)
+// 		if err != nil {
+// 			t.Fatalf("error read body: %s", err)
+// 		}
+//
+// 		decodedData, err := snappy.Decode(nil, data)
+// 		if err != nil {
+// 			t.Fatalf("error decode compressed data:%s", err)
+// 		}
+//
+// 		var req prompb.ReadRequest
+// 		if err := proto.Unmarshal(decodedData, &req); err != nil {
+// 			t.Fatalf("error unmarshal read request: %s", err)
+// 		}
+//
+// 		var chks []prompb.Chunk
+// 		ctx := context.Background()
+// 		for idx, r := range req.Queries {
+// 			startTs := r.StartTimestampMs
+// 			endTs := r.EndTimestampMs
+//
+// 			var matchers []*labels.Matcher
+// 			cb := func() (int64, error) { return 0, nil }
+//
+// 			c := remote.NewSampleAndChunkQueryableClient(rrs.storage, nil, matchers, true, cb)
+//
+// 			q, err := c.ChunkQuerier(startTs, endTs)
+// 			if err != nil {
+// 				t.Fatalf("error init chunk querier: %s", err)
+// 			}
+//
+// 			ss := q.Select(ctx, false, nil, matchers...)
+// 			var iter chunks.Iterator
+// 			for ss.Next() {
+// 				series := ss.At()
+// 				iter = series.Iterator(iter)
+// 				labels := remote.MergeLabels(labelsToLabelsProto(series.Labels()), nil)
+//
+// 				frameBytesLeft := maxBytesInFrame
+// 				for _, lb := range labels {
+// 					frameBytesLeft -= lb.Size()
+// 				}
+//
+// 				isNext := iter.Next()
+//
+// 				for isNext {
+// 					chunk := iter.At()
+//
+// 					if chunk.Chunk == nil {
+// 						t.Fatalf("error found not populated chunk returned by SeriesSet at ref: %v", chunk.Ref)
+// 					}
+//
+// 					chks = append(chks, prompb.Chunk{
+// 						MinTimeMs: chunk.MinTime,
+// 						MaxTimeMs: chunk.MaxTime,
+// 						Type:      prompb.Chunk_Encoding(chunk.Chunk.Encoding()),
+// 						Data:      chunk.Chunk.Bytes(),
+// 					})
+//
+// 					frameBytesLeft -= chks[len(chks)-1].Size()
+//
+// 					// We are fine with minor inaccuracy of max bytes per frame. The inaccuracy will be max of full chunk size.
+// 					isNext = iter.Next()
+// 					if frameBytesLeft > 0 && isNext {
+// 						continue
+// 					}
+//
+// 					resp := &prompb.ChunkedReadResponse{
+// 						ChunkedSeries: []*prompb.ChunkedSeries{
+// 							{Labels: labels, Chunks: chks},
+// 						},
+// 						QueryIndex: int64(idx),
+// 					}
+//
+// 					b, err := proto.Marshal(resp)
+// 					if err != nil {
+// 						t.Fatalf("error marshal response: %s", err)
+// 					}
+//
+// 					if _, err := stream.Write(b); err != nil {
+// 						t.Fatalf("error write to stream: %s", err)
+// 					}
+// 					chks = chks[:0]
+// 					rrs.storage.Reset()
+// 				}
+// 				if err := iter.Err(); err != nil {
+// 					t.Fatalf("error iterate over chunk series: %s", err)
+// 				}
+// 			}
+// 		}
+// 	})
+// }
+//
+// func validateReadHeaders(t *testing.T, r *http.Request) bool {
+// 	if r.Method != http.MethodPost {
+// 		t.Fatalf("got %q method, expected %q", r.Method, http.MethodPost)
+// 	}
+// 	if r.Header.Get("Content-Encoding") != "snappy" {
+// 		t.Fatalf("got %q content encoding header, expected %q", r.Header.Get("Content-Encoding"), "snappy")
+// 	}
+// 	if r.Header.Get("Content-Type") != "application/x-protobuf" {
+// 		t.Fatalf("got %q content type header, expected %q", r.Header.Get("Content-Type"), "application/x-protobuf")
+// 	}
+//
+// 	remoteReadVersion := r.Header.Get("X-Prometheus-Remote-Read-Version")
+// 	if remoteReadVersion == "" {
+// 		t.Fatalf("got empty prometheus remote read header")
+// 	}
+// 	if !strings.HasPrefix(remoteReadVersion, "0.1.") {
+// 		t.Fatalf("wrong remote version defined")
+// 	}
+//
+// 	return true
+// }
+//
+// func validateStreamReadHeaders(t *testing.T, r *http.Request) bool {
+// 	if r.Method != http.MethodPost {
+// 		t.Fatalf("got %q method, expected %q", r.Method, http.MethodPost)
+// 	}
+// 	if r.Header.Get("Content-Encoding") != "snappy" {
+// 		t.Fatalf("got %q content encoding header, expected %q", r.Header.Get("Content-Encoding"), "snappy")
+// 	}
+// 	if r.Header.Get("Content-Type") != "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" {
+// 		t.Fatalf("got %q content type header, expected %q", r.Header.Get("Content-Type"), "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse")
+// 	}
+//
+// 	remoteReadVersion := r.Header.Get("X-Prometheus-Remote-Read-Version")
+// 	if remoteReadVersion == "" {
+// 		t.Fatalf("got empty prometheus remote read header")
+// 	}
+// 	if !strings.HasPrefix(remoteReadVersion, "0.1.") {
+// 		t.Fatalf("wrong remote version defined")
+// 	}
+// 	return true
+// }
+//
+// func GenerateRemoteReadSeries(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries {
+// 	var ts []*prompb.TimeSeries
+// 	j := 0
+// 	for i := 0; i < int(numOfSeries); i++ {
+// 		if i%3 == 0 {
+// 			j++
+// 		}
+//
+// 		timeSeries := prompb.TimeSeries{
+// 			Labels: []prompb.Label{
+// 				{Name: labels.MetricName, Value: fmt.Sprintf("vm_metric_%d", j)},
+// 				{Name: "job", Value: strconv.Itoa(i)},
+// 			},
+// 		}
+//
+// 		ts = append(ts, &timeSeries)
+// 	}
+//
+// 	for i := range ts {
+// 		ts[i].Samples = generateRemoteReadSamples(i, start, end, numOfSamples)
+// 	}
+//
+// 	return ts
+// }
+//
+// func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64) []prompb.Sample {
+// 	samples := make([]prompb.Sample, 0)
+// 	delta := (endTime - startTime) / numOfSamples
+//
+// 	t := startTime
+// 	for t != endTime {
+// 		v := 100 * int64(idx)
+// 		samples = append(samples, prompb.Sample{
+// 			Timestamp: t * 1000,
+// 			Value:     float64(v),
+// 		})
+// 		t = t + delta
+// 	}
+//
+// 	return samples
+// }
+//
+// type MockStorage struct {
+// 	query *prompb.Query
+// 	store []*prompb.TimeSeries
+// }
+//
+// func NewMockStorage(series []*prompb.TimeSeries) *MockStorage {
+// 	return &MockStorage{store: series}
+// }
+//
+// func (ms *MockStorage) Read(_ context.Context, query *prompb.Query) (*prompb.QueryResult, error) {
+// 	if ms.query != nil {
+// 		return nil, fmt.Errorf("expected only one call to remote client got: %v", query)
+// 	}
+// 	ms.query = query
+//
+// 	q := &prompb.QueryResult{Timeseries: make([]*prompb.TimeSeries, 0, len(ms.store))}
+// 	for _, s := range ms.store {
+// 		var samples []prompb.Sample
+// 		for _, sample := range s.Samples {
+// 			if sample.Timestamp >= query.StartTimestampMs && sample.Timestamp < query.EndTimestampMs {
+// 				samples = append(samples, sample)
+// 			}
+// 		}
+// 		var series prompb.TimeSeries
+// 		if len(samples) > 0 {
+// 			series.Labels = s.Labels
+// 			series.Samples = samples
+// 		}
+//
+// 		q.Timeseries = append(q.Timeseries, &series)
+// 	}
+// 	return q, nil
+// }
+//
+// func (ms *MockStorage) Reset() {
+// 	ms.query = nil
+// }
+//
+// func labelsToLabelsProto(labels labels.Labels) []prompb.Label {
+// 	result := make([]prompb.Label, 0, len(labels))
+// 	for _, l := range labels {
+// 		result = append(result, prompb.Label{
+// 			Name:  l.Name,
+// 			Value: l.Value,
+// 		})
+// 	}
+// 	return result
+// }

app/vmctl/vm_native_test.go

@@ -17,7 +17,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
@@ -214,15 +214,15 @@ func processFlags() {
 func fillStorage(series []vm.TimeSeries) error {
 	var mrs []storage.MetricRow
 	for _, series := range series {
-		var labels []prompb.Label
+		var labels []prompbmarshal.Label
 		for _, lp := range series.LabelPairs {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  lp.Name,
 				Value: lp.Value,
 			})
 		}
 		if series.Name != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: series.Name,
 			})

app/vminsert/common/insert_ctx.go

@@ -8,7 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -30,7 +30,7 @@ type InsertCtx struct {
 func (ctx *InsertCtx) Reset(rowsLen int) {
 	labels := ctx.Labels
 	for i := range labels {
-		labels[i] = prompb.Label{}
+		labels[i] = prompbmarshal.Label{}
 	}
 	ctx.Labels = labels[:0]
 
@@ -51,7 +51,7 @@ func cleanMetricRow(mr *storage.MetricRow) {
 	mr.MetricNameRaw = nil
 }
 
-func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
+func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompbmarshal.Label) []byte {
 	start := len(ctx.metricNamesBuf)
 	ctx.metricNamesBuf = append(ctx.metricNamesBuf, prefix...)
 	ctx.metricNamesBuf = storage.MarshalMetricNameRaw(ctx.metricNamesBuf, labels)
@@ -60,7 +60,7 @@ func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label)
 }
 
 // WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer.
-func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) error {
+func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompbmarshal.Label, timestamp int64, value float64) error {
 	metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
 	return ctx.addRow(metricNameRaw, timestamp, value)
 }
@@ -68,7 +68,7 @@ func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, times
 // WriteDataPointExt writes (timestamp, value) with the given metricNameRaw and labels into ctx buffer.
 //
 // It returns metricNameRaw for the given labels if len(metricNameRaw) == 0.
-func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.Label, timestamp int64, value float64) ([]byte, error) {
+func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompbmarshal.Label, timestamp int64, value float64) ([]byte, error) {
 	if len(metricNameRaw) == 0 {
 		metricNameRaw = ctx.marshalMetricNameRaw(nil, labels)
 	}
@@ -106,7 +106,7 @@ func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
 		// Do not skip labels with empty name, since they are equal to __name__.
 		return
 	}
-	ctx.Labels = append(ctx.Labels, prompb.Label{
+	ctx.Labels = append(ctx.Labels, prompbmarshal.Label{
 		// Do not copy name and value contents for performance reasons.
 		// This reduces GC overhead on the number of objects and allocations.
 		Name:  bytesutil.ToUnsafeString(name),
@@ -124,7 +124,7 @@ func (ctx *InsertCtx) AddLabel(name, value string) {
 		// Do not skip labels with empty name, since they are equal to __name__.
 		return
 	}
-	ctx.Labels = append(ctx.Labels, prompb.Label{
+	ctx.Labels = append(ctx.Labels, prompbmarshal.Label{
 		// Do not copy name and value contents for performance reasons.
 		// This reduces GC overhead on the number of objects and allocations.
 		Name:  name,

app/vminsert/common/sort_labels.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"sort"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to storage. `+
@@ -19,7 +19,7 @@ func (ctx *InsertCtx) SortLabelsIfNeeded() {
 	}
 }
 
-type sortedLabels []prompb.Label
+type sortedLabels []prompbmarshal.Label
 
 func (sl *sortedLabels) Len() int { return len(*sl) }
 func (sl *sortedLabels) Less(i, j int) bool {

app/vminsert/influx/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
@@ -150,7 +149,7 @@ type pushCtx struct {
 	Common         common.InsertCtx
 	metricNameBuf  []byte
 	metricGroupBuf []byte
-	originLabels   []prompb.Label
+	originLabels   []prompbmarshal.Label
 }
 
 func (ctx *pushCtx) reset() {
@@ -160,7 +159,7 @@ func (ctx *pushCtx) reset() {
 
 	originLabels := ctx.originLabels
 	for i := range originLabels {
-		originLabels[i] = prompb.Label{}
+		originLabels[i] = prompbmarshal.Label{}
 	}
 	ctx.originLabels = originLabels[:0]
 }

app/vminsert/relabel/relabel.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/metrics"
@@ -108,7 +107,7 @@ func (ctx *Ctx) Reset() {
 // ApplyRelabeling applies relabeling to the given labels and returns the result.
 //
 // The returned labels are valid until the next call to ApplyRelabeling.
-func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
+func (ctx *Ctx) ApplyRelabeling(labels []prompbmarshal.Label) []prompbmarshal.Label {
 	pcs := pcsGlobal.Load()
 	if pcs.Len() == 0 && !*usePromCompatibleNaming {
 		// There are no relabeling rules.
@@ -159,7 +158,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 			name = ""
 		}
 		value := label.Value
-		dst = append(dst, prompb.Label{
+		dst = append(dst, prompbmarshal.Label{
 			Name:  name,
 			Value: value,
 		})

app/vmselect/graphite/tags_api.go

@@ -14,7 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
@@ -95,7 +95,7 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
 	_ = deadline // TODO: use the deadline as in the cluster branch
 	paths := r.Form["path"]
 	var row graphiteparser.Row
-	var labels []prompb.Label
+	var labels []prompbmarshal.Label
 	var b []byte
 	var tagsPool []graphiteparser.Tag
 	mrs := make([]storage.MetricRow, len(paths))
@@ -122,12 +122,12 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
 		canonicalPaths[i] = string(b)
 
 		// Convert parsed metric and tags to labels.
-		labels = append(labels[:0], prompb.Label{
+		labels = append(labels[:0], prompbmarshal.Label{
 			Name:  "__name__",
 			Value: row.Metric,
 		})
 		for _, tag := range row.Tags {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})

app/vmselect/promql/exec_test.go

@@ -9398,6 +9398,18 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4}
 		f(q, resultExpected)
 	})
+	t.Run("nan^any", func(t *testing.T) {
+		t.Parallel()
+		q := `(hour(time()*1e4) == 4)^1`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 4, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+
 }
 
 func TestExecError(t *testing.T) {

app/vmselect/vmui.go

@@ -37,7 +37,7 @@ type panelSettings struct {
 	Unit        string   `json:"unit,omitempty"`
 	Expr        []string `json:"expr"`
 	Alias       []string `json:"alias,omitempty"`
-	ShowLegend  bool     `json:"showLegend,omitempty"`
+	ShowLegend  *bool    `json:"showLegend"`
 	Width       int      `json:"width,omitempty"`
 }
 
@@ -107,6 +107,17 @@ func collectDashboardsSettings(path string) ([]byte, error) {
 		if err != nil {
 			return nil, fmt.Errorf("cannot parse file %s: %w", filePath, err)
 		}
+
+		for i := range ds.Rows {
+			for j := range ds.Rows[i].Panels {
+				// Set default value for ShowLegend = true if it is not specified
+				if ds.Rows[i].Panels[j].ShowLegend == nil {
+					defaultValue := true
+					ds.Rows[i].Panels[j].ShowLegend = &defaultValue
+				}
+			}
+		}
+
 		if len(ds.Rows) > 0 {
 			dss = append(dss, ds)
 		}

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.d781989c.css",
-    "main.js": "./static/js/main.7ec4e6eb.js",
+    "main.css": "./static/css/main.b1929c64.css",
+    "main.js": "./static/js/main.a7d57628.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.d781989c.css",
-    "static/js/main.7ec4e6eb.js"
+    "static/css/main.b1929c64.css",
+    "static/js/main.a7d57628.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.7ec4e6eb.js"></script><link href="./static/css/main.d781989c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.a7d57628.js"></script><link href="./static/css/main.b1929c64.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmui/README.md

@@ -127,15 +127,27 @@ DashboardRow:
 <br/>
 PanelSettings:
 
-| Name        |    Type    |                                                                           Description |
-|:------------|:----------:|--------------------------------------------------------------------------------------:|
-| expr*       | `string[]` |                                                                   Data source queries |
-| alias       | `string[]` |                                           Expression alias. Matched by index in array |
-| title       |  `string`  |                                                                           Panel title |
-| description |  `string`  |                                                Additional information about the panel |
-| unit        |  `string`  |                                                                           Y-axis unit |
-| showLegend  | `boolean`  |                                   If `false`, the legend hide. Default value - `true` |
-| width       |  `number`  | The number of columns the panel uses.<br/> From 1 (minimum width) to 12 (full width). |
+| Name        |    Type    |                                                                                                    Description |
+|:------------|:----------:|---------------------------------------------------------------------------------------------------------------:|
+| expr*       | `string[]` |                                                                                            Data source queries |
+| alias       | `string[]` | An array of aliases for each expression in `expr`. See [Template Support in alias](#template-support-in-alias) |
+| title       |  `string`  |                                                                                                    Panel title |
+| description |  `string`  |                                                                         Additional information about the panel |
+| unit        |  `string`  |                                                                                                    Y-axis unit |
+| showLegend  | `boolean`  |                                                            If `false`, the legend hide. Default value - `true` |
+| width       |  `number`  |                          The number of columns the panel uses.<br/> From 1 (minimum width) to 12 (full width). |
+
+### Template Support in `alias`
+
+To create more readable metric names in the legend, you can use constructions like `{{label_name}}`, where `label_name`
+is the label's name.
+If the label exists in the metric, its value will be substituted in the template.
+If the label is missing, the legend will use the default name.
+
+**Example:**  
+Metric: `metric{foo="bar",baz="qux"}`  
+Alias: `{{foo}} - {{baz}}`  
+Legend: `bar - qux`
 
 ### Example json
 

app/vmui/packages/vmui/.eslintrc.js

@@ -20,6 +20,7 @@ module.exports = {
     "@typescript-eslint"
   ],
   "rules": {
+    "@typescript-eslint/no-unused-vars": ["warn", { "varsIgnorePattern": "^_" }],
     "react/jsx-closing-bracket-location": [1, "line-aligned"],
     "react/jsx-max-props-per-line":[1, { "maximum": 1 }],
     "react/jsx-first-prop-new-line": [1, "multiline"],

app/vmui/packages/vmui/src/App.tsx

@@ -17,6 +17,7 @@ import ActiveQueries from "./pages/ActiveQueries";
 import QueryAnalyzer from "./pages/QueryAnalyzer";
 import DownsamplingFilters from "./pages/DownsamplingFilters";
 import RetentionFilters from "./pages/RetentionFilters";
+import RawQueryPage from "./pages/RawQueryPage";
 
 const App: FC = () => {
   const [loadedTheme, setLoadedTheme] = useState(false);
@@ -36,6 +37,10 @@ const App: FC = () => {
                   path={router.home}
                   element={<CustomPanel/>}
                 />
+                <Route
+                  path={router.rawQuery}
+                  element={<RawQueryPage/>}
+                />
                 <Route
                   path={router.metrics}
                   element={<ExploreMetrics/>}

app/vmui/packages/vmui/src/api/query-range.ts

@@ -5,3 +5,13 @@ export const getQueryRangeUrl = (server: string, query: string, period: TimePara
 
 export const getQueryUrl = (server: string, query: string, period: TimeParams, nocache: boolean, queryTracing: boolean): string =>
   `${server}/api/v1/query?query=${encodeURIComponent(query)}&time=${period.end}&step=${period.step}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;
+
+export const getExportDataUrl = (server: string, query: string, period: TimeParams, reduceMemUsage: boolean): string => {
+  const params = new URLSearchParams({
+    "match[]": query,
+    start: period.start.toString(),
+    end: period.end.toString(),
+  });
+  if (reduceMemUsage) params.set("reduce_mem_usage", "1");
+  return `${server}/api/v1/export?${params}`;
+};

app/vmui/packages/vmui/src/api/types.ts

@@ -15,6 +15,11 @@ export interface InstantMetricResult extends MetricBase {
   values?: [number, string][]
 }
 
+export interface ExportMetricResult extends MetricBase {
+  values: number[];
+  timestamps: number[];
+}
+
 export interface TracingData {
   message: string;
   duration_msec: number;

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -56,19 +56,23 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, isHeatmap, isAnomal
       )}
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
-          {legend.freeFormFields["__name__"]}
-          {!!freeFormFields.length && <>&#123;</>}
-          {freeFormFields.map((f, i) => (
-            <span
-              className="vm-legend-item-info__free-fields"
-              key={f.key}
-              onClick={createHandlerCopy(f.freeField)}
-              title="copy to clipboard"
-            >
-              {f.freeField}{i + 1 < freeFormFields.length && ","}
-            </span>
-          ))}
-          {!!freeFormFields.length && <>&#125;</>}
+          {legend.hasAlias ? legend.label : (
+            <>
+              {legend.freeFormFields["__name__"]}
+              {!!freeFormFields.length && <>&#123;</>}
+              {freeFormFields.map((f, i) => (
+                <span
+                  className="vm-legend-item-info__free-fields"
+                  key={f.key}
+                  onClick={createHandlerCopy(f.freeField)}
+                  title="copy to clipboard"
+                >
+                  {f.freeField}{i + 1 < freeFormFields.length && ","}
+                </span>
+              ))}
+              {!!freeFormFields.length && <>&#125;</>}
+            </>
+          )}
         </span>
       </div>
       {!isHeatmap && showStats && (

app/vmui/packages/vmui/src/components/Configurators/AdditionalSettings/AdditionalSettings.tsx

@@ -20,13 +20,17 @@ const AdditionalSettingsControls: FC<Props & {isMobile?: boolean}> = ({ isMobile
   const { autocomplete } = useQueryState();
   const queryDispatch = useQueryDispatch();
 
-  const { nocache, isTracingEnabled } = useCustomPanelState();
+  const { nocache, isTracingEnabled, reduceMemUsage } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
   const onChangeCache = () => {
     customPanelDispatch({ type: "TOGGLE_NO_CACHE" });
   };
 
+  const onChangeReduceMemUsage = () => {
+    customPanelDispatch({ type: "TOGGLE_REDUCE_MEM_USAGE" });
+  };
+
   const onChangeQueryTracing = () => {
     customPanelDispatch({ type: "TOGGLE_QUERY_TRACING" });
   };
@@ -67,12 +71,22 @@ const AdditionalSettingsControls: FC<Props & {isMobile?: boolean}> = ({ isMobile
           />
         </Tooltip>
       )}
-      <Switch
-        label={"Disable cache"}
-        value={nocache}
-        onChange={onChangeCache}
-        fullWidth={isMobile}
-      />
+      {!hideButtons?.disableCache && (
+        <Switch
+          label={"Disable cache"}
+          value={nocache}
+          onChange={onChangeCache}
+          fullWidth={isMobile}
+        />
+      )}
+      {!hideButtons?.reduceMemUsage && (
+        <Switch
+          label={"Disable deduplication"}
+          value={reduceMemUsage}
+          onChange={onChangeReduceMemUsage}
+          fullWidth={isMobile}
+        />
+      )}
       {!hideButtons?.traceQuery && (
         <Switch
           label={"Trace query"}

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx

@@ -23,6 +23,7 @@ export interface QueryEditorProps {
   stats?: QueryStats;
   label: string;
   disabled?: boolean
+  includeFunctions?: boolean;
 }
 
 const QueryEditor: FC<QueryEditorProps> = ({
@@ -35,7 +36,8 @@ const QueryEditor: FC<QueryEditorProps> = ({
   error,
   stats,
   label,
-  disabled = false
+  disabled = false,
+  includeFunctions = true
 }) => {
   const { autocompleteQuick } = useQueryState();
   const { isMobile } = useDeviceDetect();
@@ -143,6 +145,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
           anchorEl={autocompleteAnchorEl}
           caretPosition={caretPosition}
           hasHelperText={Boolean(warning || error)}
+          includeFunctions={includeFunctions}
           onSelect={handleSelect}
           onFoundOptions={handleChangeFoundOptions}
         />

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -11,6 +11,7 @@ interface QueryEditorAutocompleteProps {
   anchorEl: React.RefObject<HTMLElement>;
   caretPosition: [number, number]; // [start, end]
   hasHelperText: boolean;
+  includeFunctions: boolean;
   onSelect: (val: string, caretPosition: number) => void;
   onFoundOptions: (val: AutocompleteOptions[]) => void;
 }
@@ -20,11 +21,12 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   anchorEl,
   caretPosition,
   hasHelperText,
+  includeFunctions,
   onSelect,
   onFoundOptions
 }) => {
   const [offsetPos, setOffsetPos] = useState({ top: 0, left: 0 });
-  const metricsqlFunctions = useGetMetricsQL();
+  const metricsqlFunctions = useGetMetricsQL(includeFunctions);
 
   const values = useMemo(() => {
     if (caretPosition[0] !== caretPosition[1]) return { beforeCursor: value, afterCursor: "" };

app/vmui/packages/vmui/src/components/Main/Pagination/Pagination.tsx

@@ -0,0 +1,105 @@
+import React from "react";
+import { ArrowDownIcon } from "../Icons";
+import { useMemo } from "preact/compat";
+import classNames from "classnames";
+import "./style.scss";
+
+interface PaginationProps {
+  currentPage: number;
+  totalItems: number;
+  itemsPerPage: number;
+  onPageChange: (page: number) => void;
+  maxVisiblePages?: number;
+}
+
+const Pagination: React.FC<PaginationProps> = ({
+  currentPage,
+  totalItems,
+  itemsPerPage,
+  onPageChange,
+  maxVisiblePages = 10
+}) => {
+  const totalPages = Math.ceil(totalItems / itemsPerPage);
+  const handlePageChange = (page: number) => {
+    if (page < 1 || page > totalPages) return;
+    onPageChange(page);
+  };
+
+  const pages = useMemo(() => {
+    const pages = [];
+    if (totalPages <= maxVisiblePages) {
+      for (let i = 1; i <= totalPages; i++) {
+        pages.push(i);
+      }
+    } else {
+      const startPage = Math.max(1, currentPage - Math.floor(maxVisiblePages / 2));
+      const endPage = Math.min(totalPages, startPage + maxVisiblePages - 1);
+
+      if (startPage > 1) {
+        pages.push(1);
+        if (startPage > 2) {
+          pages.push("...");
+        }
+      }
+
+      for (let i = startPage; i <= endPage; i++) {
+        pages.push(i);
+      }
+
+      if (endPage < totalPages) {
+        if (endPage < totalPages - 1) {
+          pages.push("...");
+        }
+        pages.push(totalPages);
+      }
+    }
+    return pages;
+  }, [totalPages, currentPage, maxVisiblePages]);
+
+  const handleClickNav = (stepPage: number) => () => {
+    handlePageChange(currentPage + stepPage);
+  };
+
+  const handleClickPage = (page: number | string) => () => {
+    if (typeof page === "number") {
+      handlePageChange(page);
+    }
+  };
+
+  if (pages.length <= 1) return null;
+
+  return (
+    <div className="vm-pagination">
+      <button
+        className="vm-pagination__page vm-pagination__arrow vm-pagination__arrow_prev"
+        onClick={handleClickNav(-1)}
+        disabled={currentPage === 1}
+      >
+        <ArrowDownIcon/>
+      </button>
+      {pages.map((page, index) => (
+        <button
+          key={index}
+          onClick={handleClickPage(page)}
+          className={classNames({
+            "vm-pagination__page": true,
+            "vm-pagination__page_active": currentPage === page,
+            "vm-pagination__page_disabled": page === "..."
+          })}
+          disabled={page === "..."}
+        >
+          {page}
+        </button>
+      ))}
+      <button
+        className="vm-pagination__page vm-pagination__arrow vm-pagination__arrow_next"
+        onClick={handleClickNav(1)}
+        disabled={currentPage === totalPages}
+      >
+        <ArrowDownIcon/>
+      </button>
+    </div>
+  );
+};
+
+export default Pagination;

app/vmui/packages/vmui/src/components/Main/Pagination/PaginationControl/PaginationControl.tsx

@@ -1,52 +0,0 @@
-import React, { FC } from "preact/compat";
-import Button from "../../Button/Button";
-import { ArrowDownIcon } from "../../Icons";
-import "./style.scss";
-import useDeviceDetect from "../../../../hooks/useDeviceDetect";
-import classNames from "classnames";
-
-interface PaginationControlProps {
-  page: number;
-  length: number;
-  limit: number;
-  onChange: (page: number) => void;
-}
-
-const PaginationControl: FC<PaginationControlProps> = ({ page, length, limit, onChange }) => {
-  const { isMobile } = useDeviceDetect();
-
-  const handleChangePage = (step: number) => () => {
-    onChange(+page + step);
-    window.scrollTo(0, 0);
-  };
-
-  return (
-    <div
-      className={classNames({
-        "vm-pagination": true,
-        "vm-pagination_mobile": isMobile
-      })}
-    >
-      {page > 1 && (
-        <Button
-          variant={"text"}
-          onClick={handleChangePage(-1)}
-          startIcon={<div className="vm-pagination__icon vm-pagination__icon_prev"><ArrowDownIcon/></div>}
-        >
-          Previous
-        </Button>
-      )}
-      {length >= limit && (
-        <Button
-          variant={"text"}
-          onClick={handleChangePage(1)}
-          endIcon={<div className="vm-pagination__icon vm-pagination__icon_next"><ArrowDownIcon/></div>}
-        >
-          Next
-        </Button>
-      )}
-    </div>
-  );
-};
-
-export default PaginationControl;

app/vmui/packages/vmui/src/components/Main/Pagination/PaginationControl/style.scss

@@ -1,24 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-pagination {
-  position: sticky;
-  right: 0;
-  display: flex;
-  justify-content: flex-end;
-  gap: $padding-small;
-  padding: $padding-global 0 0;
-
-  &_mobile {
-    padding: $padding-global 0;
-  }
-
-  &__icon {
-    &_prev {
-      transform: rotate(90deg);
-    }
-
-    &_next {
-      transform: rotate(-90deg);
-    }
-  }
-}

app/vmui/packages/vmui/src/components/Main/Pagination/style.scss

@@ -0,0 +1,66 @@
+@use "../../../styles/variables" as *;
+
+.vm-pagination {
+  position: sticky;
+  left: 0;
+  display: flex;
+  justify-content: center;
+  gap: $padding-small;
+  padding: $padding-global 0;
+  font-size: $font-size;
+
+  &_mobile {
+    padding: $padding-global 0;
+  }
+
+  &__page {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 30px;
+    min-width: 30px;
+    color: $color-text;
+    padding: 0 $padding-small;
+    border-radius: $border-radius-small;
+    transition: background-color 0.3s;
+    border: 1px solid transparent;
+    cursor: pointer;
+
+    &_active {
+      background-color: $color-primary;
+      color: $color-primary-text;
+    }
+
+    &_disabled {
+      cursor: default;
+      pointer-events: none;
+      color: $color-text-disabled;
+    }
+
+    &:hover {
+      background-color: $color-hover-black;
+    }
+  }
+
+  &__arrow {
+
+    svg {
+      max-width: $font-size;
+      max-height: $font-size;
+    }
+
+    &:disabled {
+      color: $color-text-disabled;
+      cursor: default;
+      pointer-events: none;
+    }
+
+    &_prev {
+      transform: rotate(90deg);
+    }
+
+    &_next {
+      transform: rotate(-90deg);
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Table/Table.tsx

@@ -32,8 +32,7 @@ const Table = <T extends object>({ rows, columns, defaultOrderBy, defaultOrderDi
   const sortedList = useMemo(() => {
     const { startIndex, endIndex } = paginationOffset;
     return stableSort(rows as [], getComparator(orderDir, orderBy)).slice(startIndex, endIndex);
-  },
-  [rows, orderBy, orderDir, paginationOffset]);
+  }, [rows, orderBy, orderDir, paginationOffset]);
 
   const createSortHandler = (key: keyof T) => () => {
     setOrderDir((prev) => prev === "asc" && orderBy === key ? "desc" : "asc");

app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx

@@ -180,7 +180,7 @@ const GraphView: FC<GraphViewProps> = ({
     if (isAnomalyView) {
       setHideSeries(legend.map(s => s.label || "").slice(1));
     }
-  }, [data, timezone, isHistogram]);
+  }, [data, timezone, isHistogram, currentStep]);
 
   useEffect(() => {
     const tempLegend: LegendItemType[] = [];

app/vmui/packages/vmui/src/components/Views/JsonView/JsonView.tsx

@@ -12,17 +12,7 @@ export interface JsonViewProps {
 const JsonView: FC<JsonViewProps> = ({ data }) => {
   const copyToClipboard = useCopyToClipboard();
 
-  const formattedJson = useMemo(() => {
-    const space = "  ";
-    const values = data.map(item => {
-      if (Object.keys(item).length === 1) {
-        return JSON.stringify(item);
-      } else {
-        return JSON.stringify(item, null, space.length);
-      }
-    }).join(",\n").replace(/^/gm, `${space}`);
-    return `[\n${values}\n]`;
-  }, [data]);
+  const formattedJson = useMemo(() => JSON.stringify(data, null, 2), [data]);
 
   const handlerCopy = async () => {
     await copyToClipboard(formattedJson, "Formatted JSON has been copied");

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -64,7 +64,7 @@ const useLineTooltip = ({ u, metrics, series, unit, isAnomalyView }: LineTooltip
       title: groups.size > 1 && !isAnomalyView ? `Query ${group}` : "",
       dates: [date ? dayjs(date * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT) : "-"],
       value: formatPrettyNumber(value, min, max),
-      info: getMetricName(metricItem),
+      info: getMetricName(metricItem, seriesItem),
       statsFormatted: seriesItem?.statsFormatted,
       marker: `${seriesItem?.stroke}`,
     };

app/vmui/packages/vmui/src/hooks/useGetMetricsQL.tsx

@@ -48,7 +48,7 @@ const processGroups = (groups: NodeListOf<Element>): AutocompleteOptions[] => {
   }).filter(Boolean) as AutocompleteOptions[];
 };
 
-const useGetMetricsQL = () => {
+const useGetMetricsQL = (includeFunctions: boolean) => {
   const { metricsQLFunctions } = useQueryState();
   const queryDispatch = useQueryDispatch();
 
@@ -60,6 +60,7 @@ const useGetMetricsQL = () => {
   };
 
   useEffect(() => {
+    if (!includeFunctions || metricsQLFunctions.length) return;
     const fetchMarkdown = async () => {
       try {
         const resp = await fetch(MetricsQL);
@@ -70,12 +71,10 @@ const useGetMetricsQL = () => {
         console.error("Error fetching or processing the MetricsQL.md file:", e);
       }
     };
-
-    if (metricsQLFunctions.length) return;
     fetchMarkdown();
   }, []);
 
-  return metricsQLFunctions;
+  return includeFunctions ? metricsQLFunctions : [];
 };
 
 export default useGetMetricsQL;

app/vmui/packages/vmui/src/pages/CustomPanel/DisplayTypeSwitch.tsx

@@ -17,7 +17,11 @@ export const displayTypeTabs: DisplayTab[] = [
   { value: DisplayType.table, icon: <TableIcon/>, label: "Table", prometheusCode: 1 }
 ];
 
-export const DisplayTypeSwitch: FC = () => {
+interface Props {
+  tabFilter?: (tab: DisplayTab) => boolean
+}
+
+export const DisplayTypeSwitch: FC<Props> = ({ tabFilter }) => {
 
   const { displayType } = useCustomPanelState();
   const dispatch = useCustomPanelDispatch();
@@ -26,10 +30,12 @@ export const DisplayTypeSwitch: FC = () => {
     dispatch({ type: "SET_DISPLAY_TYPE", payload: newValue as DisplayType ?? displayType });
   };
 
+  const items = displayTypeTabs.filter(tabFilter ?? (() => true));
+
   return (
     <Tabs
       activeItem={displayType}
-      items={displayTypeTabs}
+      items={items}
       onChange={handleChange}
     />
   );

app/vmui/packages/vmui/src/pages/CustomPanel/DownloadReport/DownloadReport.tsx

@@ -15,6 +15,8 @@ import Alert from "../../../components/Main/Alert/Alert";
 import qs from "qs";
 import Popper from "../../../components/Main/Popper/Popper";
 import helperText from "./helperText";
+import { Link } from "react-router-dom";
+import router from "../../../router";
 
 type Props = {
   fetchUrl?: string[];
@@ -125,6 +127,15 @@ const DownloadReport: FC<Props> = ({ fetchUrl }) => {
     setStepHelper(0);
   }, [openHelper]);
 
+  const RawQueryLink = () => (
+    <Link
+      className="vm-link vm-link_underlined vm-link_colored"
+      to={router.rawQuery}
+    >
+      Raw Query
+    </Link>
+  );
+
   return (
     <>
       <Tooltip title={"Export query"}>
@@ -165,6 +176,10 @@ const DownloadReport: FC<Props> = ({ fetchUrl }) => {
                   label={"Include query trace"}
                 />
               </div>
+              <Alert variant="info">
+                If confused with the query results,
+                try viewing the raw samples for selected series in <RawQueryLink/> tab.
+              </Alert>
             </div>
             {error && <Alert variant="error">{error}</Alert>}
             <div className="vm-download-report__buttons">

app/vmui/packages/vmui/src/pages/CustomPanel/DownloadReport/style.scss

@@ -4,14 +4,15 @@
   display: grid;
   gap: $padding-large;
   padding-top: calc($padding-large - $padding-global);
-  min-width: 400px;
+  width: 700px;
+  max-width: 100%;
 
   &-settings {
     display: grid;
     gap: $padding-global;
 
     textarea {
-      min-height: 100px;
+      min-height: 200px;
     }
   }
 

app/vmui/packages/vmui/src/pages/CustomPanel/QueryConfigurator/QueryConfigurator.tsx

@@ -31,7 +31,9 @@ export interface QueryConfiguratorProps {
   setQueryErrors: Dispatch<SetStateAction<string[]>>;
   setHideError: Dispatch<SetStateAction<boolean>>;
   stats: QueryStats[];
+  label?: string;
   isLoading?: boolean;
+  includeFunctions?: boolean;
   onHideQuery?: (queries: number[]) => void
   onRunQuery: () => void;
   abortFetch?: () => void;
@@ -41,6 +43,8 @@ export interface QueryConfiguratorProps {
     autocomplete?: boolean;
     traceQuery?: boolean;
     anomalyConfig?: boolean;
+    disableCache?: boolean;
+    reduceMemUsage?: boolean;
   }
 }
 
@@ -49,7 +53,9 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
   setQueryErrors,
   setHideError,
   stats,
+  label,
   isLoading,
+  includeFunctions = true,
   onHideQuery,
   onRunQuery,
   abortFetch,
@@ -216,8 +222,9 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
             onArrowDown={createHandlerArrow(1, i)}
             onEnter={handleRunQuery}
             onChange={createHandlerChangeQuery(i)}
-            label={`Query ${stateQuery.length > 1 ? i + 1 : ""}`}
+            label={`${label || "Query"} ${stateQuery.length > 1 ? i + 1 : ""}`}
             disabled={hideQuery.includes(i)}
+            includeFunctions={includeFunctions}
           />
           {onHideQuery && (
             <Tooltip title={hideQuery.includes(i) ? "Enable query" : "Disable query"}>

app/vmui/packages/vmui/src/pages/CustomPanel/hooks/useSetQueryParams.ts

@@ -72,6 +72,16 @@ export const useSetQueryParams = () => {
         newSearchParams.set(`${group}.tenantID`, tenantId);
       }
     });
+
+    // Remove extra parameters that exceed the request size
+    const maxIndex = query.length - 1;
+    Array.from(newSearchParams.keys()).forEach(key => {
+      const match = key.match(/^g(\d+)\./);
+      if (match && parseInt(match[1], 10) > maxIndex) {
+        newSearchParams.delete(key);
+      }
+    });
+
     if (isEqualURLSearchParams(newSearchParams, searchParams) || !newSearchParams.size) return;
     setSearchParams(newSearchParams);
   }, [tenantId, displayType, query, duration, relativeTime, date, step, customStep]);

app/vmui/packages/vmui/src/pages/CustomPanel/index.tsx

@@ -85,6 +85,7 @@ const CustomPanel: FC = () => {
         onHideQuery={handleHideQuery}
         onRunQuery={handleRunQuery}
         abortFetch={abortFetch}
+        hideButtons={{ reduceMemUsage: true }}
       />
       <CustomPanelTraces
         traces={traces}

app/vmui/packages/vmui/src/pages/ExploreAnomaly/ExploreAnomaly.tsx

@@ -87,7 +87,14 @@ const ExploreAnomaly: FC = () => {
         setHideError={setHideError}
         stats={queryStats}
         onRunQuery={handleRunQuery}
-        hideButtons={{ addQuery: true, prettify: false, autocomplete: false, traceQuery: true, anomalyConfig: true }}
+        hideButtons={{
+          addQuery: true,
+          prettify: false,
+          autocomplete: false,
+          traceQuery: true,
+          anomalyConfig: true,
+          reduceMemUsage: true,
+        }}
       />
       {isLoading && <Spinner/>}
       {(!hideError && error) && <Alert variant="error">{error}</Alert>}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -54,7 +54,7 @@ const ExploreLogs: FC = () => {
     fetchLogs(newPeriod).then((isSuccess) => {
       isSuccess && !hideChart && fetchLogHits(newPeriod);
     }).catch(e => e);
-    setSearchParamsFromKeys( {
+    setSearchParamsFromKeys({
       query,
       "g0.range_input": duration,
       "g0.end_input": newPeriod.date,

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx

@@ -1,22 +1,23 @@
 import React, { FC, useState, useMemo, useRef } from "preact/compat";
-import JsonView from "../../../components/Views/JsonView/JsonView";
 import { CodeIcon, ListIcon, TableIcon } from "../../../components/Main/Icons";
 import Tabs from "../../../components/Main/Tabs/Tabs";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import { Logs } from "../../../api/types";
-import dayjs from "dayjs";
-import { useTimeState } from "../../../state/time/TimeStateContext";
 import useStateSearchParams from "../../../hooks/useStateSearchParams";
 import useSearchParamsFromObject from "../../../hooks/useSearchParamsFromObject";
 import TableSettings from "../../../components/Table/TableSettings/TableSettings";
 import useBoolean from "../../../hooks/useBoolean";
 import TableLogs from "./TableLogs";
 import GroupLogs from "../GroupLogs/GroupLogs";
-import { DATE_TIME_FORMAT } from "../../../constants/date";
-import { marked } from "marked";
+import JsonView from "../../../components/Views/JsonView/JsonView";
 import LineLoader from "../../../components/Main/LineLoader/LineLoader";
+import SelectLimit from "../../../components/Main/Pagination/SelectLimit/SelectLimit";
+
+const MemoizedTableLogs = React.memo(TableLogs);
+const MemoizedGroupLogs = React.memo(GroupLogs);
+const MemoizedJsonView = React.memo(JsonView);
 
 export interface ExploreLogBodyProps {
   data: Logs[];
@@ -37,38 +38,35 @@ const tabs = [
 
 const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, isLoading }) => {
   const { isMobile } = useDeviceDetect();
-  const { timezone } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
   const groupSettingsRef = useRef<HTMLDivElement>(null);
 
   const [activeTab, setActiveTab] = useStateSearchParams(DisplayType.group, "view");
   const [displayColumns, setDisplayColumns] = useState<string[]>([]);
+  const [rowsPerPage, setRowsPerPage] = useStateSearchParams(1000, "rows_per_page");
   const { value: tableCompact, toggle: toggleTableCompact } = useBoolean(false);
 
-  const logs = useMemo(() => data.map((item) => ({
-    ...item,
-    _vmui_time: item._time ? dayjs(item._time).tz().format(`${DATE_TIME_FORMAT}.SSS`) : "",
-    _vmui_data: JSON.stringify(item, null, 2),
-    _vmui_markdown: item._msg ? marked(item._msg.replace(/```/g, "\n```\n")) as string : ""
-  })) as Logs[], [data, timezone]);
-
   const columns = useMemo(() => {
-    if (!logs?.length) return [];
-    const hideColumns = ["_vmui_data", "_vmui_time", "_vmui_markdown"];
+    if (!data?.length) return [];
     const keys = new Set<string>();
-    for (const item of logs) {
+    for (const item of data) {
       for (const key in item) {
         keys.add(key);
       }
     }
-    return Array.from(keys).filter((col) => !hideColumns.includes(col));
-  }, [logs]);
+    return Array.from(keys);
+  }, [data]);
 
   const handleChangeTab = (view: string) => {
     setActiveTab(view as DisplayType);
     setSearchParamsFromKeys({ view });
   };
 
+  const handleSetRowsPerPage = (limit: number) => {
+    setRowsPerPage(limit);
+    setSearchParamsFromKeys({ rows_per_page: limit });
+  };
+
   return (
     <div
       className={classNames({
@@ -97,6 +95,10 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, isLoading }) => {
         </div>
         {activeTab === DisplayType.table && (
           <div className="vm-explore-logs-body-header__settings">
+            <SelectLimit
+              limit={rowsPerPage}
+              onChange={handleSetRowsPerPage}
+            />
             <TableSettings
               columns={columns}
               selectedColumns={displayColumns}
@@ -124,22 +126,22 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, isLoading }) => {
         {!!data.length && (
           <>
             {activeTab === DisplayType.table && (
-              <TableLogs
-                logs={logs}
+              <MemoizedTableLogs
+                logs={data}
                 displayColumns={displayColumns}
                 tableCompact={tableCompact}
                 columns={columns}
+                rowsPerPage={Number(rowsPerPage)}
               />
             )}
             {activeTab === DisplayType.group && (
-              <GroupLogs
-                logs={logs}
-                columns={columns}
+              <MemoizedGroupLogs
+                logs={data}
                 settingsRef={groupSettingsRef}
               />
             )}
             {activeTab === DisplayType.json && (
-              <JsonView data={data}/>
+              <MemoizedJsonView data={data}/>
             )}
           </>
         )}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/TableLogs.tsx

@@ -1,58 +1,94 @@
-import React, { FC, useMemo } from "preact/compat";
+import React, { FC, useMemo, useRef, useState } from "preact/compat";
 import "./style.scss";
 import Table from "../../../components/Table/Table";
 import { Logs } from "../../../api/types";
+import Pagination from "../../../components/Main/Pagination/Pagination";
+import { useEffect } from "react";
 
 interface TableLogsProps {
   logs: Logs[];
   displayColumns: string[];
   tableCompact: boolean;
   columns: string[];
+  rowsPerPage: number;
 }
 
-const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, columns }) => {
-  const getColumnClass = (key: string) => {
-    switch (key) {
-      case "_time":
-        return "vm-table-cell_logs-time";
-      case "_vmui_data":
-        return "vm-table-cell_logs vm-table-cell_pre";
-      default:
-        return "vm-table-cell_logs";
-    }
-  };
+const getColumnClass = (key: string) => {
+  switch (key) {
+    case "_time":
+      return "vm-table-cell_logs-time";
+    default:
+      return "vm-table-cell_logs";
+  }
+};
+
+const compactColumns = [{
+  key: "_vmui_data",
+  title: "Data",
+  className: "vm-table-cell_logs vm-table-cell_pre"
+}];
+
+const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, columns, rowsPerPage }) => {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [page, setPage] = useState(1);
+
+  const rows = useMemo(() => {
+    return logs.map((log) => {
+      const _vmui_data = JSON.stringify(log, null, 2);
+      return { ...log, _vmui_data };
+    }) as Logs[];
+  }, [logs]);
 
   const tableColumns = useMemo(() => {
-    if (tableCompact) {
-      return [{
-        key: "_vmui_data",
-        title: "Data",
-        className: getColumnClass("_vmui_data")
-      }];
-    }
     return columns.map((key) => ({
       key: key as keyof Logs,
       title: key,
       className: getColumnClass(key),
     }));
-  }, [tableCompact, columns]);
+  }, [columns]);
 
 
   const filteredColumns = useMemo(() => {
-    if (tableCompact) return tableColumns;
+    if (tableCompact) return compactColumns;
     if (!displayColumns?.length) return [];
     return tableColumns.filter(c => displayColumns.includes(c.key as string));
   }, [tableColumns, displayColumns, tableCompact]);
 
+  const paginationOffset = useMemo(() => {
+    const startIndex = (page - 1) * rowsPerPage;
+    const endIndex = startIndex + rowsPerPage;
+    return { startIndex, endIndex };
+  }, [page, rowsPerPage]);
+
+  const handlePageChange = (newPage: number) => {
+    setPage(newPage);
+    if (containerRef.current) {
+      const y = containerRef.current.getBoundingClientRect().top + window.scrollY - 50;
+      window.scrollTo({ top: y });
+    }
+  };
+
+  useEffect(() => {
+    setPage(1);
+  }, [logs, rowsPerPage]);
+
   return (
     <>
-      <Table
-        rows={logs}
-        columns={filteredColumns}
-        defaultOrderBy={"_time"}
-        defaultOrderDir={"desc"}
-        copyToClipboard={"_vmui_data"}
-        paginationOffset={{ startIndex: 0, endIndex: Infinity }}
+      <div ref={containerRef}>
+        <Table
+          rows={rows}
+          columns={filteredColumns}
+          defaultOrderBy={"_time"}
+          defaultOrderDir={"desc"}
+          copyToClipboard={"_vmui_data"}
+          paginationOffset={paginationOffset}
+        />
+      </div>
+      <Pagination
+        currentPage={page}
+        totalItems={rows.length}
+        itemsPerPage={rowsPerPage}
+        onPageChange={handlePageChange}
       />
     </>
   );

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx

@@ -20,13 +20,12 @@ import { getStreamPairs } from "../../../utils/logs";
 
 const WITHOUT_GROUPING = "No Grouping";
 
-interface TableLogsProps {
+interface Props {
   logs: Logs[];
-  columns: string[];
   settingsRef: React.RefObject<HTMLElement>;
 }
 
-const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
+const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
   const { isDarkTheme } = useAppState();
   const copyToClipboard = useCopyToClipboard();
   const [searchParams, setSearchParams] = useSearchParams();
@@ -46,19 +45,21 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
   const expandAll = useMemo(() => expandGroups.every(Boolean), [expandGroups]);
 
   const logsKeys = useMemo(() => {
-    const excludeKeys = ["_msg", "_time", "_vmui_time", "_vmui_data", "_vmui_markdown"];
+    const excludeKeys = ["_msg", "_time"];
     const uniqKeys = Array.from(new Set(logs.map(l => Object.keys(l)).flat()));
-    const keys = [WITHOUT_GROUPING, ...uniqKeys.filter(k => !excludeKeys.includes(k))];
+    return [WITHOUT_GROUPING, ...uniqKeys.filter(k => !excludeKeys.includes(k))];
+  }, [logs]);
 
-    if (!searchKey) return keys;
+  const filteredLogsKeys = useMemo(() => {
+    if (!searchKey) return logsKeys;
     try {
       const regexp = new RegExp(searchKey, "i");
-      const found = keys.filter((item) => regexp.test(item));
-      return found.sort((a,b) => (a.match(regexp)?.index || 0) - (b.match(regexp)?.index || 0));
+      return logsKeys.filter(item => regexp.test(item))
+        .sort((a, b) => (a.match(regexp)?.index || 0) - (b.match(regexp)?.index || 0));
     } catch (e) {
       return [];
     }
-  }, [logs, searchKey]);
+  }, [logsKeys, searchKey]);
 
   const groupData = useMemo(() => {
     return groupByMultipleKeys(logs, [groupBy]).map((item) => {
@@ -94,16 +95,15 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
 
   const handleToggleExpandAll = useCallback(() => {
     setExpandGroups(new Array(groupData.length).fill(!expandAll));
-  }, [expandAll]);
+  }, [expandAll, groupData.length]);
 
-  const handleChangeExpand = (i: number) => (value: boolean) => {
+  const handleChangeExpand = useCallback((i: number) => (value: boolean) => {
     setExpandGroups((prev) => {
       const newExpandGroups = [...prev];
       newExpandGroups[i] = value;
       return newExpandGroups;
     });
-
-  };
+  }, []);
 
   useEffect(() => {
     if (copied === null) return;
@@ -170,7 +170,7 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
           <Tooltip title={expandAll ? "Collapse All" : "Expand All"}>
             <Button
               variant="text"
-              startIcon={expandAll ? <CollapseIcon/> : <ExpandIcon/> }
+              startIcon={expandAll ? <CollapseIcon/> : <ExpandIcon/>}
               onClick={handleToggleExpandAll}
               ariaLabel={expandAll ? "Collapse All" : "Expand All"}
             />
@@ -179,7 +179,7 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
             <div ref={optionsButtonRef}>
               <Button
                 variant="text"
-                startIcon={<StorageIcon/> }
+                startIcon={<StorageIcon/>}
                 onClick={toggleOpenOptions}
                 ariaLabel={"Group by"}
               />
@@ -201,7 +201,7 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
                     type="search"
                   />
                 </div>
-                {logsKeys.map(id => (
+                {filteredLogsKeys.map(id => (
                   <div
                     className={classNames({
                       "vm-list-item": true,

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsFieldRow.tsx

@@ -0,0 +1,54 @@
+import React, { FC, memo, useCallback, useEffect, useState } from "preact/compat";
+import Tooltip from "../../../components/Main/Tooltip/Tooltip";
+import Button from "../../../components/Main/Button/Button";
+import { CopyIcon } from "../../../components/Main/Icons";
+import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
+
+interface Props {
+  field: string;
+  value: string;
+}
+
+const GroupLogsFieldRow: FC<Props> = ({ field, value }) => {
+  const copyToClipboard = useCopyToClipboard();
+  const [copied, setCopied] = useState<boolean>(false);
+
+  const handleCopy = useCallback(async () => {
+    if (copied) return;
+    try {
+      await copyToClipboard(`${field}: "${value}"`);
+      setCopied(true);
+    } catch (e) {
+      console.error(e);
+    }
+  }, [copied, copyToClipboard]);
+
+  useEffect(() => {
+    if (copied === null) return;
+    const timeout = setTimeout(() => setCopied(false), 2000);
+    return () => clearTimeout(timeout);
+  }, [copied]);
+
+  return (
+    <tr className="vm-group-logs-row-fields-item">
+      <td className="vm-group-logs-row-fields-item-controls">
+        <div className="vm-group-logs-row-fields-item-controls__wrapper">
+          <Tooltip title={copied ? "Copied" : "Copy to clipboard"}>
+            <Button
+              variant="text"
+              color="gray"
+              size="small"
+              startIcon={<CopyIcon/>}
+              onClick={handleCopy}
+              ariaLabel="copy to clipboard"
+            />
+          </Tooltip>
+        </div>
+      </td>
+      <td className="vm-group-logs-row-fields-item__key">{field}</td>
+      <td className="vm-group-logs-row-fields-item__value">{value}</td>
+    </tr>
+  );
+};
+
+export default memo(GroupLogsFieldRow);

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx

@@ -1,13 +1,15 @@
-import React, { FC, useEffect, useMemo, useState } from "preact/compat";
+import React, { FC, memo, useMemo } from "preact/compat";
 import { Logs } from "../../../api/types";
 import "./style.scss";
 import useBoolean from "../../../hooks/useBoolean";
-import Button from "../../../components/Main/Button/Button";
-import Tooltip from "../../../components/Main/Tooltip/Tooltip";
-import { ArrowDownIcon, CopyIcon } from "../../../components/Main/Icons";
-import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
+import { ArrowDownIcon } from "../../../components/Main/Icons";
 import classNames from "classnames";
 import { useLogsState } from "../../../state/logsPanel/LogsStateContext";
+import dayjs from "dayjs";
+import { DATE_TIME_FORMAT } from "../../../constants/date";
+import { useTimeState } from "../../../state/time/TimeStateContext";
+import GroupLogsFieldRow from "./GroupLogsFieldRow";
+import { marked } from "marked";
 
 interface Props {
   log: Logs;
@@ -20,40 +22,31 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
   } = useBoolean(false);
 
   const { markdownParsing } = useLogsState();
+  const { timezone } = useTimeState();
 
-  const excludeKeys = ["_msg", "_vmui_time", "_vmui_data", "_vmui_markdown"];
-  const fields = Object.entries(log).filter(([key]) => !excludeKeys.includes(key));
+  const formattedTime = useMemo(() => {
+    if (!log._time) return "";
+    return dayjs(log._time).tz().format(`${DATE_TIME_FORMAT}.SSS`);
+  }, [log._time, timezone]);
+
+  const formattedMarkdown = useMemo(() => {
+    if (!markdownParsing || !log._msg) return "";
+    return marked(log._msg.replace(/```/g, "\n```\n")) as string;
+  }, [log._msg, markdownParsing]);
+
+  const fields = useMemo(() => Object.entries(log).filter(([key]) => key !== "_msg"), [log]);
   const hasFields = fields.length > 0;
 
   const displayMessage = useMemo(() => {
     if (log._msg) return log._msg;
     if (!hasFields) return;
-    const dataObject = fields.reduce<{[key: string]: string}>((obj, [key, value]) => {
+    const dataObject = fields.reduce<{ [key: string]: string }>((obj, [key, value]) => {
       obj[key] = value;
       return obj;
     }, {});
     return JSON.stringify(dataObject);
   }, [log, fields, hasFields]);
 
-  const copyToClipboard = useCopyToClipboard();
-  const [copied, setCopied] = useState<number | null>(null);
-
-  const createCopyHandler = (copyValue:  string, rowIndex: number) => async () => {
-    if (copied === rowIndex) return;
-    try {
-      await copyToClipboard(copyValue);
-      setCopied(rowIndex);
-    } catch (e) {
-      console.error(e);
-    }
-  };
-
-  useEffect(() => {
-    if (copied === null) return;
-    const timeout = setTimeout(() => setCopied(null), 2000);
-    return () => clearTimeout(timeout);
-  }, [copied]);
-
   return (
     <div className="vm-group-logs-row">
       <div
@@ -74,10 +67,10 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
         <div
           className={classNames({
             "vm-group-logs-row-content__time": true,
-            "vm-group-logs-row-content__time_missing": !log._vmui_time
+            "vm-group-logs-row-content__time_missing": !formattedTime
           })}
         >
-          {log._vmui_time || "timestamp missing"}
+          {formattedTime || "timestamp missing"}
         </div>
         <div
           className={classNames({
@@ -85,7 +78,7 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
             "vm-group-logs-row-content__msg_empty-msg": !log._msg,
             "vm-group-logs-row-content__msg_missing": !displayMessage
           })}
-          dangerouslySetInnerHTML={markdownParsing && log._vmui_markdown ? { __html: log._vmui_markdown } : undefined}
+          dangerouslySetInnerHTML={(markdownParsing && formattedMarkdown) ? { __html: formattedMarkdown } : undefined}
         >
           {displayMessage || "-"}
         </div>
@@ -94,28 +87,12 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
         <div className="vm-group-logs-row-fields">
           <table>
             <tbody>
-              {fields.map(([key, value], i) => (
-                <tr
+              {fields.map(([key, value]) => (
+                <GroupLogsFieldRow
                   key={key}
-                  className="vm-group-logs-row-fields-item"
-                >
-                  <td className="vm-group-logs-row-fields-item-controls">
-                    <div className="vm-group-logs-row-fields-item-controls__wrapper">
-                      <Tooltip title={copied === i ? "Copied" : "Copy to clipboard"}>
-                        <Button
-                          variant="text"
-                          color="gray"
-                          size="small"
-                          startIcon={<CopyIcon/>}
-                          onClick={createCopyHandler(`${key}: "${value}"`, i)}
-                          ariaLabel="copy to clipboard"
-                        />
-                      </Tooltip>
-                    </div>
-                  </td>
-                  <td className="vm-group-logs-row-fields-item__key">{key}</td>
-                  <td className="vm-group-logs-row-fields-item__value">{value}</td>
-                </tr>
+                  field={key}
+                  value={value}
+                />
               ))}
             </tbody>
           </table>
@@ -125,4 +102,4 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
   );
 };
 
-export default GroupLogsItem;
+export default memo(GroupLogsItem);

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogs.ts

@@ -9,7 +9,7 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
   const [searchParams] = useSearchParams();
 
   const [logs, setLogs] = useState<Logs[]>([]);
-  const [isLoading, setIsLoading] = useState<{[key: number]: boolean;}>([]);
+  const [isLoading, setIsLoading] = useState<{ [key: number]: boolean }>({});
   const [error, setError] = useState<ErrorTypes | string>();
   const abortControllerRef = useRef(new AbortController());
 
@@ -33,8 +33,9 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
 
   const parseLineToJSON = (line: string): Logs | null => {
     try {
-      return JSON.parse(line);
+      return line && JSON.parse(line);
     } catch (e) {
+      console.error(`Failed to parse "${line}" to JSON\n`, e);
       return null;
     }
   };
@@ -56,23 +57,25 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
       if (!response.ok || !response.body) {
         setError(text);
         setLogs([]);
-        setIsLoading(prev => ({ ...prev, [id]: false }));
         return false;
       }
 
-      const lines = text.split("\n").filter(line => line).slice(0, limit);
-      const data = lines.map(parseLineToJSON).filter(line => line) as Logs[];
+      const data = text.split("\n", limit).map(parseLineToJSON).filter(line => line) as Logs[];
       setLogs(data);
-      setIsLoading(prev => ({ ...prev, [id]: false }));
       return true;
     } catch (e) {
-      setIsLoading(prev => ({ ...prev, [id]: false }));
       if (e instanceof Error && e.name !== "AbortError") {
         setError(String(e));
         console.error(e);
         setLogs([]);
       }
       return false;
+    } finally {
+      setIsLoading(prev => {
+        // Remove the `id` key from `isLoading` when its value becomes `false`
+        const { [id]: _, ...rest } = prev;
+        return rest;
+      });
     }
   }, [url, query, limit, searchParams]);
 

app/vmui/packages/vmui/src/pages/RawQueryPage/hooks/useFetchExport.ts

@@ -0,0 +1,152 @@
+import { Dispatch, SetStateAction, useCallback, useEffect, useMemo, useRef, useState } from "preact/compat";
+import { MetricBase, MetricResult, ExportMetricResult } from "../../../api/types";
+import { ErrorTypes, SeriesLimits } from "../../../types";
+import { useQueryState } from "../../../state/query/QueryStateContext";
+import { useTimeState } from "../../../state/time/TimeStateContext";
+import { useAppState } from "../../../state/common/StateContext";
+import { useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
+import { isValidHttpUrl } from "../../../utils/url";
+import { getExportDataUrl } from "../../../api/query-range";
+
+interface FetchQueryParams {
+  hideQuery?: number[];
+  showAllSeries?: boolean;
+}
+
+interface FetchQueryReturn {
+  fetchUrl?: string[],
+  isLoading: boolean,
+  data?: MetricResult[],
+  error?: ErrorTypes | string,
+  queryErrors: (ErrorTypes | string)[],
+  setQueryErrors: Dispatch<SetStateAction<string[]>>,
+  warning?: string,
+  abortFetch: () => void
+}
+
+const parseLineToJSON = (line: string): ExportMetricResult | null => {
+  try {
+    return JSON.parse(line);
+  } catch (e) {
+    return null;
+  }
+};
+
+export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams): FetchQueryReturn => {
+  const { query } = useQueryState();
+  const { period } = useTimeState();
+  const { displayType, reduceMemUsage, seriesLimits: stateSeriesLimits } = useCustomPanelState();
+  const { serverUrl } = useAppState();
+
+  const [isLoading, setIsLoading] = useState(false);
+  const [data, setData] = useState<MetricResult[]>();
+  const [error, setError] = useState<ErrorTypes | string>();
+  const [queryErrors, setQueryErrors] = useState<string[]>([]);
+  const [warning, setWarning] = useState<string>();
+
+  const abortControllerRef = useRef(new AbortController());
+
+  const fetchUrl = useMemo(() => {
+    setError("");
+    setQueryErrors([]);
+    if (!period) return;
+    if (!serverUrl) {
+      setError(ErrorTypes.emptyServer);
+    } else if (query.every(q => !q.trim())) {
+      setQueryErrors(query.map(() => ErrorTypes.validQuery));
+    } else if (isValidHttpUrl(serverUrl)) {
+      const updatedPeriod = { ...period };
+      return query.map(q => getExportDataUrl(serverUrl, q, updatedPeriod, reduceMemUsage));
+    } else {
+      setError(ErrorTypes.validServer);
+    }
+  }, [serverUrl, period, hideQuery, reduceMemUsage]);
+
+  const fetchData = useCallback(async ( { fetchUrl, stateSeriesLimits, showAllSeries }: {
+    fetchUrl: string[];
+    stateSeriesLimits: SeriesLimits;
+    showAllSeries?: boolean;
+  }) => {
+    abortControllerRef.current.abort();
+    abortControllerRef.current = new AbortController();
+    const { signal } = abortControllerRef.current;
+    setIsLoading(true);
+    try {
+      const tempData: MetricBase[] = [];
+      const seriesLimit = showAllSeries ? Infinity : +stateSeriesLimits[displayType] || Infinity;
+      let counter = 1;
+      let totalLength = 0;
+
+      for await (const url of fetchUrl) {
+
+        const isHideQuery = hideQuery?.includes(counter - 1);
+        if (isHideQuery) {
+          setQueryErrors(prev => [...prev, ""]);
+          counter++;
+          continue;
+        }
+
+        const response = await fetch(url, { signal });
+        const text = await response.text();
+
+        if (!response.ok || !response.body) {
+          tempData.push({ metric: {}, values: [], group: counter } as MetricBase);
+          setError(text);
+          setQueryErrors(prev => [...prev, `${text}`]);
+        } else {
+          setQueryErrors(prev => [...prev, ""]);
+          const freeTempSize = seriesLimit - tempData.length;
+          const lines = text.split("\n").filter(line => line);
+          const lineLimited = lines.slice(0, freeTempSize).sort();
+          lineLimited.forEach((line: string) => {
+            const jsonLine = parseLineToJSON(line);
+            if (!jsonLine) return;
+            tempData.push({
+              group: counter,
+              metric: jsonLine.metric,
+              values: jsonLine.values.map((value, index) => [(jsonLine.timestamps[index]/1000), value]),
+            } as MetricBase);
+          });
+          totalLength += lines.length;
+        }
+
+        counter++;
+      }
+      const limitText = `Showing ${tempData.length} series out of ${totalLength} series due to performance reasons. Please narrow down the query, so it returns less series`;
+      setWarning(totalLength > seriesLimit ? limitText : "");
+      setData(tempData as MetricResult[]);
+      setIsLoading(false);
+    } catch (e) {
+      setIsLoading(false);
+      if (e instanceof Error && e.name !== "AbortError") {
+        setError(error);
+        console.error(e);
+      }
+    }
+  }, [displayType, hideQuery]);
+
+  const abortFetch = useCallback(() => {
+    abortControllerRef.current.abort();
+    setData([]);
+  }, [abortControllerRef]);
+
+  useEffect(() => {
+    if (!fetchUrl?.length) return;
+    const timer = setTimeout(fetchData, 400, { fetchUrl, stateSeriesLimits, showAllSeries });
+    return () => {
+      abortControllerRef.current?.abort();
+      clearTimeout(timer);
+    };
+  }, [fetchUrl, stateSeriesLimits, showAllSeries]);
+
+  return {
+    fetchUrl,
+    isLoading,
+    data,
+    error,
+    queryErrors,
+    setQueryErrors,
+    warning,
+    abortFetch,
+  };
+};

app/vmui/packages/vmui/src/pages/RawQueryPage/index.tsx

@@ -0,0 +1,162 @@
+import React, { FC, useState } from "preact/compat";
+import LineLoader from "../../components/Main/LineLoader/LineLoader";
+import { useCustomPanelState } from "../../state/customPanel/CustomPanelStateContext";
+import { useQueryState } from "../../state/query/QueryStateContext";
+import "../CustomPanel/style.scss";
+import Alert from "../../components/Main/Alert/Alert";
+import classNames from "classnames";
+import useDeviceDetect from "../../hooks/useDeviceDetect";
+import { useRef } from "react";
+import CustomPanelTabs from "../CustomPanel/CustomPanelTabs";
+import { DisplayTypeSwitch } from "../CustomPanel/DisplayTypeSwitch";
+import QueryConfigurator from "../CustomPanel/QueryConfigurator/QueryConfigurator";
+import WarningLimitSeries from "../CustomPanel/WarningLimitSeries/WarningLimitSeries";
+import { useFetchExport } from "./hooks/useFetchExport";
+import { useSetQueryParams } from "../CustomPanel/hooks/useSetQueryParams";
+import { DisplayType } from "../../types";
+import Hyperlink from "../../components/Main/Hyperlink/Hyperlink";
+import { CloseIcon } from "../../components/Main/Icons";
+import Button from "../../components/Main/Button/Button";
+
+const RawSamplesLink = () => (
+  <Hyperlink
+    href="https://docs.victoriametrics.com/keyconcepts/#raw-samples"
+    underlined
+  >
+    raw samples
+  </Hyperlink>
+);
+
+const QueryDataLink = () => (
+  <Hyperlink
+    underlined
+    href="https://docs.victoriametrics.com/keyconcepts/#query-data"
+  >
+    Query API
+  </Hyperlink>
+);
+
+const TimeSeriesSelectorLink = () => (
+  <Hyperlink
+    underlined
+    href="https://docs.victoriametrics.com/keyconcepts/#filtering"
+  >
+    time series selector
+  </Hyperlink>
+);
+
+const RawQueryPage: FC = () => {
+  useSetQueryParams();
+  const { isMobile } = useDeviceDetect();
+
+  const { displayType } = useCustomPanelState();
+  const { query } = useQueryState();
+
+  const [hideQuery, setHideQuery] = useState<number[]>([]);
+  const [hideError, setHideError] = useState(!query[0]);
+  const [showAllSeries, setShowAllSeries] = useState(false);
+  const [showPageDescription, setShowPageDescription] = useState(true);
+
+  const {
+    data,
+    error,
+    isLoading,
+    warning,
+    queryErrors,
+    setQueryErrors,
+    abortFetch,
+  } = useFetchExport({ hideQuery, showAllSeries });
+
+  const controlsRef = useRef<HTMLDivElement>(null);
+
+  const showError = !hideError && error;
+
+  const handleHideQuery = (queries: number[]) => {
+    setHideQuery(queries);
+  };
+
+  const handleRunQuery = () => {
+    setHideError(false);
+  };
+
+  const handleHidePageDescription = () => {
+    setShowPageDescription(false);
+  };
+
+  return (
+    <div
+      className={classNames({
+        "vm-custom-panel": true,
+        "vm-custom-panel_mobile": isMobile,
+      })}
+    >
+      <QueryConfigurator
+        label={"Time series selector"}
+        queryErrors={!hideError ? queryErrors : []}
+        setQueryErrors={setQueryErrors}
+        setHideError={setHideError}
+        stats={[]}
+        isLoading={isLoading}
+        onHideQuery={handleHideQuery}
+        onRunQuery={handleRunQuery}
+        abortFetch={abortFetch}
+        hideButtons={{ traceQuery: true, disableCache: true }}
+        includeFunctions={false}
+      />
+      {showPageDescription && (
+        <Alert variant="info">
+          <div className="vm-explore-metrics-header-description">
+            <p>
+              This page provides a dedicated view for querying and displaying <RawSamplesLink/> from VictoriaMetrics.
+              It expects only <TimeSeriesSelectorLink/> as a query argument.
+              Users often assume that the <QueryDataLink/> returns data exactly as stored,
+              but data samples and timestamps may be modified by the API.
+            </p>
+            <Button
+              variant="text"
+              size="small"
+              startIcon={<CloseIcon/>}
+              onClick={handleHidePageDescription}
+              ariaLabel="close tips"
+            />
+          </div>
+        </Alert>
+      )}
+      {showError && <Alert variant="error">{error}</Alert>}
+      {warning && (
+        <WarningLimitSeries
+          warning={warning}
+          query={query}
+          onChange={setShowAllSeries}
+        />
+      )}
+      <div
+        className={classNames({
+          "vm-custom-panel-body": true,
+          "vm-custom-panel-body_mobile": isMobile,
+          "vm-block": true,
+          "vm-block_mobile": isMobile,
+        })}
+      >
+        {isLoading && <LineLoader/>}
+        <div
+          className="vm-custom-panel-body-header"
+          ref={controlsRef}
+        >
+          <div className="vm-custom-panel-body-header__tabs">
+            <DisplayTypeSwitch tabFilter={(tab) => (tab.value !== DisplayType.table)}/>
+          </div>
+        </div>
+        <CustomPanelTabs
+          graphData={data}
+          liveData={data}
+          isHistogram={false}
+          displayType={displayType}
+          controlsRef={controlsRef}
+        />
+      </div>
+    </div>
+  );
+};
+
+export default RawQueryPage;

app/vmui/packages/vmui/src/router/index.ts

@@ -15,6 +15,7 @@ const router = {
   icons: "/icons",
   anomaly: "/anomaly",
   query: "/query",
+  rawQuery: "/raw-query",
   downsamplingDebug: "/downsampling-filters-debug",
   retentionDebug: "/retention-filters-debug",
 };
@@ -45,11 +46,15 @@ const routerOptionsDefault = {
   }
 };
 
-export const routerOptions: {[key: string]: RouterOptions} = {
+export const routerOptions: { [key: string]: RouterOptions } = {
   [router.home]: {
     title: "Query",
     ...routerOptionsDefault
   },
+  [router.rawQuery]: {
+    title: "Raw query",
+    ...routerOptionsDefault
+  },
   [router.metrics]: {
     title: "Explore Prometheus metrics",
     header: {

app/vmui/packages/vmui/src/router/navigation.ts

@@ -68,6 +68,7 @@ export const getDefaultNavigation = ({
   showAlertLink,
 }: NavigationConfig): NavigationItem[] => [
   { value: router.home },
+  { value: router.rawQuery },
   { label: "Explore", submenu: getExploreNav() },
   { label: "Tools", submenu: getToolsNav(isEnterpriseLicense) },
   { value: router.dashboards, hide: !showPredefinedDashboards },

app/vmui/packages/vmui/src/state/customPanel/reducer.ts

@@ -10,6 +10,7 @@ export interface CustomPanelState {
   isTracingEnabled: boolean;
   seriesLimits: SeriesLimits
   tableCompact: boolean;
+  reduceMemUsage: boolean;
 }
 
 export type CustomPanelAction =
@@ -18,6 +19,7 @@ export type CustomPanelAction =
   | { type: "TOGGLE_NO_CACHE"}
   | { type: "TOGGLE_QUERY_TRACING" }
   | { type: "TOGGLE_TABLE_COMPACT" }
+  | { type: "TOGGLE_REDUCE_MEM_USAGE"}
 
 export const getInitialDisplayType = () => {
   const queryTab = getQueryStringValue("g0.tab", 0) as string;
@@ -33,6 +35,7 @@ export const initialCustomPanelState: CustomPanelState = {
   isTracingEnabled: false,
   seriesLimits: limitsStorage ? JSON.parse(limitsStorage) : DEFAULT_MAX_SERIES,
   tableCompact: getFromStorage("TABLE_COMPACT") as boolean || false,
+  reduceMemUsage: false
 };
 
 export function reducer(state: CustomPanelState, action: CustomPanelAction): CustomPanelState {
@@ -65,6 +68,12 @@ export function reducer(state: CustomPanelState, action: CustomPanelAction): Cus
         ...state,
         tableCompact: !state.tableCompact
       };
+    case "TOGGLE_REDUCE_MEM_USAGE":
+      saveToStorage("TABLE_COMPACT", !state.reduceMemUsage);
+      return {
+        ...state,
+        reduceMemUsage: !state.reduceMemUsage
+      };
     default:
       throw new Error();
   }

app/vmui/packages/vmui/src/types/uplot.ts

@@ -23,6 +23,7 @@ export interface SeriesItem extends Series {
     median: number;
     forecast?: ForecastType | null;
     forecastGroup?: string;
+    hasAlias?: boolean;
 }
 
 export interface HideSeriesArgs {
@@ -45,6 +46,7 @@ export interface LegendItemType {
     freeFormFields: {[key: string]: string};
     statsFormatted: SeriesItemStatsFormatted;
     median: number
+    hasAlias: boolean;
 }
 
 export interface BarSeriesItem {

app/vmui/packages/vmui/src/utils/metric.ts

@@ -2,16 +2,23 @@ import { MetricBase } from "../api/types";
 
 export const getNameForMetric = (result: MetricBase, alias?: string, showQueryNum = true): string => {
   const { __name__, ...freeFormFields } = result.metric;
-  const name = alias || `${showQueryNum ? `[Query ${result.group}] ` : ""}${__name__ || ""}`;
-  if (Object.keys(freeFormFields).length == 0) {
-    if (!name) {
-      return "value";
-    }
-    return name;
+  const queryPrefix = showQueryNum ? `[Query ${result.group}] ` : "";
+
+  if (alias) {
+    return alias.replace(/\{\{(\w+)}}/g, (_, key) => result.metric[key] || "");
   }
-  return `${name}{${Object.entries(freeFormFields).map(e =>
-    `${e[0]}=${JSON.stringify(e[1])}`
-  ).join(", ")}}`;
+
+  const name = `${queryPrefix}${__name__ || ""}`;
+
+  if (Object.keys(freeFormFields).length === 0) {
+    return name || "value";
+  }
+
+  const fieldsString = Object.entries(freeFormFields)
+    .map(([key, value]) => `${key}=${JSON.stringify(value)}`)
+    .join(", ");
+
+  return `${name}{${fieldsString}}`;
 };
 
 export const promValueToNumber = (s: string): number => {

app/vmui/packages/vmui/src/utils/uplot/helpers.ts

@@ -1,5 +1,6 @@
 import uPlot from "uplot";
 import { MetricResult } from "../../api/types";
+import { SeriesItem } from "../../types";
 
 export const formatTicks = (u: uPlot, ticks: number[], unit = ""): string[] => {
   const min = ticks[0];
@@ -53,7 +54,11 @@ export const getDashLine = (group: number): number[] => {
   return group <= 1 ? [] : [group*4, group*1.2];
 };
 
-export const getMetricName = (metricItem: MetricResult) => {
+export const getMetricName = (metricItem: MetricResult, seriesItem: SeriesItem) => {
+  if (seriesItem?.hasAlias && seriesItem?.label) {
+    return seriesItem.label;
+  }
+
   const metric = metricItem?.metric || {};
   const labelNames = Object.keys(metric).filter(x => x != "__name__");
   const labels = labelNames.map(key => `${key}=${JSON.stringify(metric[key])}`);

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -42,10 +42,12 @@ export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[],
 
   return (d: MetricResult, i: number): SeriesItem => {
     const metricInfo = isAnomalyUI ? isForecast(data[i].metric) : null;
-    const label = isAnomalyUI ? metricInfo?.group || "" : getNameForMetric(d, alias[d.group - 1]);
+    const aliasValue = alias[d.group - 1];
+    const label = isAnomalyUI ? metricInfo?.group || "" : getNameForMetric(d, aliasValue);
 
     return {
       label,
+      hasAlias: Boolean(aliasValue),
       dash: getDashSeries(metricInfo),
       width: getWidthSeries(metricInfo),
       stroke: getStrokeSeries({ metricInfo, label, isAnomalyUI, colorState }),
@@ -88,6 +90,7 @@ export const getLegendItem = (s: SeriesItem, group: number): LegendItemType => (
   freeFormFields: s.freeFormFields,
   statsFormatted: s.statsFormatted,
   median: s.median,
+  hasAlias: s.hasAlias || false,
 });
 
 export const getHideSeries = ({ hideSeries, legend, metaKey, series, isAnomalyView }: HideSeriesArgs): string[] => {
@@ -185,7 +188,27 @@ const getPointsSeries = (metricInfo: ForecastMetricInfo | null): uPlotSeries.Poi
   if (isAnomalyMetric) {
     return { size: 8, width: 4, space: 0 };
   }
-  return { size: 4.2, width: 1.4 };
+  return {
+    size: 4,
+    width: 0,
+    show: true,
+    filter: filterPoints,
+  };
+};
+
+const filterPoints = (self: uPlot, seriesIdx: number): number[] | null => {
+  const data  = self.data[seriesIdx];
+  const indices = [];
+
+  for (let i = 0; i < data.length; i++) {
+    const prev = data[i - 1];
+    const next = data[i + 1];
+    if (prev === null && next === null) {
+      indices.push(i);
+    }
+  }
+
+  return indices;
 };
 
 type GetStrokeSeriesArgs = {

apptest/client.go

@@ -1,6 +1,7 @@
 package apptest
 
 import (
+	"bytes"
 	"io"
 	"net/http"
 	"net/url"
@@ -36,13 +37,13 @@ func (c *Client) CloseConnections() {
 // the response body to the caller.
 func (c *Client) Get(t *testing.T, url string, wantStatusCode int) string {
 	t.Helper()
-	return c.do(t, http.MethodGet, url, "", "", wantStatusCode)
+	return c.do(t, http.MethodGet, url, "", nil, wantStatusCode)
 }
 
 // Post sends a HTTP POST request. Once the function receives a response, it
 // checks whether the response status code matches the expected one and returns
 // the response body to the caller.
-func (c *Client) Post(t *testing.T, url, contentType, data string, wantStatusCode int) string {
+func (c *Client) Post(t *testing.T, url, contentType string, data []byte, wantStatusCode int) string {
 	t.Helper()
 	return c.do(t, http.MethodPost, url, contentType, data, wantStatusCode)
 }
@@ -52,16 +53,16 @@ func (c *Client) Post(t *testing.T, url, contentType, data string, wantStatusCod
 // matches the expected one and returns the response body to the caller.
 func (c *Client) PostForm(t *testing.T, url string, data url.Values, wantStatusCode int) string {
 	t.Helper()
-	return c.Post(t, url, "application/x-www-form-urlencoded", data.Encode(), wantStatusCode)
+	return c.Post(t, url, "application/x-www-form-urlencoded", []byte(data.Encode()), wantStatusCode)
 }
 
 // do prepares a HTTP request, sends it to the server, receives the response
 // from the server, ensures then response code matches the expected one, reads
 // the rentire response body and returns it to the caller.
-func (c *Client) do(t *testing.T, method, url, contentType, data string, wantStatusCode int) string {
+func (c *Client) do(t *testing.T, method, url, contentType string, data []byte, wantStatusCode int) string {
 	t.Helper()
 
-	req, err := http.NewRequest(method, url, strings.NewReader(data))
+	req, err := http.NewRequest(method, url, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("could not create a HTTP request: %v", err)
 	}
@@ -128,3 +129,31 @@ func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
 	t.Fatalf("metic not found: %s", metricName)
 	return 0
 }
+
+// GetMetricsByPrefix retrieves the values of all metrics that start with given
+// prefix.
+func (app *ServesMetrics) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
+	t.Helper()
+
+	values := []float64{}
+
+	metrics := app.cli.Get(t, app.metricsURL, http.StatusOK)
+	for _, metric := range strings.Split(metrics, "\n") {
+		if !strings.HasPrefix(metric, prefix) {
+			continue
+		}
+
+		parts := strings.Split(metric, " ")
+		if len(parts) < 2 {
+			t.Fatalf("unexpected record format: got %q, want metric name and value separated by a space", metric)
+		}
+
+		value, err := strconv.ParseFloat(parts[len(parts)-1], 64)
+		if err != nil {
+			t.Fatalf("could not parse metric value %s: %v", metric, err)
+		}
+
+		values = append(values, value)
+	}
+	return values
+}

apptest/model.go

@@ -3,27 +3,83 @@ package apptest
 import (
 	"encoding/json"
 	"fmt"
+	"net/url"
+	"slices"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
+
+	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // PrometheusQuerier contains methods available to Prometheus-like HTTP API for Querying
 type PrometheusQuerier interface {
-	PrometheusAPIV1Query(t *testing.T, query, time, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse
-	PrometheusAPIV1QueryRange(t *testing.T, query, start, end, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse
+	PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
+	PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
+	PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
 	PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse
 }
 
 // PrometheusWriter contains methods available to Prometheus-like HTTP API for Writing new data
 type PrometheusWriter interface {
+	PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
 	PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
 }
 
+// StorageFlusher defines a method that forces the flushing of data inserted
+// into the storage, so it becomes available for searching immediately.
+type StorageFlusher interface {
+	ForceFlush(t *testing.T)
+}
+
+// PrometheusWriteQuerier encompasses the methods for writing, flushing and
+// querying the data.
+type PrometheusWriteQuerier interface {
+	PrometheusWriter
+	PrometheusQuerier
+	StorageFlusher
+}
+
 // QueryOpts contains various params used for querying or ingesting data
 type QueryOpts struct {
-	Tenant  string
-	Timeout string
+	Tenant       string
+	Timeout      string
+	Start        string
+	End          string
+	Time         string
+	Step         string
+	ExtraFilters []string
+	ExtraLabels  []string
+}
+
+func (qos *QueryOpts) asURLValues() url.Values {
+	uv := make(url.Values)
+	addNonEmpty := func(name string, values ...string) {
+		for _, value := range values {
+			if len(value) == 0 {
+				continue
+			}
+			uv.Add(name, value)
+		}
+	}
+	addNonEmpty("start", qos.Start)
+	addNonEmpty("end", qos.End)
+	addNonEmpty("time", qos.Time)
+	addNonEmpty("step", qos.Step)
+	addNonEmpty("timeout", qos.Timeout)
+	addNonEmpty("extra_label", qos.ExtraLabels...)
+	addNonEmpty("extra_filters", qos.ExtraFilters...)
+
+	return uv
+}
+
+// getTenant returns tenant with optional default value
+func (qos *QueryOpts) getTenant() string {
+	if qos.Tenant == "" {
+		return "0"
+	}
+	return qos.Tenant
 }
 
 // PrometheusAPIV1QueryResponse is an inmemory representation of the
@@ -40,7 +96,7 @@ func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1Que
 
 	res := &PrometheusAPIV1QueryResponse{}
 	if err := json.Unmarshal([]byte(s), res); err != nil {
-		t.Fatalf("could not unmarshal query response: %v", err)
+		t.Fatalf("could not unmarshal query response data=\n%s\n: %v", string(s), err)
 	}
 	return res
 }
@@ -81,7 +137,7 @@ func NewSample(t *testing.T, timeStr string, value float64) *Sample {
 // UnmarshalJSON populates the sample fields from a JSON string.
 func (s *Sample) UnmarshalJSON(b []byte) error {
 	var (
-		ts int64
+		ts float64
 		v  string
 	)
 	raw := []any{&ts, &v}
@@ -91,7 +147,7 @@ func (s *Sample) UnmarshalJSON(b []byte) error {
 	if got, want := len(raw), 2; got != want {
 		return fmt.Errorf("unexpected number of fields: got %d, want %d (raw sample: %s)", got, want, string(b))
 	}
-	s.Timestamp = ts
+	s.Timestamp = int64(ts)
 	var err error
 	s.Value, err = strconv.ParseFloat(v, 64)
 	if err != nil {
@@ -115,7 +171,23 @@ func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1Se
 
 	res := &PrometheusAPIV1SeriesResponse{}
 	if err := json.Unmarshal([]byte(s), res); err != nil {
-		t.Fatalf("could not unmarshal series response: %v", err)
+		t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", string(s), err)
 	}
 	return res
 }
+
+// Sort sorts the response data.
+func (r *PrometheusAPIV1SeriesResponse) Sort() {
+	str := func(m map[string]string) string {
+		s := []string{}
+		for k, v := range m {
+			s = append(s, k+v)
+		}
+		slices.Sort(s)
+		return strings.Join(s, "")
+	}
+
+	slices.SortFunc(r.Data, func(a, b map[string]string) int {
+		return strings.Compare(str(a), str(b))
+	})
+}

apptest/testcase.go

@@ -1,9 +1,12 @@
 package apptest
 
 import (
+	"fmt"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/google/go-cmp/cmp"
 )
 
 // TestCase holds the state and defines clean-up procedure common for all test
@@ -51,6 +54,17 @@ func (tc *TestCase) Stop() {
 	}
 }
 
+// MustStartDefaultVmsingle is a test helper function that starts an instance of
+// vmsingle with defaults suitable for most tests.
+func (tc *TestCase) MustStartDefaultVmsingle() *Vmsingle {
+	tc.t.Helper()
+
+	return tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmsingle",
+		"-retentionPeriod=100y",
+	})
+}
+
 // MustStartVmsingle is a test helper function that starts an instance of
 // vmsingle and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
@@ -103,6 +117,125 @@ func (tc *TestCase) MustStartVminsert(instance string, flags []string) *Vminsert
 	return app
 }
 
+type vmcluster struct {
+	*Vminsert
+	*Vmselect
+	vmstorages []*Vmstorage
+}
+
+func (c *vmcluster) ForceFlush(t *testing.T) {
+	for _, s := range c.vmstorages {
+		s.ForceFlush(t)
+	}
+}
+
+// MustStartDefaultCluster is a typical cluster configuration suitable for most
+// tests.
+//
+// The cluster consists of two vmstorages, one vminsert and one vmselect, no
+// data replication.
+//
+// Such configuration is suitable for tests that don't verify the
+// cluster-specific behavior (such as sharding, replication, or multilevel
+// vmselect) but instead just need a typical cluster configuration to verify
+// some business logic (such as API surface, or MetricsQL). Such cluster
+// tests usually come paired with corresponding vmsingle tests.
+func (tc *TestCase) MustStartDefaultCluster() PrometheusWriteQuerier {
+	tc.t.Helper()
+
+	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
+		"-retentionPeriod=100y",
+	})
+	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
+		"-retentionPeriod=100y",
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
+		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	})
+
+	return &vmcluster{vminsert, vmselect, []*Vmstorage{vmstorage1, vmstorage2}}
+}
+
 func (tc *TestCase) addApp(app Stopper) {
 	tc.startedApps = append(tc.startedApps, app)
 }
+
+// AssertOptions hold the assertion params, such as got and wanted values as
+// well as the message that should be included into the assertion error message
+// in case of failure.
+//
+// In VictoriaMetrics (especially the cluster version) the inserted data does
+// not become visible for querying right away. Therefore, the first comparisons
+// may fail. AssertOptions allow to configure how many times the actual result
+// must be retrieved and compared with the expected one and for long to wait
+// between the retries. If these two params (`Retries` and `Period`) are not
+// set, the default values will be used.
+//
+// If it is known that the data is available, then the retry functionality can
+// be disabled by setting the `DoNotRetry` field.
+//
+// AssertOptions are used by the TestCase.Assert() method, and this method uses
+// cmp.Diff() from go-cmp package for comparing got and wanted values.
+// AssertOptions, therefore, allows to pass cmp.Options to cmp.Diff() via
+// `CmpOpts` field.
+//
+// Finally the `FailNow` field controls whether the assertion should fail using
+// `testing.T.Errorf()` or `testing.T.Fatalf()`.
+type AssertOptions struct {
+	Msg        string
+	Got        func() any
+	Want       any
+	CmpOpts    []cmp.Option
+	DoNotRetry bool
+	Retries    int
+	Period     time.Duration
+	FailNow    bool
+}
+
+// Assert compares the actual result with the expected one possibly multiple
+// times in order to account for the fact that the inserted data does not become
+// available for querying right away (especially in cluster version of
+// VictoriaMetrics).
+func (tc *TestCase) Assert(opts *AssertOptions) {
+	tc.t.Helper()
+
+	const (
+		defaultRetries = 20
+		defaultPeriod  = 100 * time.Millisecond
+	)
+
+	if opts.DoNotRetry {
+		opts.Retries = 1
+		opts.Period = 0
+	} else {
+		if opts.Retries <= 0 {
+			opts.Retries = defaultRetries
+		}
+		if opts.Period <= 0 {
+			opts.Period = defaultPeriod
+		}
+	}
+
+	var diff string
+
+	for range opts.Retries {
+		diff = cmp.Diff(opts.Want, opts.Got(), opts.CmpOpts...)
+		if diff == "" {
+			return
+		}
+		time.Sleep(opts.Period)
+	}
+
+	msg := fmt.Sprintf("%s (-want, +got):\n%s", opts.Msg, diff)
+
+	if opts.FailNow {
+		tc.t.Fatal(msg)
+	} else {
+		tc.t.Error(msg)
+	}
+}

apptest/tests/key_concepts_test.go

@@ -1,11 +1,11 @@
 package tests
 
 import (
+	"testing"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"testing"
-	"time"
 )
 
 // Data used in examples in
@@ -28,127 +28,180 @@ var docData = []string{
 }
 
 // TestSingleKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+// for vm-single.
 func TestSingleKeyConceptsQuery(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
 
-	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
-		"-storageDataPath=" + tc.Dir() + "/vmstorage",
-		"-retentionPeriod=100y",
-	})
+	sut := tc.MustStartDefaultVmsingle()
 
-	opts := apptest.QueryOpts{Timeout: "5s"}
-
-	// Insert example data from documentation.
-	vmsingle.PrometheusAPIV1ImportPrometheus(t, docData, opts)
-	vmsingle.ForceFlush(t)
-
-	testInstantQuery(t, vmsingle, opts)
-	testRangeQuery(t, vmsingle, opts)
+	testKeyConceptsQueryData(t, sut)
 }
 
-// TestClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
-func TestClusterKeyConceptsQuery(t *testing.T) {
+// TestClusterKeyConceptsQueryData verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+// for vm-cluster.
+func TestClusterKeyConceptsQueryData(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
 
-	// Set up the following cluster configuration:
-	//
-	// - two vmstorage instances
-	// - vminsert points to the two vmstorages, its replication setting
-	//   is off which means it will only shard the incoming data across the two
-	//   vmstorages.
-	// - vmselect points to the two vmstorages and is expected to query both
-	//   vmstorages and build the full result out of the two partial results.
+	sut := tc.MustStartDefaultCluster()
 
-	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
-		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
-		"-retentionPeriod=100y",
-	})
-	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
-		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
-		"-retentionPeriod=100y",
-	})
-	vminsert := tc.MustStartVminsert("vminsert", []string{
-		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
-	})
-	vmselect := tc.MustStartVmselect("vmselect", []string{
-		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
-	})
-
-	opts := apptest.QueryOpts{Timeout: "5s", Tenant: "0"}
-
-	// Insert example data from documentation.
-	vminsert.PrometheusAPIV1ImportPrometheus(t, docData, opts)
-	time.Sleep(2 * time.Second)
-
-	vmstorage1.ForceFlush(t)
-	vmstorage2.ForceFlush(t)
-
-	testInstantQuery(t, vmselect, opts)
-	testRangeQuery(t, vmselect, opts)
+	testKeyConceptsQueryData(t, sut)
 }
 
-// vmsingleInstantQuery verifies the statements made in the
-// `Instant query` section of the VictoriaMetrics documentation. See:
+// testClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+func testKeyConceptsQueryData(t *testing.T, sut apptest.PrometheusWriteQuerier) {
+
+	// Insert example data from documentation.
+	sut.PrometheusAPIV1ImportPrometheus(t, docData, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+
+	testInstantQuery(t, sut)
+	testRangeQuery(t, sut)
+	testRangeQueryIsEquivalentToManyInstantQueries(t, sut)
+}
+
+// testInstantQuery verifies the statements made in the `Instant query` section
+// of the VictoriaMetrics documentation. See:
 // https://docs.victoriametrics.com/keyconcepts/#instant-query
-func testInstantQuery(t *testing.T, q apptest.PrometheusQuerier, opts apptest.QueryOpts) {
-	// Get the value of the foo_bar time series at 2022-05-10Z08:03:00Z with the
+func testInstantQuery(t *testing.T, q apptest.PrometheusQuerier) {
+	// Get the value of the foo_bar time series at 2022-05-10T08:03:00Z with the
 	// step of 5m and timeout 5s. There is no sample at exactly this timestamp.
 	// Therefore, VictoriaMetrics will search for the nearest sample within the
 	// [time-5m..time] interval.
-	got := q.PrometheusAPIV1Query(t, "foo_bar", "2022-05-10T08:03:00.000Z", "5m", opts)
+	got := q.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
 	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
 	opt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
 	if diff := cmp.Diff(want, got, opt); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
-	// Get the value of the foo_bar time series at 2022-05-10Z08:18:00Z with the
+	// Get the value of the foo_bar time series at 2022-05-10T08:18:00Z with the
 	// step of 1m and timeout 5s. There is no sample at this timestamp.
 	// Therefore, VictoriaMetrics will search for the nearest sample within the
 	// [time-1m..time] interval. Since the nearest sample is 2m away and the
 	// step is 1m, then the VictoriaMetrics must return empty response.
-	got = q.PrometheusAPIV1Query(t, "foo_bar", "2022-05-10T08:18:00.000Z", "1m", opts)
+	got = q.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
 	if len(got.Data.Result) > 0 {
 		t.Errorf("unexpected response: got non-empty result, want empty result:\n%v", got)
 	}
 }
 
-// vmsingleRangeQuery verifies the statements made in the
-// `Range query` section of the VictoriaMetrics documentation. See:
+// testRangeQuery verifies the statements made in the `Range query` section of
+// the VictoriaMetrics documentation. See:
 // https://docs.victoriametrics.com/keyconcepts/#range-query
-func testRangeQuery(t *testing.T, q apptest.PrometheusQuerier, opts apptest.QueryOpts) {
-	// Get the values of the foo_bar time series for
-	// [2022-05-10Z07:59:00Z..2022-05-10Z08:17:00Z] time interval with the step
-	// of 1m and timeout 5s.
-	got := q.PrometheusAPIV1QueryRange(t, "foo_bar", "2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "1m", opts)
-	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
-	s := make([]*apptest.Sample, 17)
-	// Sample for 2022-05-10T07:59:00Z is missing because the time series has
-	// samples only starting from 8:00.
-	s[0] = apptest.NewSample(t, "2022-05-10T08:00:00Z", 1)
-	s[1] = apptest.NewSample(t, "2022-05-10T08:01:00Z", 2)
-	s[2] = apptest.NewSample(t, "2022-05-10T08:02:00Z", 3)
-	s[3] = apptest.NewSample(t, "2022-05-10T08:03:00Z", 3)
-	s[4] = apptest.NewSample(t, "2022-05-10T08:04:00Z", 5)
-	s[5] = apptest.NewSample(t, "2022-05-10T08:05:00Z", 5)
-	s[6] = apptest.NewSample(t, "2022-05-10T08:06:00Z", 5.5)
-	s[7] = apptest.NewSample(t, "2022-05-10T08:07:00Z", 5.5)
-	s[8] = apptest.NewSample(t, "2022-05-10T08:08:00Z", 4)
-	s[9] = apptest.NewSample(t, "2022-05-10T08:09:00Z", 4)
-	// Sample for 2022-05-10T08:10:00Z is missing because there is no sample
-	// within the [8:10 - 1m .. 8:10] interval.
-	s[10] = apptest.NewSample(t, "2022-05-10T08:11:00Z", 3.5)
-	s[11] = apptest.NewSample(t, "2022-05-10T08:12:00Z", 3.25)
-	s[12] = apptest.NewSample(t, "2022-05-10T08:13:00Z", 3)
-	s[13] = apptest.NewSample(t, "2022-05-10T08:14:00Z", 2)
-	s[14] = apptest.NewSample(t, "2022-05-10T08:15:00Z", 1)
-	s[15] = apptest.NewSample(t, "2022-05-10T08:16:00Z", 4)
-	s[16] = apptest.NewSample(t, "2022-05-10T08:17:00Z", 4)
-	want.Data.Result[0].Samples = s
-	opt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
-	if diff := cmp.Diff(want, got, opt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+func testRangeQuery(t *testing.T, q apptest.PrometheusQuerier) {
+	f := func(start, end, step string, wantSamples []*apptest.Sample) {
+		t.Helper()
+
+		got := q.PrometheusAPIV1QueryRange(t, "foo_bar", apptest.QueryOpts{Start: start, End: end, Step: step})
+		want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
+		want.Data.Result[0].Samples = wantSamples
+		opt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+		if diff := cmp.Diff(want, got, opt); diff != "" {
+			t.Errorf("unexpected response (-want, +got):\n%s", diff)
+		}
 	}
+
+	// Verify the statement that the query result for
+	// [2022-05-10T07:59:00Z..2022-05-10T08:17:00Z] time range and 1m step will
+	// contain 17 points.
+	f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "1m", []*apptest.Sample{
+		// Sample for 2022-05-10T07:59:00Z is missing because the time series has
+		// samples only starting from 8:00.
+		apptest.NewSample(t, "2022-05-10T08:00:00Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:01:00Z", 2),
+		apptest.NewSample(t, "2022-05-10T08:02:00Z", 3),
+		apptest.NewSample(t, "2022-05-10T08:03:00Z", 3),
+		apptest.NewSample(t, "2022-05-10T08:04:00Z", 5),
+		apptest.NewSample(t, "2022-05-10T08:05:00Z", 5),
+		apptest.NewSample(t, "2022-05-10T08:06:00Z", 5.5),
+		apptest.NewSample(t, "2022-05-10T08:07:00Z", 5.5),
+		apptest.NewSample(t, "2022-05-10T08:08:00Z", 4),
+		apptest.NewSample(t, "2022-05-10T08:09:00Z", 4),
+		// Sample for 2022-05-10T08:10:00Z is missing because there is no sample
+		// within the [8:10 - 1m .. 8:10] interval.
+		apptest.NewSample(t, "2022-05-10T08:11:00Z", 3.5),
+		apptest.NewSample(t, "2022-05-10T08:12:00Z", 3.25),
+		apptest.NewSample(t, "2022-05-10T08:13:00Z", 3),
+		apptest.NewSample(t, "2022-05-10T08:14:00Z", 2),
+		apptest.NewSample(t, "2022-05-10T08:15:00Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:16:00Z", 4),
+		apptest.NewSample(t, "2022-05-10T08:17:00Z", 4),
+	})
+
+	// Verify the statement that a query is executed at start, start+step,
+	// start+2*step, …, step+N*step timestamps, where N is the whole number
+	// of steps that fit between start and end.
+	f("2022-05-10T08:00:01.000Z", "2022-05-10T08:02:00.000Z", "1m", []*apptest.Sample{
+		apptest.NewSample(t, "2022-05-10T08:00:01Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:01:01Z", 2),
+	})
+
+	// Verify the statement that a query is executed at start, start+step,
+	// start+2*step, …, end timestamps, when end = start + N*step.
+	f("2022-05-10T08:00:00.000Z", "2022-05-10T08:02:00.000Z", "1m", []*apptest.Sample{
+		apptest.NewSample(t, "2022-05-10T08:00:00Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:01:00Z", 2),
+		apptest.NewSample(t, "2022-05-10T08:02:00Z", 3),
+	})
+
+	// If the step isn’t set, then it defaults to 5m (5 minutes).
+	f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "", []*apptest.Sample{
+		// Sample for 2022-05-10T07:59:00Z is missing because the time series has
+		// samples only starting from 8:00.
+		apptest.NewSample(t, "2022-05-10T08:04:00Z", 5),
+		apptest.NewSample(t, "2022-05-10T08:09:00Z", 4),
+		apptest.NewSample(t, "2022-05-10T08:14:00Z", 2),
+	})
+}
+
+// testRangeQueryIsEquivalentToManyInstantQueries verifies the statement made in
+// the `Range query` section of the VictoriaMetrics documentation that a range
+// query is actually an instant query executed 1 + (start-end)/step times on the
+// time range from start to end. See:
+// https://docs.victoriametrics.com/keyconcepts/#range-query
+func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q apptest.PrometheusQuerier) {
+	f := func(timestamp string, want *apptest.Sample) {
+		t.Helper()
+
+		gotInstant := q.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{Time: timestamp, Step: "1m"})
+		if want == nil {
+			if got, want := len(gotInstant.Data.Result), 0; got != want {
+				t.Errorf("unexpected instant result size: got %d, want %d", got, want)
+			}
+		} else {
+			got := gotInstant.Data.Result[0].Sample
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("unexpected instant sample (-want, +got):\n%s", diff)
+			}
+		}
+	}
+
+	rangeRes := q.PrometheusAPIV1QueryRange(t, "foo_bar", apptest.QueryOpts{
+		Start: "2022-05-10T07:59:00.000Z",
+		End:   "2022-05-10T08:17:00.000Z",
+		Step:  "1m",
+	})
+	rangeSamples := rangeRes.Data.Result[0].Samples
+
+	f("2022-05-10T07:59:00.000Z", nil)
+	f("2022-05-10T08:00:00.000Z", rangeSamples[0])
+	f("2022-05-10T08:01:00.000Z", rangeSamples[1])
+	f("2022-05-10T08:02:00.000Z", rangeSamples[2])
+	f("2022-05-10T08:03:00.000Z", rangeSamples[3])
+	f("2022-05-10T08:04:00.000Z", rangeSamples[4])
+	f("2022-05-10T08:05:00.000Z", rangeSamples[5])
+	f("2022-05-10T08:06:00.000Z", rangeSamples[6])
+	f("2022-05-10T08:07:00.000Z", rangeSamples[7])
+	f("2022-05-10T08:08:00.000Z", rangeSamples[8])
+	f("2022-05-10T08:09:00.000Z", rangeSamples[9])
+	f("2022-05-10T08:10:00.000Z", nil)
+	f("2022-05-10T08:11:00.000Z", rangeSamples[10])
+	f("2022-05-10T08:12:00.000Z", rangeSamples[11])
+	f("2022-05-10T08:13:00.000Z", rangeSamples[12])
+	f("2022-05-10T08:14:00.000Z", rangeSamples[13])
+	f("2022-05-10T08:15:00.000Z", rangeSamples[14])
+	f("2022-05-10T08:16:00.000Z", rangeSamples[15])
+	f("2022-05-10T08:17:00.000Z", rangeSamples[16])
 }

apptest/tests/metricsql_test.go

@@ -0,0 +1,130 @@
+package tests
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+func millis(s string) int64 {
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		panic(fmt.Sprintf("could not parse time %q: %v", s, err))
+	}
+	return t.UnixMilli()
+}
+
+var staleNaNsData = func() []pb.TimeSeries {
+	return []pb.TimeSeries{
+		{
+			Labels: []pb.Label{
+				{
+					Name:  "__name__",
+					Value: "metric",
+				},
+			},
+			Samples: []pb.Sample{
+				{
+					Value:     1,
+					Timestamp: millis("2024-01-01T00:01:00Z"),
+				},
+				{
+					Value:     decimal.StaleNaN,
+					Timestamp: millis("2024-01-01T00:02:00Z"),
+				},
+			},
+		},
+	}
+}()
+
+func TestSingleInstantQueryDoesNotReturnStaleNaNs(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultVmsingle()
+
+	testInstantQueryDoesNotReturnStaleNaNs(t, sut)
+}
+
+func TestClusterInstantQueryDoesNotReturnStaleNaNs(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartDefaultCluster()
+
+	testInstantQueryDoesNotReturnStaleNaNs(t, sut)
+}
+
+func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.PrometheusWriteQuerier) {
+
+	sut.PrometheusAPIV1Write(t, staleNaNsData, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+
+	var got, want *apptest.PrometheusAPIV1QueryResponse
+	cmpOptions := []cmp.Option{
+		cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		cmpopts.EquateNaNs(),
+	}
+
+	// Verify that instant query returns the first point.
+
+	got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
+		Step: "5m",
+		Time: "2024-01-01T00:01:00.000Z",
+	})
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
+	want.Data.Result[0].Sample = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Verify that instant query does not return stale NaN.
+
+	got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
+		Step: "5m",
+		Time: "2024-01-01T00:02:00.000Z",
+	})
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": []}}`)
+	// Empty response, stale NaN is not included into response
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Verify that instant query with default rollup function returns stale NaN
+	// while it must not.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5806
+
+	got = sut.PrometheusAPIV1Query(t, "metric[2m]", apptest.QueryOpts{
+		Step: "5m",
+		Time: "2024-01-01T00:02:00.000Z",
+	})
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
+	s := make([]*apptest.Sample, 2)
+	s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
+	s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
+	want.Data.Result[0].Samples = s
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Verify that exported data contains stale NaN.
+
+	got = sut.PrometheusAPIV1Export(t, `{__name__="metric"}`, apptest.QueryOpts{
+		Start: "2024-01-01T00:01:00.000Z",
+		End:   "2024-01-01T00:02:00.000Z",
+	})
+	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
+	s = make([]*apptest.Sample, 2)
+	s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
+	s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
+	want.Data.Result[0].Samples = s
+	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+}