lib/stringsutil: optimize AppendLowercase

The optimization includes the following improvements:
- Implementation of a function that processes 8 bytes per loop iteration to locate ASCII characters using bitwise manipulations.
- Implementation of the ToLowercaseFunc function that prevents string copying if the string is already in lowercase.
- Use of a lookup table for converting ASCII characters to lowercase, with logic copied from the VictoriaLogs repository.

.github/workflows/test.yml

@@ -66,8 +66,8 @@ jobs:
     strategy:
       matrix:
         scenario:
-          - 'test-full'
-          - 'test-full-386'
+          - 'test'
+          - 'test-386'
           - 'test-pure'
 
     steps:
@@ -88,11 +88,6 @@ jobs:
       - name: Run tests
         run: make ${{ matrix.scenario}}
 
-      - name: Publish coverage
-        uses: codecov/codecov-action@v6
-        with:
-          files: ./coverage.txt
-
   apptest:
     name: apptest
     runs-on: apptest

Makefile

@@ -457,6 +457,9 @@ test:
 test-race:
 	go test -tags 'synctest' -race ./lib/... ./app/...
 
+test-386:
+	GOARCH=386 go test -tags 'synctest' ./lib/... ./app/...
+
 test-pure:
 	CGO_ENABLED=0 go test -tags 'synctest' ./lib/... ./app/...
 

README.md

@@ -4,7 +4,6 @@
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
-[![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)
 [![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)](https://x.com/VictoriaMetrics/)

app/vmagent/remotewrite/client.go

@@ -290,7 +290,7 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
 	accessKey := awsAccessKey.GetOptionalArg(argIdx)
 	secretKey := awsSecretKey.GetOptionalArg(argIdx)
 	service := awsService.GetOptionalArg(argIdx)
-	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
+	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service, "")
 	if err != nil {
 		return nil, err
 	}

app/vmagent/remotewrite/remotewrite.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"math"
 	"net/http"
 	"net/url"
 	"path/filepath"
@@ -11,6 +12,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
+
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -31,8 +36,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
 )
 
 var (
@@ -81,10 +84,14 @@ var (
 		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
 		`Enabled sorting for labels can slow down ingestion performance a bit`)
-	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
-		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
-	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
-		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
+	maxHourlySeries = flag.Int64("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. "+
+		fmt.Sprintf("Setting this flag to '-1' sets limit to maximum possible value (%d) which is useful in order to enable series tracking without enforcing limits. ", math.MaxInt32)+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
+	maxDailySeries = flag.Int64("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. "+
+		fmt.Sprintf("Setting this flag to '-1' sets limit to maximum possible value (%d) which is useful in order to enable series tracking without enforcing limits. ", math.MaxInt32)+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
@@ -160,8 +167,8 @@ func Init() {
 	if len(*remoteWriteURLs) == 0 {
 		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
 	}
-	if *maxHourlySeries > 0 {
-		hourlySeriesLimiter = bloomfilter.NewLimiter(*maxHourlySeries, time.Hour)
+	if limit := getMaxHourlySeries(); limit > 0 {
+		hourlySeriesLimiter = bloomfilter.NewLimiter(limit, time.Hour)
 		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
 			return float64(hourlySeriesLimiter.MaxItems())
 		})
@@ -169,8 +176,8 @@ func Init() {
 			return float64(hourlySeriesLimiter.CurrentItems())
 		})
 	}
-	if *maxDailySeries > 0 {
-		dailySeriesLimiter = bloomfilter.NewLimiter(*maxDailySeries, 24*time.Hour)
+	if limit := getMaxDailySeries(); limit > 0 {
+		dailySeriesLimiter = bloomfilter.NewLimiter(limit, 24*time.Hour)
 		_ = metrics.NewGauge(`vmagent_daily_series_limit_max_series`, func() float64 {
 			return float64(dailySeriesLimiter.MaxItems())
 		})
@@ -1141,3 +1148,21 @@ func newMapFromStrings(a []string) map[string]struct{} {
 	}
 	return m
 }
+
+func getMaxHourlySeries() int {
+	limit := *maxHourlySeries
+	if limit == -1 || limit > math.MaxInt32 {
+		return math.MaxInt32
+	}
+
+	return int(limit)
+}
+
+func getMaxDailySeries() int {
+	limit := *maxDailySeries
+	if limit == -1 || limit > math.MaxInt32 {
+		return math.MaxInt32
+	}
+
+	return int(limit)
+}

app/vmalert/remotewrite/client.go

@@ -13,8 +13,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/golang/snappy"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -115,8 +117,10 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		input:         make(chan prompb.TimeSeries, cfg.MaxQueueSize),
 	}
 
-	for range cc {
-		c.run(ctx)
+	for i := 0; i < cc; i++ {
+		c.wg.Go(func() {
+			c.run(ctx, i)
+		})
 	}
 	return c, nil
 }
@@ -158,8 +162,7 @@ func (c *Client) Close() error {
 	return nil
 }
 
-func (c *Client) run(ctx context.Context) {
-	ticker := time.NewTicker(c.flushInterval)
+func (c *Client) run(ctx context.Context, id int) {
 	wr := &prompb.WriteRequest{}
 	shutdown := func() {
 		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
@@ -176,45 +179,72 @@ func (c *Client) run(ctx context.Context) {
 		cancel()
 	}
 
-	c.wg.Go(func() {
-		defer ticker.Stop()
-		for {
+	// add jitter to spread remote write flushes over the flush interval to avoid congestion at the remote write destination
+	h := xxhash.Sum64(bytesutil.ToUnsafeBytes(fmt.Sprintf("%d", id)))
+	randJitter := uint64(float64(c.flushInterval) * (float64(h) / (1 << 64)))
+	timer := time.NewTimer(time.Duration(randJitter))
+addJitter:
+	for {
+		select {
+		case <-c.doneCh:
+			timer.Stop()
+			shutdown()
+			return
+		case <-ctx.Done():
+			timer.Stop()
+			shutdown()
+			return
+		case <-timer.C:
+			break addJitter
+		}
+	}
+
+	ticker := time.NewTicker(c.flushInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-c.doneCh:
+			shutdown()
+			return
+		case <-ctx.Done():
+			shutdown()
+			return
+		case <-ticker.C:
+			c.flush(ctx, wr)
+			// drain the potential stale tick to avoid small or empty flushes after a slow flush.
 			select {
-			case <-c.doneCh:
-				shutdown()
-				return
-			case <-ctx.Done():
-				shutdown()
-				return
 			case <-ticker.C:
+			default:
+			}
+		case ts, ok := <-c.input:
+			if !ok {
+				continue
+			}
+			wr.Timeseries = append(wr.Timeseries, ts)
+			if len(wr.Timeseries) >= c.maxBatchSize {
 				c.flush(ctx, wr)
-				// drain the potential stale tick to avoid small or empty flushes after a slow flush.
-				select {
-				case <-ticker.C:
-				default:
-				}
-			case ts, ok := <-c.input:
-				if !ok {
-					continue
-				}
-				wr.Timeseries = append(wr.Timeseries, ts)
-				if len(wr.Timeseries) >= c.maxBatchSize {
-					c.flush(ctx, wr)
-				}
 			}
 		}
-	})
+	}
 }
 
 var (
 	rwErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
 	rwTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 
-	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
-	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
-	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	sendDuration        = metrics.NewFloatCounter(`vmalert_remotewrite_send_duration_seconds_total`)
-	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
+	// sentRows and sentBytes are historical counters that can now be replaced by flushedRows and flushedBytes histograms. They may be deprecated in the future after the new histograms have been adopted for some time.
+	sentRows             = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes            = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	flushedRows          = metrics.NewHistogram(`vmalert_remotewrite_sent_rows`)
+	flushedBytes         = metrics.NewHistogram(`vmalert_remotewrite_sent_bytes`)
+	droppedRows          = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	sendDuration         = metrics.NewFloatCounter(`vmalert_remotewrite_send_duration_seconds_total`)
+	bufferFlushDuration  = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
+	remoteWriteQueueSize = metrics.NewHistogram(`vmalert_remotewrite_queue_size`)
+
+	_ = metrics.NewGauge(`vmalert_remotewrite_queue_capacity`, func() float64 {
+		return float64(*maxQueueSize)
+	})
 
 	_ = metrics.NewGauge(`vmalert_remotewrite_concurrency`, func() float64 {
 		return float64(*concurrency)
@@ -228,6 +258,7 @@ func GetDroppedRows() int { return int(droppedRows.Get()) }
 // it to remote-write endpoint. Flush performs limited amount of retries
 // if request fails.
 func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
+	remoteWriteQueueSize.Update(float64(len(c.input)))
 	if len(wr.Timeseries) < 1 {
 		return
 	}
@@ -256,6 +287,8 @@ L:
 		if err == nil {
 			sentRows.Add(len(wr.Timeseries))
 			sentBytes.Add(len(b))
+			flushedRows.Update(float64(len(wr.Timeseries)))
+			flushedBytes.Update(float64(len(b)))
 			return
 		}
 

app/vmalert/rule/group.go

@@ -381,7 +381,9 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 
 		if len(g.Rules) < 1 {
 			g.metrics.iterationDuration.UpdateDuration(start)
+			g.mu.Lock()
 			g.LastEvaluation = start
+			g.mu.Unlock()
 			return ts
 		}
 
@@ -395,7 +397,9 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 			}
 		}
 		g.metrics.iterationDuration.UpdateDuration(start)
+		g.mu.Lock()
 		g.LastEvaluation = start
+		g.mu.Unlock()
 		return ts
 	}
 
@@ -405,11 +409,11 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	g.mu.Unlock()
 	defer g.evalCancel()
 
-	realEvalTS := eval(evalCtx, evalTS)
-
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 
+	realEvalTS := eval(evalCtx, evalTS)
+
 	// restore the rules state after the first evaluation
 	// so only active alerts can be restored.
 	if rr != nil {

app/vmalert/web.go

@@ -52,7 +52,13 @@ var (
 		"alert":  rule.TypeAlerting,
 		"record": rule.TypeRecording,
 	}
-	ruleStates = []string{"ok", "nomatch", "inactive", "firing", "pending", "unhealthy"}
+
+	// The "recovering", "noData", "normal", "error" states are used by Grafana.
+	// Ignore "recovering" since it is not currently acknowledged by vmalert,
+	// treat "noData" as an alias for "nomatch",
+	// treat "normal" as an alias for "inactive",
+	// treat "error" as an alias for "unhealthy"
+	ruleStates = []string{"ok", "nomatch", "inactive", "firing", "pending", "unhealthy", "recovering", "noData", "normal", "error"}
 )
 
 type requestHandler struct {
@@ -363,6 +369,15 @@ func newRulesFilter(r *http.Request) (*rulesFilter, *httpserver.ErrorWithStatusC
 			if !slices.Contains(ruleStates, v) {
 				return nil, errResponse(fmt.Errorf(`invalid parameter "state": contains not supported value %q`, v), http.StatusBadRequest)
 			}
+			// Replace grafana states with supported internal states
+			switch v {
+			case "noData":
+				v = "nomatch"
+			case "normal":
+				v = "inactive"
+			case "error":
+				v = "unhealthy"
+			}
 			rf.states = append(rf.states, v)
 		}
 	}

app/vmauth/main.go

@@ -357,6 +357,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 
 	maxBufSize := max(requestBufferSize.IntN(), maxRequestBodySizeToRetry.IntN())
 	if maxBufSize <= 0 {
+		// Request buffering is disabled.
 		return r, nil
 	}
 
@@ -792,10 +793,11 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 }
 
 // bufferedBody serves two purposes:
-//  1. Enables request retries when the body size does not exceed maxBodySize
-//     by fully buffering the body in memory.
-//  2. Prevents slow clients from reducing effective server capacity by
-//     buffering the request body before acquiring a per-user concurrency slot.
+//
+//  1. It enables request retries when the request body size does not exceed maxBufSize
+//     by fully buffering the request body in memory.
+//  2. It prevents slow clients from reducing effective server capacity
+//     by buffering the request body before acquiring a per-user concurrency slot.
 //
 // See bufferRequestBody for details on how bufferedBody is used.
 type bufferedBody struct {
@@ -819,7 +821,7 @@ func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
 
 	if len(buf) < maxBufSize {
-		// Read the full request body into buf.
+		// The full request body has been already read into buf.
 		r = nil
 	}
 
@@ -832,7 +834,7 @@ func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody
 // Read implements io.Reader interface.
 func (bb *bufferedBody) Read(p []byte) (int, error) {
 	if bb.cannotRetry {
-		return 0, fmt.Errorf("cannot read already closed body")
+		return 0, fmt.Errorf("cannot read already closed request body")
 	}
 	if bb.bufOffset < len(bb.buf) {
 		n := copy(p, bb.buf[bb.bufOffset:])

app/vmbackup/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot/snapshotutil"
 )

app/vmselect/prometheus/export.qtpl

@@ -11,6 +11,16 @@
 
 {% stripspace %}
 
+{% func ExportCSVHeader(fieldNames []string) %}
+	{% if len(fieldNames) == 0 %}{% return %}{% endif %}
+	{%s= fieldNames[0] %}
+	{% for _, fieldName := range fieldNames[1:] %}
+		,
+		{%s= fieldName %}
+	{% endfor %}
+	{% newline %}
+{% endfunc %}
+
 {% func ExportCSVLine(xb *exportBlock, fieldNames []string) %}
 	{% if len(xb.timestamps) == 0 || len(fieldNames) == 0 %}{% return %}{% endif %}
 	{% for i, timestamp := range xb.timestamps %}

app/vmselect/prometheus/export_test.go

@@ -0,0 +1,132 @@
+package prometheus
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+func TestExportCSVHeader(t *testing.T) {
+	f := func(fieldNames []string, expected string) {
+		t.Helper()
+		got := ExportCSVHeader(fieldNames)
+		if got != expected {
+			t.Fatalf("ExportCSVHeader(%v): got %q; want %q", fieldNames, got, expected)
+		}
+	}
+
+	f(nil, "")
+	f([]string{}, "")
+
+	f([]string{"__value__"}, "__value__\n")
+	f([]string{"__timestamp__"}, "__timestamp__\n")
+	f([]string{"__timestamp__:rfc3339"}, "__timestamp__:rfc3339\n")
+	f([]string{"__name__"}, "__name__\n")
+	f([]string{"job"}, "job\n")
+
+	f([]string{"__timestamp__:rfc3339", "__value__"}, "__timestamp__:rfc3339,__value__\n")
+	f([]string{"__value__", "__timestamp__"}, "__value__,__timestamp__\n")
+	f([]string{"job", "instance"}, "job,instance\n")
+
+	f([]string{"__name__", "__value__", "__timestamp__:unix_s"}, "__name__,__value__,__timestamp__:unix_s\n")
+	f([]string{"job", "instance", "__value__", "__timestamp__:unix_ms"}, "job,instance,__value__,__timestamp__:unix_ms\n")
+	f([]string{"__timestamp__:custom:2006-01-02", "__value__", "host", "dc", "env"},
+		"__timestamp__:custom:2006-01-02,__value__,host,dc,env\n")
+
+	// duplicate fields
+	f([]string{"__value__", "__value__"}, "__value__,__value__\n")
+	f([]string{"__timestamp__", "__timestamp__:rfc3339"}, "__timestamp__,__timestamp__:rfc3339\n")
+}
+
+func TestExportCSVLine(t *testing.T) {
+	localBak := time.Local
+	time.Local = time.UTC
+	defer func() { time.Local = localBak }()
+
+	f := func(mn *storage.MetricName, timestamps []int64, values []float64, fieldNames []string, expected string) {
+		t.Helper()
+		xb := &exportBlock{
+			mn:         mn,
+			timestamps: timestamps,
+			values:     values,
+		}
+		got := ExportCSVLine(xb, fieldNames)
+		if got != expected {
+			t.Fatalf("ExportCSVLine: got %q; want %q", got, expected)
+		}
+	}
+
+	mn := &storage.MetricName{
+		MetricGroup: []byte("cpu_usage"),
+		Tags: []storage.Tag{
+			{Key: []byte("job"), Value: []byte("node")},
+			{Key: []byte("instance"), Value: []byte("localhost:9090")},
+		},
+	}
+
+	// empty inputs
+	f(mn, nil, nil, []string{"__value__"}, "")
+	f(mn, []int64{}, []float64{}, []string{"__value__"}, "")
+	f(mn, []int64{1000}, []float64{1.5}, nil, "")
+	f(mn, []int64{1000}, []float64{1.5}, []string{}, "")
+
+	f(mn, []int64{1000}, []float64{42.5}, []string{"__value__"}, "42.5\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__"}, "1704067200000\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:unix_s"}, "1704067200\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:unix_ms"}, "1704067200000\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:unix_ns"}, "1704067200000000000\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:rfc3339"}, "2024-01-01T00:00:00Z\n")
+
+	f(mn, []int64{1000}, []float64{1}, []string{"__name__"}, "cpu_usage\n")
+	f(mn, []int64{1000}, []float64{1}, []string{"job"}, "node\n")
+	f(mn, []int64{1000}, []float64{1}, []string{"instance"}, "localhost:9090\n")
+	f(mn, []int64{1000}, []float64{1}, []string{"missing_label"}, "\n")
+
+	// multiple fields
+	f(mn, []int64{1704067200000}, []float64{99.9},
+		[]string{"__timestamp__:unix_s", "__value__", "job"},
+		"1704067200,99.9,node\n")
+
+	// multiple rows
+	f(mn, []int64{1000, 2000}, []float64{1.1, 2.2},
+		[]string{"__value__", "__timestamp__"},
+		"1.1,1000\n2.2,2000\n")
+	f(mn, []int64{1000, 2000, 3000}, []float64{10, 20, 30},
+		[]string{"__timestamp__:unix_s", "__value__"},
+		"1,10\n2,20\n3,30\n")
+
+	// escaping for special characters in tag values
+	f(&storage.MetricName{
+		MetricGroup: []byte("m"),
+		Tags:        []storage.Tag{{Key: []byte("desc"), Value: []byte("a,b")}},
+	}, []int64{1000}, []float64{1}, []string{"desc"}, "\"a,b\"\n")
+
+	f(&storage.MetricName{
+		MetricGroup: []byte("m"),
+		Tags:        []storage.Tag{{Key: []byte("desc"), Value: []byte(`say "hello"`)}},
+	}, []int64{1000}, []float64{1}, []string{"desc"}, "\"say \\\"hello\\\"\"\n")
+
+	f(&storage.MetricName{
+		MetricGroup: []byte("m"),
+		Tags:        []storage.Tag{{Key: []byte("desc"), Value: []byte("line1\nline2")}},
+	}, []int64{1000}, []float64{1}, []string{"desc"}, "\"line1\\nline2\"\n")
+
+	// header and data line field counts must match
+	fieldNames := []string{"__name__", "job", "instance", "__value__", "__timestamp__:unix_s"}
+	header := ExportCSVHeader(fieldNames)
+	line := ExportCSVLine(&exportBlock{
+		mn:         mn,
+		timestamps: []int64{1704067200000},
+		values:     []float64{99.9},
+	}, fieldNames)
+	headerCommas := strings.Count(header, ",")
+	lineCommas := strings.Count(line, ",")
+	if headerCommas != lineCommas {
+		t.Fatalf("header has %d commas, data line has %d commas", headerCommas, lineCommas)
+	}
+	if headerCommas != len(fieldNames)-1 {
+		t.Fatalf("expected %d commas in header, got %d", len(fieldNames)-1, headerCommas)
+	}
+}

app/vmselect/prometheus/prometheus.go

@@ -175,6 +175,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 	w.Header().Set("Content-Type", "text/csv; charset=utf-8")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
+	WriteExportCSVHeader(bw, fieldNames)
 	sw := newScalableWriter(bw)
 	writeCSVLine := func(xb *exportBlock, workerID uint) error {
 		if len(xb.timestamps) == 0 {

app/vmselect/vmui/index.html

@@ -37,9 +37,9 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-KEOgEEMl.js"></script>
+  <script type="module" crossorigin src="./assets/index-C24BPpD_.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-Mr0bmX1E.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-BWBgVCcr.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
   <link rel="stylesheet" crossorigin href="./assets/index-D2OEy8Ra.css">
 </head>

app/vmstorage/main.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"strconv"
 	"strings"
@@ -56,11 +57,13 @@ var (
 	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod. "+
 		"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. "+
 		"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee")
-	maxHourlySeries = flag.Int("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
+	maxHourlySeries = flag.Int64("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
+		fmt.Sprintf("Setting this flag to '-1' sets limit to maximum possible value (%d) which is useful in order to enable series tracking without enforcing limits. ", math.MaxInt32)+
 		"See also -storage.maxDailySeries")
-	maxDailySeries = flag.Int("storage.maxDailySeries", 0, "The maximum number of unique series can be added to the storage during the last 24 hours. "+
+	maxDailySeries = flag.Int64("storage.maxDailySeries", 0, "The maximum number of unique series can be added to the storage during the last 24 hours. "+
 		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
+		fmt.Sprintf("Setting this flag to '-1' sets limit to maximum possible value (%d) which is useful in order to enable series tracking without enforcing limits. ", math.MaxInt32)+
 		"See also -storage.maxHourlySeries")
 
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 100e6, "The minimum free disk space at -storageDataPath after which the storage stops accepting new data")
@@ -142,8 +145,8 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	WG = syncwg.WaitGroup{}
 	opts := storage.OpenOptions{
 		Retention:             retentionPeriod.Duration(),
-		MaxHourlySeries:       *maxHourlySeries,
-		MaxDailySeries:        *maxDailySeries,
+		MaxHourlySeries:       getMaxHourlySeries(),
+		MaxDailySeries:        getMaxDailySeries(),
 		DisablePerDayIndex:    *disablePerDayIndex,
 		TrackMetricNamesStats: *trackMetricNamesStats,
 		IDBPrefillStart:       *idbPrefillStart,
@@ -603,10 +606,10 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="big_timestamp"}`, m.TooBigTimestampRows)
 	metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="small_timestamp"}`, m.TooSmallTimestampRows)
 	metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, m.InvalidRawMetricNames)
-	if *maxHourlySeries > 0 {
+	if getMaxHourlySeries() > 0 {
 		metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, m.HourlySeriesLimitRowsDropped)
 	}
-	if *maxDailySeries > 0 {
+	if getMaxDailySeries() > 0 {
 		metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`, m.DailySeriesLimitRowsDropped)
 	}
 
@@ -616,13 +619,13 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_slow_row_inserts_total`, m.SlowRowInserts)
 	metrics.WriteCounterUint64(w, `vm_slow_per_day_index_inserts_total`, m.SlowPerDayIndexInserts)
 
-	if *maxHourlySeries > 0 {
+	if getMaxHourlySeries() > 0 {
 		metrics.WriteGaugeUint64(w, `vm_hourly_series_limit_current_series`, m.HourlySeriesLimitCurrentSeries)
 		metrics.WriteGaugeUint64(w, `vm_hourly_series_limit_max_series`, m.HourlySeriesLimitMaxSeries)
 		metrics.WriteCounterUint64(w, `vm_hourly_series_limit_rows_dropped_total`, m.HourlySeriesLimitRowsDropped)
 	}
 
-	if *maxDailySeries > 0 {
+	if getMaxDailySeries() > 0 {
 		metrics.WriteGaugeUint64(w, `vm_daily_series_limit_current_series`, m.DailySeriesLimitCurrentSeries)
 		metrics.WriteGaugeUint64(w, `vm_daily_series_limit_max_series`, m.DailySeriesLimitMaxSeries)
 		metrics.WriteCounterUint64(w, `vm_daily_series_limit_rows_dropped_total`, m.DailySeriesLimitRowsDropped)
@@ -747,3 +750,21 @@ func jsonResponseError(w http.ResponseWriter, err error) {
 	errStr := err.Error()
 	fmt.Fprintf(w, `{"status":"error","msg":%s}`, stringsutil.JSONString(errStr))
 }
+
+func getMaxHourlySeries() int {
+	limit := *maxHourlySeries
+	if limit == -1 || limit > math.MaxInt32 {
+		return math.MaxInt32
+	}
+
+	return int(limit)
+}
+
+func getMaxDailySeries() int {
+	limit := *maxDailySeries
+	if limit == -1 || limit > math.MaxInt32 {
+		return math.MaxInt32
+	}
+
+	return int(limit)
+}

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.1 AS build-web-stage
+FROM golang:1.26.2 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/package-lock.json

@@ -17,7 +17,7 @@
         "react-input-mask": "^2.0.4",
         "react-router-dom": "^7.13.2",
         "uplot": "^1.6.32",
-        "vite": "^8.0.2",
+        "vite": "^8.0.7",
         "web-vitals": "^5.1.0"
       },
       "devDependencies": {
@@ -903,25 +903,27 @@
       }
     },
     "node_modules/@napi-rs/wasm-runtime": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz",
-      "integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==",
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.2.tgz",
+      "integrity": "sha512-sNXv5oLJ7ob93xkZ1XnxisYhGYXfaG9f65/ZgYuAu3qt7b3NadcOEhLvx28hv31PgX8SZJRYrAIPQilQmFpLVw==",
       "license": "MIT",
       "optional": true,
       "dependencies": {
-        "@emnapi/core": "^1.7.1",
-        "@emnapi/runtime": "^1.7.1",
         "@tybys/wasm-util": "^0.10.1"
       },
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/Brooooooklyn"
+      },
+      "peerDependencies": {
+        "@emnapi/core": "^1.7.1",
+        "@emnapi/runtime": "^1.7.1"
       }
     },
     "node_modules/@oxc-project/types": {
-      "version": "0.122.0",
-      "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.122.0.tgz",
-      "integrity": "sha512-oLAl5kBpV4w69UtFZ9xqcmTi+GENWOcPF7FCrczTiBbmC0ibXxCwyvZGbO39rCVEuLGAZM84DH0pUIyyv/YJzA==",
+      "version": "0.123.0",
+      "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.123.0.tgz",
+      "integrity": "sha512-YtECP/y8Mj1lSHiUWGSRzy/C6teUKlS87dEfuVKT09LgQbUsBW1rNg+MiJ4buGu3yuADV60gbIvo9/HplA56Ew==",
       "license": "MIT",
       "funding": {
         "url": "https://github.com/sponsors/Boshen"
@@ -1050,9 +1052,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1073,9 +1072,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1096,9 +1092,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1119,9 +1112,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1142,9 +1132,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1165,9 +1152,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1334,9 +1318,9 @@
       }
     },
     "node_modules/@rolldown/binding-android-arm64": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.11.tgz",
-      "integrity": "sha512-SJ+/g+xNnOh6NqYxD0V3uVN4W3VfnrGsC9/hoglicgTNfABFG9JjISvkkU0dNY84MNHLWyOgxP9v9Y9pX4S7+A==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.13.tgz",
+      "integrity": "sha512-5ZiiecKH2DXAVJTNN13gNMUcCDg4Jy8ZjbXEsPnqa248wgOVeYRX0iqXXD5Jz4bI9BFHgKsI2qmyJynstbmr+g==",
       "cpu": [
         "arm64"
       ],
@@ -1350,9 +1334,9 @@
       }
     },
     "node_modules/@rolldown/binding-darwin-arm64": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.11.tgz",
-      "integrity": "sha512-7WQgR8SfOPwmDZGFkThUvsmd/nwAWv91oCO4I5LS7RKrssPZmOt7jONN0cW17ydGC1n/+puol1IpoieKqQidmg==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.13.tgz",
+      "integrity": "sha512-tz/v/8G77seu8zAB3A5sK3UFoOl06zcshEzhUO62sAEtrEuW/H1CcyoupOrD+NbQJytYgA4CppXPzlrmp4JZKA==",
       "cpu": [
         "arm64"
       ],
@@ -1366,9 +1350,9 @@
       }
     },
     "node_modules/@rolldown/binding-darwin-x64": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.11.tgz",
-      "integrity": "sha512-39Ks6UvIHq4rEogIfQBoBRusj0Q0nPVWIvqmwBLaT6aqQGIakHdESBVOPRRLacy4WwUPIx4ZKzfZ9PMW+IeyUQ==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.13.tgz",
+      "integrity": "sha512-8DakphqOz8JrMYWTJmWA+vDJxut6LijZ8Xcdc4flOlAhU7PNVwo2MaWBF9iXjJAPo5rC/IxEFZDhJ3GC7NHvug==",
       "cpu": [
         "x64"
       ],
@@ -1382,9 +1366,9 @@
       }
     },
     "node_modules/@rolldown/binding-freebsd-x64": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.11.tgz",
-      "integrity": "sha512-jfsm0ZHfhiqrvWjJAmzsqiIFPz5e7mAoCOPBNTcNgkiid/LaFKiq92+0ojH+nmJmKYkre4t71BWXUZDNp7vsag==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.13.tgz",
+      "integrity": "sha512-4wBQFfjDuXYN/SVI8inBF3Aa+isq40rc6VMFbk5jcpolUBTe5cYnMsHZ51nFWsx3PVyyNN3vgoESki0Hmr/4BA==",
       "cpu": [
         "x64"
       ],
@@ -1398,9 +1382,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-arm-gnueabihf": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.11.tgz",
-      "integrity": "sha512-zjQaUtSyq1nVe3nxmlSCuR96T1LPlpvmJ0SZy0WJFEsV4kFbXcq2u68L4E6O0XeFj4aex9bEauqjW8UQBeAvfQ==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.13.tgz",
+      "integrity": "sha512-JW/e4yPIXLms+jmnbwwy5LA/LxVwZUWLN8xug+V200wzaVi5TEGIWQlh8o91gWYFxW609euI98OCCemmWGuPrw==",
       "cpu": [
         "arm"
       ],
@@ -1414,15 +1398,12 @@
       }
     },
     "node_modules/@rolldown/binding-linux-arm64-gnu": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.11.tgz",
-      "integrity": "sha512-WMW1yE6IOnehTcFE9eipFkm3XN63zypWlrJQ2iF7NrQ9b2LDRjumFoOGJE8RJJTJCTBAdmLMnJ8uVitACUUo1Q==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.13.tgz",
+      "integrity": "sha512-ZfKWpXiUymDnavepCaM6KG/uGydJ4l2nBmMxg60Ci4CbeefpqjPWpfaZM7PThOhk2dssqBAcwLc6rAyr0uTdXg==",
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1433,15 +1414,12 @@
       }
     },
     "node_modules/@rolldown/binding-linux-arm64-musl": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.11.tgz",
-      "integrity": "sha512-jfndI9tsfm4APzjNt6QdBkYwre5lRPUgHeDHoI7ydKUuJvz3lZeCfMsI56BZj+7BYqiKsJm7cfd/6KYV7ubrBg==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.13.tgz",
+      "integrity": "sha512-bmRg3O6Z0gq9yodKKWCIpnlH051sEfdVwt+6m5UDffAQMUUqU0xjnQqqAUm+Gu7ofAAly9DqiQDtKu2nPDEABA==",
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1452,15 +1430,12 @@
       }
     },
     "node_modules/@rolldown/binding-linux-ppc64-gnu": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.11.tgz",
-      "integrity": "sha512-ZlFgw46NOAGMgcdvdYwAGu2Q+SLFA9LzbJLW+iyMOJyhj5wk6P3KEE9Gct4xWwSzFoPI7JCdYmYMzVtlgQ+zfw==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.13.tgz",
+      "integrity": "sha512-8Wtnbw4k7pMYN9B/mOEAsQ8HOiq7AZ31Ig4M9BKn2So4xRaFEhtCSa4ZJaOutOWq50zpgR4N5+L/opnlaCx8wQ==",
       "cpu": [
         "ppc64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1471,15 +1446,12 @@
       }
     },
     "node_modules/@rolldown/binding-linux-s390x-gnu": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.11.tgz",
-      "integrity": "sha512-hIOYmuT6ofM4K04XAZd3OzMySEO4K0/nc9+jmNcxNAxRi6c5UWpqfw3KMFV4MVFWL+jQsSh+bGw2VqmaPMTLyw==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.13.tgz",
+      "integrity": "sha512-D/0Nlo8mQuxSMohNJUF2lDXWRsFDsHldfRRgD9bRgktj+EndGPj4DOV37LqDKPYS+osdyhZEH7fTakTAEcW7qg==",
       "cpu": [
         "s390x"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1490,15 +1462,12 @@
       }
     },
     "node_modules/@rolldown/binding-linux-x64-gnu": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.11.tgz",
-      "integrity": "sha512-qXBQQO9OvkjjQPLdUVr7Nr2t3QTZI7s4KZtfw7HzBgjbmAPSFwSv4rmET9lLSgq3rH/ndA3ngv3Qb8l2njoPNA==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.13.tgz",
+      "integrity": "sha512-eRrPvat2YaVQcwwKi/JzOP6MKf1WRnOCr+VaI3cTWz3ZoLcP/654z90lVCJ4dAuMEpPdke0n+qyAqXDZdIC4rA==",
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1509,15 +1478,12 @@
       }
     },
     "node_modules/@rolldown/binding-linux-x64-musl": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.11.tgz",
-      "integrity": "sha512-/tpFfoSTzUkH9LPY+cYbqZBDyyX62w5fICq9qzsHLL8uTI6BHip3Q9Uzft0wylk/i8OOwKik8OxW+QAhDmzwmg==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.13.tgz",
+      "integrity": "sha512-PsdONiFRp8hR8KgVjTWjZ9s7uA3uueWL0t74/cKHfM4dR5zXYv4AjB8BvA+QDToqxAFg4ZkcVEqeu5F7inoz5w==",
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1528,9 +1494,9 @@
       }
     },
     "node_modules/@rolldown/binding-openharmony-arm64": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.11.tgz",
-      "integrity": "sha512-mcp3Rio2w72IvdZG0oQ4bM2c2oumtwHfUfKncUM6zGgz0KgPz4YmDPQfnXEiY5t3+KD/i8HG2rOB/LxdmieK2g==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.13.tgz",
+      "integrity": "sha512-hCNXgC5dI3TVOLrPT++PKFNZ+1EtS0mLQwfXXXSUD/+rGlB65gZDwN/IDuxLpQP4x8RYYHqGomlUXzpO8aVI2w==",
       "cpu": [
         "arm64"
       ],
@@ -1544,25 +1510,27 @@
       }
     },
     "node_modules/@rolldown/binding-wasm32-wasi": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.11.tgz",
-      "integrity": "sha512-LXk5Hii1Ph9asuGRjBuz8TUxdc1lWzB7nyfdoRgI0WGPZKmCxvlKk8KfYysqtr4MfGElu/f/pEQRh8fcEgkrWw==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.13.tgz",
+      "integrity": "sha512-viLS5C5et8NFtLWw9Sw3M/w4vvnVkbWkO7wSNh3C+7G1+uCkGpr6PcjNDSFcNtmXY/4trjPBqUfcOL+P3sWy/g==",
       "cpu": [
         "wasm32"
       ],
       "license": "MIT",
       "optional": true,
       "dependencies": {
-        "@napi-rs/wasm-runtime": "^1.1.1"
+        "@emnapi/core": "1.9.1",
+        "@emnapi/runtime": "1.9.1",
+        "@napi-rs/wasm-runtime": "^1.1.2"
       },
       "engines": {
         "node": ">=14.0.0"
       }
     },
     "node_modules/@rolldown/binding-win32-arm64-msvc": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.11.tgz",
-      "integrity": "sha512-dDwf5otnx0XgRY1yqxOC4ITizcdzS/8cQ3goOWv3jFAo4F+xQYni+hnMuO6+LssHHdJW7+OCVL3CoU4ycnh35Q==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.13.tgz",
+      "integrity": "sha512-Fqa3Tlt1xL4wzmAYxGNFV36Hb+VfPc9PYU+E25DAnswXv3ODDu/yyWjQDbXMo5AGWkQVjLgQExuVu8I/UaZhPQ==",
       "cpu": [
         "arm64"
       ],
@@ -1576,9 +1544,9 @@
       }
     },
     "node_modules/@rolldown/binding-win32-x64-msvc": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.11.tgz",
-      "integrity": "sha512-LN4/skhSggybX71ews7dAj6r2geaMJfm3kMbK2KhFMg9B10AZXnKoLCVVgzhMHL0S+aKtr4p8QbAW8k+w95bAA==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.13.tgz",
+      "integrity": "sha512-/pLI5kPkGEi44TDlnbio3St/5gUFeN51YWNAk/Gnv6mEQBOahRBh52qVFVBpmrnU01n2yysvBML9Ynu7K4kGAQ==",
       "cpu": [
         "x64"
       ],
@@ -1592,9 +1560,9 @@
       }
     },
     "node_modules/@rolldown/pluginutils": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.11.tgz",
-      "integrity": "sha512-xQO9vbwBecJRv9EUcQ/y0dzSTJgA7Q6UVN7xp6B81+tBGSLVAK03yJ9NkJaUA7JFD91kbjxRSC/mDnmvXzbHoQ==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.13.tgz",
+      "integrity": "sha512-3ngTAv6F/Py35BsYbeeLeecvhMKdsKm4AoOETVhAA+Qc8nrA2I0kF7oa93mE9qnIurngOSpMnQ0x2nQY2FPviA==",
       "license": "MIT"
     },
     "node_modules/@rollup/pluginutils": {
@@ -5178,9 +5146,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -5201,9 +5166,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -5224,9 +5186,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -5247,9 +5206,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "musl"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -6158,13 +6114,13 @@
       }
     },
     "node_modules/rolldown": {
-      "version": "1.0.0-rc.11",
-      "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.11.tgz",
-      "integrity": "sha512-NRjoKMusSjfRbSYiH3VSumlkgFe7kYAa3pzVOsVYVFY3zb5d7nS+a3KGQ7hJKXuYWbzJKPVQ9Wxq2UvyK+ENpw==",
+      "version": "1.0.0-rc.13",
+      "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.13.tgz",
+      "integrity": "sha512-bvVj8YJmf0rq4pSFmH7laLa6pYrhghv3PRzrCdRAr23g66zOKVJ4wkvFtgohtPLWmthgg8/rkaqRHrpUEh0Zbw==",
       "license": "MIT",
       "dependencies": {
-        "@oxc-project/types": "=0.122.0",
-        "@rolldown/pluginutils": "1.0.0-rc.11"
+        "@oxc-project/types": "=0.123.0",
+        "@rolldown/pluginutils": "1.0.0-rc.13"
       },
       "bin": {
         "rolldown": "bin/cli.mjs"
@@ -6173,21 +6129,21 @@
         "node": "^20.19.0 || >=22.12.0"
       },
       "optionalDependencies": {
-        "@rolldown/binding-android-arm64": "1.0.0-rc.11",
-        "@rolldown/binding-darwin-arm64": "1.0.0-rc.11",
-        "@rolldown/binding-darwin-x64": "1.0.0-rc.11",
-        "@rolldown/binding-freebsd-x64": "1.0.0-rc.11",
-        "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.11",
-        "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.11",
-        "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.11",
-        "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.11",
-        "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.11",
-        "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.11",
-        "@rolldown/binding-linux-x64-musl": "1.0.0-rc.11",
-        "@rolldown/binding-openharmony-arm64": "1.0.0-rc.11",
-        "@rolldown/binding-wasm32-wasi": "1.0.0-rc.11",
-        "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.11",
-        "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.11"
+        "@rolldown/binding-android-arm64": "1.0.0-rc.13",
+        "@rolldown/binding-darwin-arm64": "1.0.0-rc.13",
+        "@rolldown/binding-darwin-x64": "1.0.0-rc.13",
+        "@rolldown/binding-freebsd-x64": "1.0.0-rc.13",
+        "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.13",
+        "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.13",
+        "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.13",
+        "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.13",
+        "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.13",
+        "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.13",
+        "@rolldown/binding-linux-x64-musl": "1.0.0-rc.13",
+        "@rolldown/binding-openharmony-arm64": "1.0.0-rc.13",
+        "@rolldown/binding-wasm32-wasi": "1.0.0-rc.13",
+        "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.13",
+        "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.13"
       }
     },
     "node_modules/rxjs": {
@@ -6437,7 +6393,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6454,7 +6409,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6471,7 +6425,6 @@
       "cpu": [
         "arm"
       ],
-      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6488,7 +6441,6 @@
       "cpu": [
         "arm64"
       ],
-      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6505,7 +6457,6 @@
       "cpu": [
         "riscv64"
       ],
-      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6522,7 +6473,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6539,7 +6489,6 @@
       "cpu": [
         "riscv64"
       ],
-      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6556,7 +6505,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -7382,15 +7330,15 @@
       "license": "MIT"
     },
     "node_modules/vite": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.2.tgz",
-      "integrity": "sha512-1gFhNi+bHhRE/qKZOJXACm6tX4bA3Isy9KuKF15AgSRuRazNBOJfdDemPBU16/mpMxApDPrWvZ08DcLPEoRnuA==",
+      "version": "8.0.7",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.7.tgz",
+      "integrity": "sha512-P1PbweD+2/udplnThz3btF4cf6AgPky7kk23RtHUkJIU5BIxwPprhRGmOAHs6FTI7UiGbTNrgNP6jSYD6JaRnw==",
       "license": "MIT",
       "dependencies": {
         "lightningcss": "^1.32.0",
-        "picomatch": "^4.0.3",
+        "picomatch": "^4.0.4",
         "postcss": "^8.5.8",
-        "rolldown": "1.0.0-rc.11",
+        "rolldown": "1.0.0-rc.13",
         "tinyglobby": "^0.2.15"
       },
       "bin": {
@@ -7408,7 +7356,7 @@
       "peerDependencies": {
         "@types/node": "^20.19.0 || >=22.12.0",
         "@vitejs/devtools": "^0.1.0",
-        "esbuild": "^0.27.0",
+        "esbuild": "^0.27.0 || ^0.28.0",
         "jiti": ">=1.21.0",
         "less": "^4.0.0",
         "sass": "^1.70.0",

app/vmui/packages/vmui/package.json

@@ -29,7 +29,7 @@
     "react-input-mask": "^2.0.4",
     "react-router-dom": "^7.13.2",
     "uplot": "^1.6.32",
-    "vite": "^8.0.2",
+    "vite": "^8.0.7",
     "web-vitals": "^5.1.0"
   },
   "devDependencies": {

app/vmui/packages/vmui/src/api/query-range.ts

@@ -16,23 +16,29 @@ export const getExportDataUrl = (server: string, query: string, period: TimePara
   return `${server}/api/v1/export?${params}`;
 };
 
-export const getExportCSVDataUrl = (server: string, query: string[], period: TimeParams, reduceMemUsage: boolean): string => {
+const getBaseParams = (period: TimeParams, query: string[]): URLSearchParams => {
   const params = new URLSearchParams({
     start: period.start.toString(),
     end: period.end.toString(),
-    format: "__name__,__value__,__timestamp__:unix_ms",
   });
   query.forEach((q => params.append("match[]", q)));
+  return params;
+};
+
+export const getLabelsUrl = (server: string, query: string[], period: TimeParams): string => {
+  const params = getBaseParams(period, query);
+  return `${server}/api/v1/labels?${params}`;
+};
+
+export const getExportCSVDataUrl = (server: string, query: string[], period: TimeParams, reduceMemUsage: boolean, format: string): string => {
+  const params = getBaseParams(period, query);
+  params.set("format", format);
   if (reduceMemUsage) params.set("reduce_mem_usage", "1");
   return `${server}/api/v1/export/csv?${params}`;
 };
 
 export const getExportJSONDataUrl = (server: string, query: string[], period: TimeParams, reduceMemUsage: boolean): string => {
-  const params = new URLSearchParams({
-    start: period.start.toString(),
-    end: period.end.toString(),
-  });
-  query.forEach((q => params.append("match[]", q)));
+  const params = getBaseParams(period, query);
   if (reduceMemUsage) params.set("reduce_mem_usage", "1");
   return `${server}/api/v1/export?${params}`;
 };

app/vmui/packages/vmui/src/api/raw-query.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it, vi } from "vitest";
+import { fetchRawQueryCSVExport } from "./raw-query";
+
+describe("fetchRawQueryCSVExport", () => {
+  it.skip("requests all label columns before exporting CSV data", async () => {
+    const fetchMock = vi.fn()
+      .mockResolvedValueOnce({
+        ok: true,
+        json: async () => ({ data: ["job", "__name__", "instance"] }),
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        text: async () => "up,localhost:9100,node_exporter,1,1710000000000",
+      });
+
+    const result = await fetchRawQueryCSVExport(
+      "http://localhost:8428",
+      ["up"],
+      { start: 1710000000, end: 1710000300, step: "15s", date: "2024-03-09T16:05:00Z" },
+      false,
+      fetchMock as unknown as typeof fetch,
+    );
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(fetchMock.mock.calls[0][0]).toBe("http://localhost:8428/api/v1/labels?start=1710000000&end=1710000300&match%5B%5D=up");
+    expect(fetchMock.mock.calls[1][0]).toBe("http://localhost:8428/api/v1/export/csv?start=1710000000&end=1710000300&match%5B%5D=up&format=__name__%2Cinstance%2Cjob%2C__value__%2C__timestamp__%3Aunix_ms");
+    expect(result).toBe("up,localhost:9100,node_exporter,1,1710000000000");
+  });
+});

app/vmui/packages/vmui/src/api/raw-query.ts

@@ -0,0 +1,31 @@
+import { getExportCSVDataUrl, getLabelsUrl } from "./query-range";
+import { TimeParams } from "../types";
+import { getCSVExportColumns } from "../utils/csv";
+
+interface LabelsResponse {
+  data?: string[];
+}
+
+export const fetchRawQueryCSVExport = async (
+  serverUrl: string,
+  query: string[],
+  period: TimeParams,
+  reduceMemUsage: boolean,
+  fetchFn: typeof fetch = fetch,
+): Promise<string> => {
+  const labelsResponse = await fetchFn(getLabelsUrl(serverUrl, query, period));
+  if (!labelsResponse.ok) {
+    throw new Error(await labelsResponse.text());
+  }
+
+  const { data = [] } = (await labelsResponse.json()) as LabelsResponse;
+  const columns = getCSVExportColumns(data);
+  const format = columns.join(",");
+
+  const response = await fetchFn(getExportCSVDataUrl(serverUrl, query, period, reduceMemUsage, format));
+  if (!response.ok) {
+    throw new Error(await response.text());
+  }
+
+  return await response.text();
+};

app/vmui/packages/vmui/src/pages/RawQueryPage/hooks/useFetchExport.ts

@@ -6,10 +6,11 @@ import { useTimeState } from "../../../state/time/TimeStateContext";
 import { useAppState } from "../../../state/common/StateContext";
 import { useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
 import { isValidHttpUrl } from "../../../utils/url";
-import { getExportCSVDataUrl, getExportDataUrl, getExportJSONDataUrl } from "../../../api/query-range";
+import { getExportDataUrl, getExportJSONDataUrl } from "../../../api/query-range";
 import { parseLineToJSON } from "../../../utils/json";
 import { downloadCSV, downloadJSON } from "../../../utils/file";
 import { useSnack } from "../../../contexts/Snackbar";
+import { fetchRawQueryCSVExport } from "../../../api/raw-query";
 
 interface FetchQueryParams {
   hideQuery?: number[];
@@ -67,11 +68,8 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
     const getFilename = (format: ExportFormats) => `vmui_export_${query.join("_")}_${period.start}_${period.end}.${format}`;
     return {
       csv: async () => {
-        const url = getExportCSVDataUrl(serverUrl, query, period, reduceMemUsage);
-        const response = await fetch(url);
         try {
-          let text = await response.text();
-          text = "name,value,timestamp\n" + text;
+          const text = await fetchRawQueryCSVExport(serverUrl, query, period, reduceMemUsage);
           downloadCSV(text, getFilename("csv"));
         } catch (e) {
           console.error(e);

app/vmui/packages/vmui/src/utils/csv.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { formatValueToCSV } from "./csv";
+import { formatValueToCSV, getCSVExportColumns } from "./csv";
 
 describe("formatValueToCSV", () => {
   it("should wrap value in quotes if it contains a comma", () => {
@@ -32,3 +32,10 @@ describe("formatValueToCSV", () => {
     expect(result).toBe("");
   });
 });
+
+describe("getCSVExportColumns", () => {
+  it("should prepend metric name and append value and timestamp columns", () => {
+    const result = getCSVExportColumns(["instance", "__name__", "job", "instance"]);
+    expect(result.join(",")).toEqual("__name__,instance,job,__value__,__timestamp__:unix_ms");
+  });
+});

app/vmui/packages/vmui/src/utils/csv.ts

@@ -2,3 +2,8 @@ export const formatValueToCSV= (value: string) =>
   (value.includes(",") || value.includes("\n") || value.includes("\""))
     ? "\"" + value.replace(/"/g, "\"\"") + "\""
     : value;
+
+export const getCSVExportColumns = (labelNames: string[]) => {
+  const labels = Array.from(new Set(labelNames.filter((label) => label && label !== "__name__"))).sort();
+  return ["__name__", ...labels, "__value__", "__timestamp__:unix_ms"];
+};

apptest/model.go

@@ -88,6 +88,7 @@ type QueryOpts struct {
 	MaxLookback    string
 	LatencyOffset  string
 	Format         string
+	NoCache        string
 }
 
 func (qos *QueryOpts) asURLValues() url.Values {
@@ -112,6 +113,7 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("max_lookback", qos.MaxLookback)
 	addNonEmpty("latency_offset", qos.LatencyOffset)
 	addNonEmpty("format", qos.Format)
+	addNonEmpty("nocache", qos.NoCache)
 
 	return uv
 }

apptest/tests/backup_restore_test.go

@@ -28,7 +28,6 @@ func TestSingleBackupRestore(t *testing.T) {
 			return tc.MustStartVmsingle("vmsingle", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
-				"-search.maxStalenessInterval=1m",
 			})
 		},
 		stopSUT: func() {
@@ -70,9 +69,7 @@ func TestClusterBackupRestore(t *testing.T) {
 				VminsertInstance: "vminsert",
 				VminsertFlags:    []string{},
 				VmselectInstance: "vmselect",
-				VmselectFlags: []string{
-					"-search.maxStalenessInterval=1m",
-				},
+				VmselectFlags:    []string{},
 			})
 		},
 		stopSUT: func() {
@@ -100,15 +97,14 @@ func TestClusterBackupRestore(t *testing.T) {
 func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	t := tc.T()
 
-	const msecPerMinute = 60 * 1000
-	genData := func(count int, prefix string, start int64) (recs []string, wantSeries []map[string]string, wantQueryResults []*apptest.QueryResult) {
+	genData := func(count int, prefix string, start, step int64) (recs []string, wantSeries []map[string]string, wantQueryResults []*apptest.QueryResult) {
 		recs = make([]string, count)
 		wantSeries = make([]map[string]string, count)
 		wantQueryResults = make([]*apptest.QueryResult, count)
 		for i := range count {
 			name := fmt.Sprintf("%s_%03d", prefix, i)
 			value := float64(i)
-			timestamp := start + int64(i)*msecPerMinute
+			timestamp := start + int64(i)*step
 
 			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
 			wantSeries[i] = map[string]string{"__name__": name}
@@ -148,15 +144,17 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 
 	// assertSeries retrieves all data from the storage and compares it with the
 	// expected result.
-	assertQueryResults := func(app apptest.PrometheusQuerier, query string, start, end int64, want []*apptest.QueryResult) {
+	assertQueryResults := func(app apptest.PrometheusQuerier, query string, start, end, step int64, want []*apptest.QueryResult) {
 		t.Helper()
 		tc.Assert(&apptest.AssertOptions{
 			Msg: "unexpected /api/v1/query_range response",
 			Got: func() any {
 				return app.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
-					Start: fmt.Sprintf("%d", start),
-					End:   fmt.Sprintf("%d", end),
-					Step:  "60s",
+					Start:       fmt.Sprintf("%d", start),
+					End:         fmt.Sprintf("%d", end),
+					Step:        fmt.Sprintf("%dms", step),
+					MaxLookback: fmt.Sprintf("%dms", step-1),
+					NoCache:     "1",
 				})
 			},
 			Want: &apptest.PrometheusAPIV1QueryResponse{
@@ -167,7 +165,6 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 				},
 			},
 			FailNow: true,
-			Retries: 300,
 		})
 	}
 
@@ -194,8 +191,9 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	// below.
 	const numMetrics = 1000
 	// With 1000 metrics (one per minute), the time range spans 2 months.
-	end := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
-	start := end - numMetrics*msecPerMinute
+	start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	step := (end - start) / numMetrics
 
 	// Verify backup/restore:
 	//
@@ -209,8 +207,8 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	// - Start vmsingle
 	// - Ensure that the queries return batch1 data only.
 
-	batch1Data, wantBatch1Series, wantBatch1QueryResults := genData(numMetrics, "batch1", start)
-	batch2Data, wantBatch2Series, wantBatch2QueryResults := genData(numMetrics, "batch2", start)
+	batch1Data, wantBatch1Series, wantBatch1QueryResults := genData(numMetrics, "batch1", start, step)
+	batch2Data, wantBatch2Series, wantBatch2QueryResults := genData(numMetrics, "batch2", start, step)
 	wantBatch12Series := slices.Concat(wantBatch1Series, wantBatch2Series)
 	wantBatch12QueryResults := slices.Concat(wantBatch1QueryResults, wantBatch2QueryResults)
 
@@ -219,13 +217,14 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	sut.PrometheusAPIV1ImportPrometheus(t, batch1Data, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
-	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, step, wantBatch1QueryResults)
+
 	createBackup(sut, "batch1")
 
 	sut.PrometheusAPIV1ImportPrometheus(t, batch2Data, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 	assertSeries(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12Series)
-	assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12QueryResults)
+	assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, step, wantBatch12QueryResults)
 	createBackup(sut, "batch12")
 
 	opts.stopSUT()
@@ -235,5 +234,5 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	sut = opts.startSUT()
 
 	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
-	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, step, wantBatch1QueryResults)
 }

apptest/tests/ingestprotocols_test.go

@@ -1,7 +1,9 @@
 package tests
 
 import (
+	"fmt"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -297,6 +299,132 @@ func TestSingleIngestionProtocols(t *testing.T) {
 
 }
 
+func TestSingleCardinalityLimiter(t *testing.T) {
+	waitFor := func(f func() bool) {
+		const (
+			retries = 20
+			period  = 100 * time.Millisecond
+		)
+
+		t.Helper()
+
+		for i := 0; i < retries; i++ {
+			if f() {
+				return
+			}
+			time.Sleep(period)
+		}
+		t.Fatalf("timed out waiting for retry #%d", retries)
+	}
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	singleHourly := tc.MustStartVmsingle("vmsingle-hourly", []string{
+		"-retentionPeriod=100y",
+		"-storage.maxHourlySeries=1",
+	})
+
+	singleHourly.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	if v := singleHourly.GetIntMetric(t, "vm_hourly_series_limit_max_series"); v != 1 {
+		t.Fatalf("unexpected vm_hourly_series_limit_max_series value: %d", v)
+	}
+
+	if v := singleHourly.GetIntMetric(t, "vm_hourly_series_limit_current_series"); v != 1 {
+		t.Fatalf("unexpected vm_hourly_series_limit_current_series value: %d", v)
+	}
+
+	if v := singleHourly.GetIntMetric(t, "vm_hourly_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_hourly_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	singleHourly.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar2 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return singleHourly.GetIntMetric(t, "vm_hourly_series_limit_rows_dropped_total") > 0
+		},
+	)
+
+	singleDaily := tc.MustStartVmsingle("vmsingle-daily", []string{
+		"-retentionPeriod=100y",
+		"-storage.maxDailySeries=1",
+	})
+
+	singleDaily.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	if v := singleDaily.GetIntMetric(t, "vm_daily_series_limit_max_series"); v != 1 {
+		t.Fatalf("unexpected vm_daily_series_limit_max_series value: %d", v)
+	}
+
+	if v := singleDaily.GetIntMetric(t, "vm_daily_series_limit_current_series"); v != 1 {
+		t.Fatalf("unexpected vm_daily_series_limit_current_series value: %d", v)
+	}
+
+	if v := singleDaily.GetIntMetric(t, "vm_daily_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_daily_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	singleDaily.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar2 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return singleDaily.GetIntMetric(t, "vm_daily_series_limit_rows_dropped_total") > 0
+		},
+	)
+
+	singleUnlimited := tc.MustStartVmsingle("vmsingle-unlimited", []string{
+		"-retentionPeriod=100y",
+		"-storage.maxHourlySeries=-1",
+		"-storage.maxDailySeries=-1",
+	})
+	metrics := make([]string, 0, 100)
+	for i := range 100 {
+		metrics = append(metrics, fmt.Sprintf("foo_bar%d 1 1652169600000", i)) // 2022-05-10T08:00:00Z
+	}
+
+	singleUnlimited.PrometheusAPIV1ImportPrometheus(t, metrics, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return singleUnlimited.GetIntMetric(t, "vm_hourly_series_limit_current_series") > 0
+		},
+	)
+
+	if v := singleUnlimited.GetIntMetric(t, "vm_hourly_series_limit_max_series"); v == 0 {
+		t.Fatalf("unexpected vm_hourly_series_limit_max_series value: %d", v)
+	}
+
+	if v := singleUnlimited.GetIntMetric(t, "vm_hourly_series_limit_current_series"); v != 100 {
+		t.Fatalf("unexpected vm_hourly_series_limit_current_series value: %d", v)
+	}
+
+	if v := singleUnlimited.GetIntMetric(t, "vm_hourly_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_hourly_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	if v := singleUnlimited.GetIntMetric(t, "vm_daily_series_limit_max_series"); v == 0 {
+		t.Fatalf("unexpected vm_daily_series_limit_max_series value: %d", v)
+	}
+
+	if v := singleUnlimited.GetIntMetric(t, "vm_daily_series_limit_current_series"); v != 100 {
+		t.Fatalf("unexpected vm_daily_series_limit_current_series value: %d", v)
+	}
+
+	if v := singleUnlimited.GetIntMetric(t, "vm_daily_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_daily_series_limit_rows_dropped_total value: %d", v)
+	}
+}
+
 func TestClusterIngestionProtocols(t *testing.T) {
 	fs.MustRemoveDir(t.Name())
 	tc := apptest.NewTestCase(t)
@@ -591,3 +719,145 @@ func TestClusterIngestionProtocols(t *testing.T) {
 	})
 
 }
+
+func TestClusterCardinalityLimiter(t *testing.T) {
+	waitFor := func(f func() bool) {
+		const (
+			retries = 20
+			period  = 100 * time.Millisecond
+		)
+
+		t.Helper()
+
+		for i := 0; i < retries; i++ {
+			if f() {
+				return
+			}
+			time.Sleep(period)
+		}
+		t.Fatalf("timed out waiting for retry #%d", retries)
+	}
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	// Test hourly series limit
+	vmstorageHourly := tc.MustStartVmstorage("vmstorage-hourly", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-hourly",
+		"-retentionPeriod=100y",
+		"-storage.maxHourlySeries=1",
+	})
+	vminsertHourly := tc.MustStartVminsert("vminsert-hourly", []string{
+		"-storageNode=" + vmstorageHourly.VminsertAddr(),
+	})
+
+	vminsertHourly.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	if v := vmstorageHourly.GetIntMetric(t, "vm_hourly_series_limit_max_series"); v != 1 {
+		t.Fatalf("unexpected vm_hourly_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmstorageHourly.GetIntMetric(t, "vm_hourly_series_limit_current_series"); v != 1 {
+		t.Fatalf("unexpected vm_hourly_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmstorageHourly.GetIntMetric(t, "vm_hourly_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_hourly_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	vminsertHourly.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar2 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return vmstorageHourly.GetIntMetric(t, "vm_hourly_series_limit_rows_dropped_total") > 0
+		},
+	)
+
+	// Test daily series limit
+	vmstorageDaily := tc.MustStartVmstorage("vmstorage-daily", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-daily",
+		"-retentionPeriod=100y",
+		"-storage.maxDailySeries=1",
+	})
+	vminsertDaily := tc.MustStartVminsert("vminsert-daily", []string{
+		"-storageNode=" + vmstorageDaily.VminsertAddr(),
+	})
+
+	vminsertDaily.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	if v := vmstorageDaily.GetIntMetric(t, "vm_daily_series_limit_max_series"); v != 1 {
+		t.Fatalf("unexpected vm_daily_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmstorageDaily.GetIntMetric(t, "vm_daily_series_limit_current_series"); v != 1 {
+		t.Fatalf("unexpected vm_daily_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmstorageDaily.GetIntMetric(t, "vm_daily_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_daily_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	vminsertDaily.PrometheusAPIV1ImportPrometheus(t, []string{
+		"foo_bar2 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return vmstorageDaily.GetIntMetric(t, "vm_daily_series_limit_rows_dropped_total") > 0
+		},
+	)
+
+	// Test unlimited series
+	vmstorageUnlimited := tc.MustStartVmstorage("vmstorage-unlimited", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-unlimited",
+		"-retentionPeriod=100y",
+		"-storage.maxHourlySeries=-1",
+		"-storage.maxDailySeries=-1",
+	})
+	vminsertUnlimited := tc.MustStartVminsert("vminsert-unlimited", []string{
+		"-storageNode=" + vmstorageUnlimited.VminsertAddr(),
+	})
+
+	metrics := make([]string, 0, 100)
+	for i := range 100 {
+		metrics = append(metrics, fmt.Sprintf("foo_bar%d 1 1652169600000", i)) // 2022-05-10T08:00:00Z
+	}
+
+	vminsertUnlimited.PrometheusAPIV1ImportPrometheus(t, metrics, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return vmstorageUnlimited.GetIntMetric(t, "vm_hourly_series_limit_current_series") > 0
+		},
+	)
+
+	if v := vmstorageUnlimited.GetIntMetric(t, "vm_hourly_series_limit_max_series"); v == 0 {
+		t.Fatalf("unexpected vm_hourly_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmstorageUnlimited.GetIntMetric(t, "vm_hourly_series_limit_current_series"); v != 100 {
+		t.Fatalf("unexpected vm_hourly_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmstorageUnlimited.GetIntMetric(t, "vm_hourly_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_hourly_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	if v := vmstorageUnlimited.GetIntMetric(t, "vm_daily_series_limit_max_series"); v == 0 {
+		t.Fatalf("unexpected vm_daily_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmstorageUnlimited.GetIntMetric(t, "vm_daily_series_limit_current_series"); v != 100 {
+		t.Fatalf("unexpected vm_daily_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmstorageUnlimited.GetIntMetric(t, "vm_daily_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vm_daily_series_limit_rows_dropped_total value: %d", v)
+	}
+}

apptest/tests/multilevel_test.go

@@ -3,7 +3,9 @@ package tests
 import (
 	"fmt"
 	"math/rand/v2"
+	"net"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
@@ -70,3 +72,162 @@ func TestClusterMultilevelSelect(t *testing.T) {
 	assertSeries(vmselectL1)
 	assertSeries(vmselectL2)
 }
+
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678.
+func TestClusterMultilevelPartialResponse(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+	// Set up the following multi-level cluster configuration:
+	//
+	//				  			     				|--> available vmstorage
+	//					 	  |	------> vmselect1 --|
+	//						  |	     				|--> available vmstorage
+	// global-vmselect -------|
+	//				  		  |	     				|--> available vmstorage
+	// 					 	  |	------> vmselect2 --|
+	//						  	     				|--> unavailable vmstorage
+
+	vmstorage1 := tc.MustStartVmstorage("vmstorage1", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage1",
+	})
+	vmstorage2 := tc.MustStartVmstorage("vmstorage2", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage2",
+	})
+	regionalVmselect1 := tc.MustStartVmselect("regional-vmselect1", []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	})
+	regionalVmselect2 := tc.MustStartVmselect("regional-vmselect2", []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + noopTCPServerAddr(t),
+	})
+	globalVmselect := tc.MustStartVmselect("global-vmselect", []string{
+		"-storageNode=" + regionalVmselect1.ClusternativeListenAddr() + "," + regionalVmselect2.ClusternativeListenAddr(),
+	})
+
+	// 1. /api/v1/query
+	qopts := apptest.QueryOpts{Tenant: "0"}
+	assertQuery := func(app *apptest.Vmselect, want *apptest.PrometheusAPIV1QueryResponse) {
+		t.Helper()
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/query response",
+			Got: func() any {
+				res := app.PrometheusAPIV1Query(t, `{__name__=~".*"}`, qopts)
+				res.Sort()
+				return res
+			},
+			Want: want,
+		})
+	}
+	// regional-vmselect1 should return full response.
+	assertQuery(regionalVmselect1, &apptest.PrometheusAPIV1QueryResponse{
+		Status:    "success",
+		IsPartial: false,
+		Data:      &apptest.QueryData{ResultType: "vector", Result: []*apptest.QueryResult{}},
+	})
+	// regional-vmselect2 should return partial response.
+	assertQuery(regionalVmselect2, &apptest.PrometheusAPIV1QueryResponse{
+		Status:    "success",
+		IsPartial: true,
+		Data:      &apptest.QueryData{ResultType: "vector", Result: []*apptest.QueryResult{}},
+	})
+	// global-vmselect should return partial response.
+	assertQuery(globalVmselect, &apptest.PrometheusAPIV1QueryResponse{
+		Status:    "success",
+		IsPartial: true,
+		Data:      &apptest.QueryData{ResultType: "vector", Result: []*apptest.QueryResult{}},
+	})
+
+	// 2. /api/v1/labels
+	start := time.Now().Unix()
+	assertLabel := func(app *apptest.Vmselect, want *apptest.PrometheusAPIV1LabelsResponse) {
+		t.Helper()
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/label response",
+
+			Got: func() any {
+				res := app.PrometheusAPIV1Labels(t, `{__name__="up"}`, apptest.QueryOpts{
+					Start: fmt.Sprintf("%d", start-100),
+					End:   fmt.Sprintf("%d", start),
+				})
+				return res
+			},
+			Want: want,
+		})
+	}
+
+	// regional-vmselect1 should return full response.
+	assertLabel(regionalVmselect1, &apptest.PrometheusAPIV1LabelsResponse{
+		Status:    "success",
+		IsPartial: false,
+		Data:      make([]string, 0),
+	})
+	// regional-vmselect2 should return partial response.
+	assertLabel(regionalVmselect2, &apptest.PrometheusAPIV1LabelsResponse{
+		Status:    "success",
+		IsPartial: true,
+		Data:      make([]string, 0),
+	})
+	// global-vmselect should return partial response.
+	assertLabel(globalVmselect, &apptest.PrometheusAPIV1LabelsResponse{
+		Status:    "success",
+		IsPartial: true,
+		Data:      make([]string, 0),
+	})
+
+	// 3. /api/v1/label/%s/values
+	assertSeries := func(app *apptest.Vmselect, want *apptest.PrometheusAPIV1SeriesResponse) {
+		t.Helper()
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+
+			Got: func() any {
+				res := app.PrometheusAPIV1Series(t, `{__name__="up"}`, apptest.QueryOpts{
+					Start: fmt.Sprintf("%d", start-100),
+					End:   fmt.Sprintf("%d", start),
+				})
+				return res
+			},
+			Want: want,
+		})
+	}
+
+	// regional-vmselect1 should return full response.
+	assertSeries(regionalVmselect1, &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: false,
+		Data:      make([]map[string]string, 0),
+	})
+	// regional-vmselect2 should return partial response.
+	assertSeries(regionalVmselect2, &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: true,
+		Data:      make([]map[string]string, 0),
+	})
+	// global-vmselect should return partial response.
+	assertSeries(globalVmselect, &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: true,
+		Data:      make([]map[string]string, 0),
+	})
+}
+
+// noopTCPServerAddr start local tcp server,
+// which immediately closes any incoming connections
+// and return it's address
+func noopTCPServerAddr(t *testing.T) string {
+	t.Helper()
+	ln, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("failed to create listener: %v", err)
+	}
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			conn.Close()
+		}
+	}()
+	t.Cleanup(func() { ln.Close() })
+	return ln.Addr().String()
+}

apptest/tests/vmagent_remotewrite_test.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -304,6 +305,11 @@ func TestSingleVMAgentDropOnOverload(t *testing.T) {
 		// See initRemoteWriteCtxs function in remotewrite.go for details.
 		"-remoteWrite.maxRowsPerBlock=1000000000",
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
+
+		// Delay retry logic to avoid race conditions with waitFor assertions.
+		// It improves the test stability on resource-constrained runners.
+		// Should be bigger than retries * period
+		"-remoteWrite.retryMinInterval=3s",
 	}, ``)
 
 	const (
@@ -362,3 +368,147 @@ func TestSingleVMAgentDropOnOverload(t *testing.T) {
 		},
 	)
 }
+
+func TestSingleVMAgentCardinalityLimiter(t *testing.T) {
+	waitFor := func(f func() bool) {
+		const (
+			retries = 20
+			period  = 100 * time.Millisecond
+		)
+
+		t.Helper()
+
+		for i := 0; i < retries; i++ {
+			if f() {
+				return
+			}
+			time.Sleep(period)
+		}
+		t.Fatalf("timed out waiting for retry #%d", retries)
+	}
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	remoteWriteSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer remoteWriteSrv.Close()
+
+	// Verify hourly limit is applied
+	vmagent := tc.MustStartVmagent("vmagent-hourly", []string{
+		`-remoteWrite.flushInterval=50ms`,
+		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
+		"-remoteWrite.maxRowsPerBlock=1",
+		"-remoteWrite.maxHourlySeries=1",
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-hourly",
+	}, ``)
+
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	if v := vmagent.GetIntMetric(t, "vmagent_hourly_series_limit_max_series"); v != 1 {
+		t.Fatalf("unexpected vmagent_hourly_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmagent.GetIntMetric(t, "vmagent_hourly_series_limit_current_series"); v != 1 {
+		t.Fatalf("unexpected vmagent_hourly_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmagent.GetIntMetric(t, "vmagent_hourly_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vmagent_hourly_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	vmagent.APIV1ImportPrometheusNoWaitFlush(t, []string{
+		"foo_bar2 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return vmagent.GetIntMetric(t, "vmagent_hourly_series_limit_rows_dropped_total") > 0
+		},
+	)
+
+	// Daily limits
+	vmagent2 := tc.MustStartVmagent("vmagent-daily", []string{
+		`-remoteWrite.flushInterval=50ms`,
+		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
+		"-remoteWrite.maxRowsPerBlock=1",
+		"-remoteWrite.maxDailySeries=1",
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-daily",
+	}, ``)
+
+	vmagent2.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	if v := vmagent2.GetIntMetric(t, "vmagent_daily_series_limit_max_series"); v != 1 {
+		t.Fatalf("unexpected vmagent_daily_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmagent2.GetIntMetric(t, "vmagent_daily_series_limit_current_series"); v != 1 {
+		t.Fatalf("unexpected vmagent_daily_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmagent2.GetIntMetric(t, "vmagent_daily_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vmagent_daily_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	vmagent2.APIV1ImportPrometheusNoWaitFlush(t, []string{
+		"foo_bar2 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return vmagent2.GetIntMetric(t, "vmagent_daily_series_limit_rows_dropped_total") > 0
+		},
+	)
+
+	// test running with unlimited tracker
+	vmagent3 := tc.MustStartVmagent("vmagent-unlimited", []string{
+		`-remoteWrite.flushInterval=50ms`,
+		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
+		"-remoteWrite.maxRowsPerBlock=10",
+		"-remoteWrite.maxDailySeries=-1",
+		"-remoteWrite.maxHourlySeries=-1",
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-unlimited",
+	}, ``)
+
+	metrics := make([]string, 0, 100)
+	for i := range 100 {
+		metrics = append(metrics, fmt.Sprintf("foo_bar%d 1 1652169600000", i)) // 2022-05-10T08:00:00Z
+	}
+
+	vmagent3.APIV1ImportPrometheusNoWaitFlush(t, metrics, apptest.QueryOpts{})
+
+	waitFor(
+		func() bool {
+			return vmagent3.GetIntMetric(t, "vmagent_hourly_series_limit_current_series") > 0
+		},
+	)
+
+	if v := vmagent3.GetIntMetric(t, "vmagent_hourly_series_limit_max_series"); v != math.MaxInt32 {
+		t.Fatalf("unexpected vmagent_hourly_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmagent3.GetIntMetric(t, "vmagent_hourly_series_limit_current_series"); v != 100 {
+		t.Fatalf("unexpected vmagent_hourly_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmagent3.GetIntMetric(t, "vmagent_hourly_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vmagent_hourly_series_limit_rows_dropped_total value: %d", v)
+	}
+
+	if v := vmagent3.GetIntMetric(t, "vmagent_daily_series_limit_max_series"); v != math.MaxInt32 {
+		t.Fatalf("unexpected vmagent_daily_series_limit_max_series value: %d", v)
+	}
+
+	if v := vmagent3.GetIntMetric(t, "vmagent_daily_series_limit_current_series"); v != 100 {
+		t.Fatalf("unexpected vmagent_daily_series_limit_current_series value: %d", v)
+	}
+
+	if v := vmagent3.GetIntMetric(t, "vmagent_daily_series_limit_rows_dropped_total"); v != 0 {
+		t.Fatalf("unexpected vmagent_daily_series_limit_rows_dropped_total value: %d", v)
+	}
+}

apptest/vminsert.go

@@ -57,7 +57,7 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		extractREs = append(extractREs, regexp.MustCompile(logRecord))
 	}
 
-	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert", flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert-race", flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":                              "127.0.0.1:0",
 			"-clusternativeListenAddr":                     "127.0.0.1:0",
@@ -237,8 +237,22 @@ func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str
 	data := []byte(strings.Join(records, "\n"))
 	var recordsCount int
 	var metadataRecords int
+	uniqueMetadataMetricNames := make(map[string]struct{})
 	for _, record := range records {
-		if strings.HasPrefix(record, "#") {
+		// metric metadata has the following format:
+		//# HELP importprometheus_series
+		//# TYPE importprometheus_series
+		// it results into single metadata record
+		if strings.HasPrefix(record, "# ") {
+			metadataItems := strings.Split(record, " ")
+			if len(metadataItems) < 2 {
+				t.Fatalf("BUG: unexpected metadata format=%q", record)
+			}
+			metricName := metadataItems[2]
+			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
+				continue
+			}
+			uniqueMetadataMetricNames[metricName] = struct{}{}
 			metadataRecords++
 			continue
 		}

apptest/vmselect.go

@@ -25,7 +25,7 @@ type Vmselect struct {
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)
 func StartVmselect(instance string, flags []string, cli *Client, output io.Writer) (*Vmselect, error) {
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect", flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect-race", flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-clusternativeListenAddr": "127.0.0.1:0",

codecov.yml

@@ -1,9 +0,0 @@
-# see https://docs.codecov.com/docs/common-recipe-list#set-non-blocking-status-checks
-coverage:
-  status:
-    project:
-      default:
-        informational: true
-    patch:
-      default:
-        informational: true

dashboards/alert-statistics.json

@@ -119,7 +119,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "yellow",
@@ -199,7 +200,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -208,14 +210,14 @@
       },
       "gridPos": {
         "h": 4,
-        "w": 9,
+        "w": 6,
         "x": 0,
         "y": 14
       },
       "id": 5,
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
+        "colorMode": "none",
+        "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "auto",
         "percentChangeColorMode": "standard",
@@ -257,7 +259,7 @@
         "type": "prometheus",
         "uid": "$ds"
       },
-      "description": "",
+      "description": "Shows the total number of loaded alerting rules across selected instances and groups.",
       "fieldConfig": {
         "defaults": {
           "mappings": [],
@@ -266,7 +268,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -275,11 +278,11 @@
       },
       "gridPos": {
         "h": 4,
-        "w": 7,
-        "x": 9,
+        "w": 6,
+        "x": 6,
         "y": 14
       },
-      "id": 4,
+      "id": 8,
       "options": {
         "colorMode": "value",
         "graphMode": "area",
@@ -320,6 +323,144 @@
       "title": "Alerting rules",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds"
+      },
+      "description": "Shows the total number of pendings alerts in selected instances and grouping groups.",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "yellow",
+                "value": 0
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 14
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "text": {
+          "valueSize": 80
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "12.0.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(vmalert_alerts_pending{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Alerting pending",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds"
+      },
+      "description": "Shows the total number of firing alerts in selected instances and grouping groups.",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 14
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "text": {
+          "valueSize": 80
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "12.0.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Alerting firing",
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -332,6 +473,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -339,7 +483,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -352,7 +497,7 @@
           {
             "matcher": {
               "id": "byName",
-              "options": "Count (sum)"
+              "options": "Count"
             },
             "properties": [
               {
@@ -372,20 +517,12 @@
       "id": 2,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "frameIndex": 1,
         "showHeader": true,
         "sortBy": [
           {
             "desc": true,
-            "displayName": "Count (sum)"
+            "displayName": "Count"
           }
         ]
       },
@@ -398,7 +535,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk_max(100, sum(increases_over_time(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by(group, alertname) > 0)",
+          "expr": "topk_max(100, sum(increases_over_time(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by(group) > 0)",
           "format": "table",
           "instant": true,
           "key": "Q-3934f0fb-8ad6-4519-a98d-c26d0fc6b312-0",
@@ -414,8 +551,9 @@
           "options": {
             "excludeByName": {
               "Time": true,
-              "alertname": false
+              "alertname": true
             },
+            "includeByName": {},
             "indexByName": {
               "Time": 0,
               "Value": 3,
@@ -428,23 +566,6 @@
               "group": "Group"
             }
           }
-        },
-        {
-          "id": "groupBy",
-          "options": {
-            "fields": {
-              "Count": {
-                "aggregations": [
-                  "sum"
-                ],
-                "operation": "aggregate"
-              },
-              "Group": {
-                "aggregations": [],
-                "operation": "groupby"
-              }
-            }
-          }
         }
       ],
       "type": "table"
@@ -468,7 +589,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -531,16 +653,14 @@
       "id": 1,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "frameIndex": 1,
-        "showHeader": true
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Count"
+          }
+        ]
       },
       "pluginVersion": "12.0.2",
       "targets": [

dashboards/vm/operator.json

@@ -115,7 +115,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 4,
+        "h": 3,
         "w": 4,
         "x": 0,
         "y": 1

dashboards/vm/vmalert.json

@@ -185,7 +185,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -250,7 +250,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -313,7 +313,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -380,7 +380,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -460,7 +460,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -557,7 +557,7 @@
         },
         "showHeader": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -663,7 +663,7 @@
           "sort": "asc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -783,7 +783,7 @@
           "sort": "desc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -889,7 +889,7 @@
           "sort": "desc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -993,7 +993,7 @@
           "sort": "desc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -1097,7 +1097,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -1222,7 +1222,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1337,7 +1337,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1447,7 +1447,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1562,7 +1562,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1695,7 +1695,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1800,7 +1800,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1921,7 +1921,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2043,7 +2043,7 @@
               "sort": "none"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2151,7 +2151,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2260,7 +2260,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2368,7 +2368,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2472,7 +2472,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2592,7 +2592,7 @@
               }
             ]
           },
-          "pluginVersion": "10.4.2",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2722,7 +2722,7 @@
               "sort": "none"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2827,7 +2827,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.1.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2942,7 +2942,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3044,7 +3044,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3146,7 +3146,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3247,7 +3247,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3346,7 +3346,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3462,7 +3462,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3564,7 +3564,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "8.0.3",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3663,7 +3663,7 @@
               "sort": "none"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3707,11 +3707,13 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
+                "barWidthFactor": 0.6,
                 "drawStyle": "line",
                 "fillOpacity": 0,
                 "gradientMode": "none",
@@ -3720,6 +3722,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3727,6 +3730,7 @@
                   "type": "linear"
                 },
                 "showPoints": "auto",
+                "showValues": false,
                 "spanNulls": false,
                 "stacking": {
                   "group": "A",
@@ -3741,7 +3745,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": 0
                   },
                   {
                     "color": "red",
@@ -3756,7 +3761,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 351
+            "y": 30
           },
           "id": 52,
           "options": {
@@ -3767,10 +3772,12 @@
               "showLegend": true
             },
             "tooltip": {
+              "hideZeros": false,
               "mode": "single",
               "sort": "desc"
             }
           },
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3778,7 +3785,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "sum(rate(vmalert_remotewrite_sent_rows_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
+              "expr": "sum(rate(vmalert_remotewrite_sent_rows_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) default sum(rate(vmalert_remotewrite_sent_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
               "legendFormat": "__auto",
               "range": true,
               "refId": "A"
@@ -3792,18 +3799,20 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows the number of datapoints dropped by vmalert while sending to the configured remote write URL. vmalert performs up to 5 retries before dropping the data. Check vmalert's error logs for the specific error message.",
+          "description": "Shows the global rate for number of written bytes via remote write connections.",
           "fieldConfig": {
             "defaults": {
               "color": {
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
+                "barWidthFactor": 0.6,
                 "drawStyle": "line",
                 "fillOpacity": 0,
                 "gradientMode": "none",
@@ -3812,6 +3821,115 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "decbytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 30
+          },
+          "id": 60,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": true,
+              "expr": "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Bytes write rate ($instance)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Data could be dropped in two scenarios:\n1. The remote write queue(configured by -remoteWrite.maxQueueSize) is full:  \nThe queue can fill rapidly if heavy rules generate millions of series, or if remote write requests are unable to send data to the destination in a timely manner, resulting in data being jammed in the queue. Consider tuning -remoteWrite.maxQueueSize or -remoteWrite.concurrency.\nSee also Rows per request pannel.\n2. The request to the configured remote write URL failed(vmalert performs up to 5 retries before dropping the data).\n\nCheck vmalert's error logs for the specific error message.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3819,6 +3937,7 @@
                   "type": "linear"
                 },
                 "showPoints": "auto",
+                "showValues": false,
                 "spanNulls": false,
                 "stacking": {
                   "group": "A",
@@ -3833,7 +3952,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": 0
                   },
                   {
                     "color": "red",
@@ -3847,8 +3967,8 @@
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 351
+            "x": 0,
+            "y": 38
           },
           "id": 53,
           "options": {
@@ -3859,10 +3979,12 @@
               "showLegend": true
             },
             "tooltip": {
+              "hideZeros": false,
               "mode": "single",
               "sort": "desc"
             }
           },
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3879,6 +4001,237 @@
           "title": "Datapoints drop rate ($instance)",
           "type": "timeseries"
         },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Displays the maximum 99th percentile of the number of time series pending in the remote write queue.\n\nThe maximum queue size is configured by remoteWrite.maxQueueSize. \nvmalert will begin dropping data if the queue has no room for newly generated data.\nThe queue can fill rapidly when heavy rules generate millions of series, or when remote write requests are unable to send data to the destination in a timely manner, causing data to accumulate in the queue. Consider tuning -remoteWrite.maxQueueSize or -remoteWrite.concurrency.\n\nSee also the Rows per request panel.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "max"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 38
+          },
+          "id": 68,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(increase(vmalert_remotewrite_queue_size_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))) > 1",
+              "legendFormat": "current",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "min(vmalert_remotewrite_queue_capacity{job=~\"$job\", instance=~\"$instance\"})",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "max",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Remote write queue size ($instance)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Displays the maximum 99th percentile of the number of data samples sent per request to the configured remote write URL.\nThe value is influenced by the utilization of the remote write queue. It is normal for this metric to remain low when there are no rules generating a large number of series, and to spike when heavy rules generate thousands of series.\nDuring periods of high load (when many heavy rules are generating results), the optimal value for rows per request should be high to maximize the efficiency of each push operation.\n\nIf you observe datapoint drops and consistently low values for rows per request, try checking the write latency between vmalert and the remote write destination, or increasing `-remoteWrite.flushInterval`.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 46
+          },
+          "id": 69,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "single",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(increase(vmalert_remotewrite_sent_rows_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))",
+              "legendFormat": "max",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Rows per request ($instance)",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "victoriametrics-metrics-datasource",
@@ -3897,6 +4250,7 @@
                 "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
+                "barWidthFactor": 0.6,
                 "drawStyle": "line",
                 "fillOpacity": 0,
                 "gradientMode": "none",
@@ -3913,6 +4267,7 @@
                   "type": "linear"
                 },
                 "showPoints": "never",
+                "showValues": false,
                 "spanNulls": false,
                 "stacking": {
                   "group": "A",
@@ -3929,7 +4284,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": 0
                   },
                   {
                     "color": "red",
@@ -3944,8 +4300,8 @@
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 0,
-            "y": 378
+            "x": 12,
+            "y": 46
           },
           "id": 54,
           "options": {
@@ -3960,10 +4316,12 @@
               "showLegend": true
             },
             "tooltip": {
+              "hideZeros": false,
               "mode": "multi",
               "sort": "desc"
             }
           },
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3981,109 +4339,6 @@
           ],
           "title": "Connections ($instance)",
           "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "victoriametrics-metrics-datasource",
-            "uid": "$ds"
-          },
-          "description": "Shows the global rate for number of written bytes via remote write connections.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "decbytes"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 12,
-            "y": 378
-          },
-          "id": 60,
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "victoriametrics-metrics-datasource",
-                "uid": "$ds"
-              },
-              "editorMode": "code",
-              "exemplar": true,
-              "expr": "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0",
-              "interval": "",
-              "legendFormat": "__auto",
-              "range": true,
-              "refId": "A"
-            }
-          ],
-          "title": "Bytes write rate ($instance)",
-          "type": "timeseries"
         }
       ],
       "title": "Remote write",

dashboards/vmalert.json

@@ -184,7 +184,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -249,7 +249,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -312,7 +312,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -379,7 +379,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -459,7 +459,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -556,7 +556,7 @@
         },
         "showHeader": true
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -662,7 +662,7 @@
           "sort": "asc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -782,7 +782,7 @@
           "sort": "desc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -888,7 +888,7 @@
           "sort": "desc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -992,7 +992,7 @@
           "sort": "desc"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -1096,7 +1096,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.0.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -1221,7 +1221,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1336,7 +1336,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1446,7 +1446,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1561,7 +1561,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1694,7 +1694,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1799,7 +1799,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -1920,7 +1920,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2042,7 +2042,7 @@
               "sort": "none"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2150,7 +2150,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2259,7 +2259,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2367,7 +2367,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2471,7 +2471,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "11.5.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2591,7 +2591,7 @@
               }
             ]
           },
-          "pluginVersion": "10.4.2",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2721,7 +2721,7 @@
               "sort": "none"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2826,7 +2826,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.1.0",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -2941,7 +2941,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3043,7 +3043,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3145,7 +3145,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3246,7 +3246,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3345,7 +3345,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3461,7 +3461,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3563,7 +3563,7 @@
               "sort": "desc"
             }
           },
-          "pluginVersion": "8.0.3",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3662,7 +3662,7 @@
               "sort": "none"
             }
           },
-          "pluginVersion": "9.2.6",
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3706,11 +3706,13 @@
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
+                "barWidthFactor": 0.6,
                 "drawStyle": "line",
                 "fillOpacity": 0,
                 "gradientMode": "none",
@@ -3719,6 +3721,7 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3726,6 +3729,7 @@
                   "type": "linear"
                 },
                 "showPoints": "auto",
+                "showValues": false,
                 "spanNulls": false,
                 "stacking": {
                   "group": "A",
@@ -3740,7 +3744,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": 0
                   },
                   {
                     "color": "red",
@@ -3755,7 +3760,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 351
+            "y": 30
           },
           "id": 52,
           "options": {
@@ -3766,10 +3771,12 @@
               "showLegend": true
             },
             "tooltip": {
+              "hideZeros": false,
               "mode": "single",
               "sort": "desc"
             }
           },
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3777,7 +3784,7 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "sum(rate(vmalert_remotewrite_sent_rows_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
+              "expr": "sum(rate(vmalert_remotewrite_sent_rows_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) default sum(rate(vmalert_remotewrite_sent_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
               "legendFormat": "__auto",
               "range": true,
               "refId": "A"
@@ -3791,18 +3798,20 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows the number of datapoints dropped by vmalert while sending to the configured remote write URL. vmalert performs up to 5 retries before dropping the data. Check vmalert's error logs for the specific error message.",
+          "description": "Shows the global rate for number of written bytes via remote write connections.",
           "fieldConfig": {
             "defaults": {
               "color": {
                 "mode": "palette-classic"
               },
               "custom": {
+                "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
+                "barWidthFactor": 0.6,
                 "drawStyle": "line",
                 "fillOpacity": 0,
                 "gradientMode": "none",
@@ -3811,6 +3820,115 @@
                   "tooltip": false,
                   "viz": false
                 },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "decbytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 30
+          },
+          "id": 60,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": true,
+              "expr": "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Bytes write rate ($instance)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Data could be dropped in two scenarios:\n1. The remote write queue(configured by -remoteWrite.maxQueueSize) is full:  \nThe queue can fill rapidly if heavy rules generate millions of series, or if remote write requests are unable to send data to the destination in a timely manner, resulting in data being jammed in the queue. Consider tuning -remoteWrite.maxQueueSize or -remoteWrite.concurrency.\nSee also Rows per request pannel.\n2. The request to the configured remote write URL failed(vmalert performs up to 5 retries before dropping the data).\n\nCheck vmalert's error logs for the specific error message.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
                 "lineInterpolation": "linear",
                 "lineWidth": 1,
                 "pointSize": 5,
@@ -3818,6 +3936,7 @@
                   "type": "linear"
                 },
                 "showPoints": "auto",
+                "showValues": false,
                 "spanNulls": false,
                 "stacking": {
                   "group": "A",
@@ -3832,7 +3951,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": 0
                   },
                   {
                     "color": "red",
@@ -3846,8 +3966,8 @@
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 351
+            "x": 0,
+            "y": 38
           },
           "id": 53,
           "options": {
@@ -3858,10 +3978,12 @@
               "showLegend": true
             },
             "tooltip": {
+              "hideZeros": false,
               "mode": "single",
               "sort": "desc"
             }
           },
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3878,6 +4000,237 @@
           "title": "Datapoints drop rate ($instance)",
           "type": "timeseries"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Displays the maximum 99th percentile of the number of time series pending in the remote write queue.\n\nThe maximum queue size is configured by remoteWrite.maxQueueSize. \nvmalert will begin dropping data if the queue has no room for newly generated data.\nThe queue can fill rapidly when heavy rules generate millions of series, or when remote write requests are unable to send data to the destination in a timely manner, causing data to accumulate in the queue. Consider tuning -remoteWrite.maxQueueSize or -remoteWrite.concurrency.\n\nSee also the Rows per request panel.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "max"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 38
+          },
+          "id": 68,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(increase(vmalert_remotewrite_queue_size_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))) > 1",
+              "legendFormat": "current",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "min(vmalert_remotewrite_queue_capacity{job=~\"$job\", instance=~\"$instance\"})",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "max",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Remote write queue size ($instance)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Displays the maximum 99th percentile of the number of data samples sent per request to the configured remote write URL.\nThe value is influenced by the utilization of the remote write queue. It is normal for this metric to remain low when there are no rules generating a large number of series, and to spike when heavy rules generate thousands of series.\nDuring periods of high load (when many heavy rules are generating results), the optimal value for rows per request should be high to maximize the efficiency of each push operation.\n\nIf you observe datapoint drops and consistently low values for rows per request, try checking the write latency between vmalert and the remote write destination, or increasing `-remoteWrite.flushInterval`.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 46
+          },
+          "id": 69,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "single",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(increase(vmalert_remotewrite_sent_rows_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))",
+              "legendFormat": "max",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Rows per request ($instance)",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "prometheus",
@@ -3896,6 +4249,7 @@
                 "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
+                "barWidthFactor": 0.6,
                 "drawStyle": "line",
                 "fillOpacity": 0,
                 "gradientMode": "none",
@@ -3912,6 +4266,7 @@
                   "type": "linear"
                 },
                 "showPoints": "never",
+                "showValues": false,
                 "spanNulls": false,
                 "stacking": {
                   "group": "A",
@@ -3928,7 +4283,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": 0
                   },
                   {
                     "color": "red",
@@ -3943,8 +4299,8 @@
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 0,
-            "y": 378
+            "x": 12,
+            "y": 46
           },
           "id": 54,
           "options": {
@@ -3959,10 +4315,12 @@
               "showLegend": true
             },
             "tooltip": {
+              "hideZeros": false,
               "mode": "multi",
               "sort": "desc"
             }
           },
+          "pluginVersion": "12.2.0",
           "targets": [
             {
               "datasource": {
@@ -3980,109 +4338,6 @@
           ],
           "title": "Connections ($instance)",
           "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds"
-          },
-          "description": "Shows the global rate for number of written bytes via remote write connections.",
-          "fieldConfig": {
-            "defaults": {
-              "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "line",
-                "fillOpacity": 0,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 1,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
-              },
-              "links": [],
-              "mappings": [],
-              "min": 0,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green"
-                  },
-                  {
-                    "color": "red",
-                    "value": 80
-                  }
-                ]
-              },
-              "unit": "decbytes"
-            },
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 12,
-            "y": 378
-          },
-          "id": 60,
-          "options": {
-            "legend": {
-              "calcs": [
-                "mean",
-                "lastNotNull",
-                "max"
-              ],
-              "displayMode": "table",
-              "placement": "bottom",
-              "showLegend": true
-            },
-            "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
-            }
-          },
-          "targets": [
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$ds"
-              },
-              "editorMode": "code",
-              "exemplar": true,
-              "expr": "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0",
-              "interval": "",
-              "legendFormat": "__auto",
-              "range": true,
-              "refId": "A"
-            }
-          ],
-          "title": "Bytes write rate ($instance)",
-          "type": "timeseries"
         }
       ],
       "title": "Remote write",

deployment/docker/Makefile

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.23.3
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.23.3
 
-GO_BUILDER_IMAGE := golang:1.26.1
+GO_BUILDER_IMAGE := golang:1.26.2
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.139.0
+    image: victoriametrics/vmauth:v1.140.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts-vmalert.yml

@@ -85,6 +85,20 @@ groups:
             to the configured remote write URL. This may result into gaps in recording rules or alerts state.
             Check vmalert's logs for detailed error message."
 
+      - alert: RemoteWriteQueueHighUsage
+        expr: histogram_quantile(0.99, sum(increase(vmalert_remotewrite_queue_size_bucket[5m])) by (job, instance, vmrange)) / vmalert_remotewrite_queue_capacity > 0.8
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Remote write queue capacity on the vmalert instance {{ $labels.instance }} has exceeded 80% utilization"
+          description: "The remote write queue on vmalert instance {{ $labels.instance }} has consistently high utilization.
+            The queue acts as a buffer between rules generating series and remote-write client consuming and pushing these series. When queue overflows, vmalert will start dropping newly generated series.
+            Queue may overflow due to multiple reasons:
+             1. Some bad rules produce too many series at once. This can be limited using the global `-rule.resultsLimit` flag or `limit` param at the rule group level.
+             2. Remote write connection is slow.  Increase `-remoteWrite.concurrency`,  so vmalert could establish more concurrent connections.
+             3. The queue size is too small. Increase `-remoteWrite.maxQueueSize` to extend the buffer size. Note that a larger queue will result in higher memory consumption when the queue is full."
+
       - alert: AlertmanagerErrors
         expr: increase(vmalert_alerts_send_errors_total[5m]) > 0
         for: 15m
@@ -94,3 +108,4 @@ groups:
           summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
           description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
             Check vmalert's logs for detailed error message."
+

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - "victoriametrics"
     ports:

docs/Makefile

@@ -41,18 +41,8 @@ docs-debug: docs docs-image
 		$(foreach dir,$(wildcard ./docs/$(dir)/*), -v ./docs/$(notdir $(dir)):/opt/docs/content/$(notdir $(dir))) \
 			vmdocs-docker-package
 
-docs-update-version: docs-image
-	$(if $(filter v%,$(PKG_TAG)), \
-		docker run \
-			--rm \
-			--entrypoint /usr/bin/find \
-			--platform $(DOCKER_PLATFORM) \
-			--name vmdocs-docker-container \
-			-v ./docs:/opt/docs/content/victoriametrics vmdocs-docker-package \
-				content \
-					-regex ".*\.md" \
-					-exec sed -i 's/{{% available_from "#" %}}/{{% available_from "$(PKG_TAG)" %}}/g' {} \;, \
-		$(info "Skipping docs version update, invalid $$PKG_TAG: $(PKG_TAG)"))
+docs-update-version:
+	find docs/victoriametrics/ -name '*.md' -exec sed -i 's/{{% available_from "#" %}}/{{% available_from "$(TAG)" %}}/g' {} \;
 
 # Converts images at docs folder to webp format
 # See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#images-in-documentation
@@ -342,4 +332,4 @@ endif
 	$(MAKE) docs-update-vmagent-flags && git checkout "$$orig_branch" && \
 	$(MAKE) docs-update-vmselect-flags && git checkout "$$orig_branch" && \
 	$(MAKE) docs-update-vminsert-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmstorage-flags && git checkout "$$orig_branch"
+	$(MAKE) docs-update-vmstorage-flags && git checkout "$$orig_branch"

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -238,23 +238,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -306,7 +306,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.139.0
+    image: docker.io/victoriametrics/vmauth:v1.140.0
     ports:
       - 8427:8427
     volumes:
@@ -420,7 +420,7 @@ Create two Prometheus datasources in Grafana with the following URLs: `http://vm
 ![Prometheus datasource](grafana-datasource-prometheus.webp)
 
 You can also use the VictoriaMetrics [Grafana datasource](https://github.com/VictoriaMetrics/victoriametrics-datasource) plugin.
-See installation instructions in [Grafana datasource - Installation](https://docs.victoriametrics.com/victoriametrics/victoriametrics-datasource/#installation).
+See installation instructions in [Grafana datasource - Installation](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/#victoriametrics-datasource).
 
 Users with the `vm_access` claim will be able to query metrics from the specified tenant with extra filters applied.
 

docs/guides/understand-your-setup-size/README.md

@@ -125,7 +125,7 @@ As a reference, see resource consumption of VictoriaMetrics cluster on our [play
 The Retention Period is the number of days or months for storing data. It affects the disk space usage.
 The formula for calculating required disk space is the following:
 ```
-Bytes Per Sample * Ingestion rate * Replication Factor * (Retention Period in Seconds +1 Retention Cycle(day or month)) * 1.2 (recommended 20% of free space for merges ) 
+Bytes Per Sample * Ingestion rate * Replication Factor * (Retention Period in Seconds +1 Retention Cycle(day or month)) * 1.25 (recommended 20% of free space for merges ) 
 ```
 
 The **Retention Cycle** is one **day** or one **month**. If the retention period is higher than 30 days cycle is a month; otherwise day.
@@ -143,7 +143,7 @@ Keep at least **20%** of free space for VictoriaMetrics to remain efficient with
 
 A Kubernetes environment that produces 5k time series per second with 1-year of the Retention Period and ReplicationFactor=2:
 
-`(1 byte-per-sample * 5000 time series * 2 replication factor * 34128000 seconds) * 1.2 ) / 2^30 = 381 GB`
+`(1 byte-per-sample * 5000 time series * 2 replication factor * 34128000 seconds) * 1.25 ) / 2^30 = 397 GB`
 
 VictoriaMetrics requires additional disk space for the index. The lower Churn Rate, the lower is disk space usage for the index.
 Usually, index takes about **20%** of the disk space for storing data. High cardinality setups may use **>50%** of storage size for index. If your indexdb looks unexpectedly large, see [FAQ: Why indexdb size is so large?](https://docs.victoriametrics.com/victoriametrics/faq/#why-indexdb-size-is-so-large) for typical ratios and troubleshooting tips.

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.139.0-enterprise
+    image: victoriametrics/vmauth:v1.140.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - victoriametrics
       - alertmanager
@@ -198,7 +198,7 @@ If you open the sidebar and select **Alerting** > **Alert rules**, you should be
 
 Open the sidebar again and go to **Alerting** > **Active notifications** to see the active alert reported by Alertmanager.
 
-![Screenshot of Grafana Active notifications Page](grafana-active-notifications.webp)
+![Screenshot of Grafana Active notifications Page](grafana-notifications.webp)
 
 You can also see the alerts in VMUI by opening the browser in `http://localhost:8428/vmui/?#/rules`. This is possible only when we have configured `-vmalert.proxyURL` in VictoriaMetrics.
 

docs/playgrounds/README.md

@@ -1,19 +1,132 @@
-VictoriaMetrics provides a rich set of public playgrounds that let you explore the full observability stack — metrics, logs, and traces — and even use migration tools without installing or configuring anything locally. 
+VictoriaMetrics offers public playgrounds where you can try the full observability stack online.
 
-These playgrounds are backed by real VictoriaMetrics components and data, making them ideal for:
+Some playgrounds are based on the [OpenTelemetry Astronomy Shop demo](https://github.com/open-telemetry/opentelemetry-demo), a sample microservices application that generates realistic metrics, logs, and traces. Other playgrounds use benchmark workloads such as [prometheus-benchmark](https://github.com/VictoriaMetrics/prometheus-benchmark) to demonstrate ingestion and query performance for Prometheus-compatible systems.
 
-- Learning VictoriaMetrics query languages
-- Trying dashboards and queries interactively
-- Validating migration paths
+These playgrounds are ideal for:
+
+- Learning [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/)
+- Trying out dashboards and queries interactively
 - Demonstrating features in talks or workshops
 
-In this section, we'll walk through each available playground, explain what it does, and link to the relevant GitHub repositories.
+In the following sections, we'll walk through each playground, explain its purpose, and link to the corresponding GitHub repositories.
 
-## Docker Compose Demo
+## VictoriaMetrics Playground
+
+A fast, cost-effective, and scalable monitoring solution and time series database designed for collecting, storing, querying, and alerting on metrics.
+
+- Try it: <https://play.victoriametrics.com/>
+- Documentation: <https://docs.victoriametrics.com/victoriametrics/>
+
+The playground is represented by [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
+It collects metrics from [OpenTelemetry Astronomy Shop demo](https://github.com/open-telemetry/opentelemetry-demo) and
+various services in our Kubernetes playground namespace.
+
+VictoriaMetrics web UI is represented via [VMUI](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui).
+It allows querying metrics, plotting graphs, exploring cardinality, viewing alerting and recording rules, debugging relabeling rules, etc.
+
+The best place to get started is with the [Cardinality Explorer](https://play.victoriametrics.com/select/0/prometheus/graph/#/cardinality) page, found under **Explore** > **Explore Cardinality**.
+This view shows what's stored in the database for the specified day, lets you browse top metrics and labels in the dataset, and lets you drill down into them without writing a single line of MetricsQL.
+
+![Screenshot of VMUI](vmui-cardinality-explorer.webp)
+<figcaption style="text-align: center; font-style: italic;">Cardinality Explorer in VictoriaMetrics</figcaption>
+
+Try plotting graphs on the **Query** tab - [`sum(rate(vm_http_requests_total[5m])) by(path) > 0`](https://play.victoriametrics.com/select/0/prometheus/graph/#/?g0.range_input=30m&g0.relative_time=last_30_minutes&g0.tab=0&g0.expr=sum%28rate%28vm_http_requests_total%5B5m%5D%29%29+by%28path%29+%3E+0):
+
+![Screenshot of VMUI](vmui-rate-http-requests-total.webp)
+
+> VictoriaMetrics is also represented as a datasource in [Grafana playground](#grafana-playground).
+
+## VictoriaLogs Playground
+
+High-performance, lightweight, zero-config, schema-free database for logs that is easy to use and scales both vertically and horizontally, 
+from very small setups to large-scale deployments handling terabytes per day.
+
+- Try it: <https://play-vmlogs.victoriametrics.com/>
+- Documentation: <https://docs.victoriametrics.com/victorialogs/>
+
+The playground is represented by [cluster version of VictoriaLogs](https://docs.victoriametrics.com/victorialogs/cluster/).
+It stores logs from [OpenTelemetry Astronomy Shop demo](https://github.com/open-telemetry/opentelemetry-demo) and various
+other services that run in our k8s playground namespace.
+
+VictoriaLogs [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui) allows querying log entries, plotting graphs,
+exploring stored datasets, live tailing.
+
+To start exploring the data, look at the sidebar. It shows the stream fields available in the dataset, 
+and clicking any field or value automatically applies a filter, so you can browse the logs before writing your own query.
+
+![Screenshot of VictoriaLogs VMUI](vl-vmui.webp)
+
+On the `Query` tab, click on the `Query examples` button to see examples of the most common search queries to start with.
+
+> VictoriaLogs is also represented as a datasource in [Grafana playground](#grafana-playground).
+
+## VictoriaTraces Playground
+
+VictoriaTraces is a fast and scalable database for traces, built on top of VictoriaLogs.
+
+- Try it: <https://play-grafana.victoriametrics.com/explore> (choose `Jaeger` datasource)
+- Documentation: <https://docs.victoriametrics.com/victoriatraces/>
+
+> [!NOTE]
+> VictoriaTraces doesn't have its own web UI. Instead, it implements Jaeger API for [integrating with Jaeger UI or Grafana](https://docs.victoriametrics.com/victoriatraces/querying/).
+
+VictoriaTraces playground stores traces from [OpenTelemetry Astronomy Shop demo](https://github.com/open-telemetry/opentelemetry-demo).
+
+To view trace data, follow these steps:
+1. On the [Grafana Playground](https://play-grafana.victoriametrics.com/), select **Explore** in the sidebar
+2. Select VictoriaTraces / Jaeger in the combo box near the top-left corner
+3. In **Query Type** select "Search"
+4. Select one of the services and press **Run Query**
+
+![Screenshot of Grafana](vt-grafana.webp)
+
+> VictoriaTraces is also represented as a datasource in [Grafana playground](#grafana-playground).
+
+## Grafana Playground
+
+- Try it: <https://play-grafana.victoriametrics.com>
+
+Grafana playground contains examples of dashboards and datasources for VictoriaMetrics, VictoriaLogs, and VictoriaTraces.
+It allows viewing metrics, logs, and traces generated by [OpenTelemetry Astronomy Shop demo](https://github.com/open-telemetry/opentelemetry-demo).
+
+Try viewing available dashboards or just browse via Grafana's explore page to query and [correlate signals](https://docs.victoriametrics.com/opentelemetry/#correlations).
+
+![Screenshot of Grafana Dashboard](grafana-node-exporter.webp)
+
+## VMAnomaly Playground
+
+VMAnomaly analyzes metrics, logs, or traces using VictoriaMetrics' built-in anomaly detection model to generate an [anomaly score](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score). An `anomaly_score > 1` indicates an anomalous condition that deserves attention.
+
+The [anomaly metrics playground](https://play-vmanomaly.victoriametrics.com/metrics/) shows anomalies in CPU utilization.
+
+![Screenshot of VMUI](vmanomaly-metrics.webp)
+<figcaption style="text-align: center; font-style: italic;">Exploring anomalies on metric data on CPU utilization</figcaption>
+
+The [anomaly logs playground](https://play-vmanomaly.victoriametrics.com/logs/) shows by default anomalies in log ingestion.
+
+![Screenshot of VMUI](vmanomaly-logs.webp)
+<figcaption style="text-align: center; font-style: italic;">Finding anomalies in log ingestion</figcaption>
+
+And the [anomaly trace playground](https://play-vmanomaly.victoriametrics.com/traces/) analyzes spans with error status.
+
+![Screenshot of VMUI](vmanomaly-traces.webp)
+<figcaption style="text-align: center; font-style: italic;">Analyzing traces for service error anomalies</figcaption>
+
+## Docker Compose Playgrounds
+
+We provide Docker Compose examples for:
 
-We provide Docker Compose files for:
 - [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/README.md)
 - [VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/blob/master/deployment/docker/README.md)
-- [VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/deployment/docker/README.md). 
+- [VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/deployment/docker/README.md) 
+
+The Docker Compose examples demonstrate how various components could be configured, provisioned, and interconnected.
+These examples aren't intended for production use.
+
+## VictoriaMetrics Cloud
+
+VictoriaMetrics UIs are also included in the [Explore](https://docs.victoriametrics.com/victoriametrics-cloud/exploring-data/) section of VictoriaMetrics and VictoriaLogs deployments, embedded in VictoriaMetrics Cloud.
+
+You can experiment with your own data during the month‑long trial without deploying VictoriaStack in your infrastructure. To get started, follow [this guide](https://docs.victoriametrics.com/victoriametrics-cloud/get-started/quickstart/).
+
 
-The compose files are already configured, provisioned, and interconnected. They can be used to quickly set up a demo environment, suitable for a [quick start](https://docs.victoriametrics.com/victoriametrics/quick-start/).

docs/playgrounds/_index.md

@@ -10,5 +10,11 @@ tags:
   - logs
   - traces
   - playground
+aliases:
+  - /playgrounds/victoriametrics/
+  - /playgrounds/victorialogs/
+  - /playgrounds/victoriatraces/
+  - /playgrounds/cloud/
+  - /playgrounds/vmanomaly/
 ---
 {{% content "README.md" %}}

docs/playgrounds/cloud.md

@@ -1,21 +0,0 @@
----
-weight: 6 
-title: Cloud Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 6
-tags:
-- victoriametrics
-- cloud
-- playground
-- monitoring
----
-
-- Try it: <https://console.victoriametrics.cloud/explore>
-
-VictoriaMetrics UIs are included in the explore section of VictoriaMetrics and VictoriaLogs deployments embedded in VictoriaMetrics Cloud.
-
-You can experiment with your own data without the need to deploy the VictoriaMetrics Stack in your local environment for free for a month by following this guide: [VictoriaMetrics Cloud Quickstart](https://docs.victoriametrics.com/victoriametrics-cloud/get-started/quickstart/).
-
-Once set up, follow this guide to explore your data: <https://docs.victoriametrics.com/victoriametrics-cloud/exploring-data/>

docs/playgrounds/grafana.md

@@ -1,45 +0,0 @@
----
-weight: 4 
-title: Grafana Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 4
-tags:
-- grafana
-- playground
-- metrics
-- logs
-- traces
----
-          
-- Try it: <https://play-grafana.victoriametrics.com/>
-
-This playground is particularly useful if you already use Grafana and want to see how VictoriaMetrics integrates into existing workflows. It provides a hosted Grafana instance preconfigured with:
-
-- [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) as a metrics data source
-- [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) as a logs data source
-- [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) as a Jaeger data source for traces
-
-## What can you do here?
-
-- Explore [real dashboards](https://play-grafana.victoriametrics.com/dashboards) built on top of VictoriaMetrics
-- See how [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) are used in Grafana panels
-- Explore correlation with the help of the [OpenTelemetry Collector dashboard](https://play-grafana.victoriametrics.com/d/BKf2sowmj/opentelemetry-collector)
-- Learn dashboard design and visualization best practices
-
-![Screenshot of VMUI](grafana.webp)
-<figcaption style="text-align: center; font-style: italic;">Grafana dashboard in the playground</figcaption>
-
-The OpenTelemetry Collector dashboard is built on the official [OpenTelemetry Astronomy Shop demo](https://github.com/VictoriaMetrics-Community/opentelemetry-demo). It lets you visualize and understand telemetry data alongside VictoriaMetrics Stack observability signals, using VictoriaMetrics for metrics, VictoriaLogs for logs, and VictoriaTraces for traces. 
-
-For an always-updated list of dashboards, bookmark this playground.
-
-## Distribution
-
-Relevant GitHub:
-- VictoriaMetrics Grafana datasource: <https://github.com/VictoriaMetrics/victoriametrics-datasource>
-- VictoriaLogs Grafana datasource: <https://github.com/VictoriaMetrics/victorialogs-datasource>
-
-
-

docs/playgrounds/log-to-logsql.md

@@ -1,42 +0,0 @@
----
-weight: 8 
-title: LogQL to LogsQL Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 8
-tags:
-- logsql
-- loki
-- victorialogs
-- playground
-- monitoring
----
-
-- Try it: <https://play-logql.victoriametrics.com/>
-
-For teams migrating from Grafana Loki, this simple UI provides a useful [translator from LogQL to LogsQL](https://docs.victoriametrics.com/victorialogs/logql-to-logsql/).
-
-![Screenshot of LogQL to LogsQL](logql-to-logsql.webp)
-
-## What can you do here?
-
-The query-language translation tool automatically converts Loki queries into VictoriaLogs queries, reducing friction when adopting VictoriaLogs in environments already using Loki.
-
-Type your LogQL query and press **Execute**.
-
-For example, this LogQL query:
-
-```text
-{collector="otel-collector"} |= "POST"
-```
-
-Translates into the equivalent [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/)
-
-```text
-{collector="otel-collector"} "POST"
-```
-
-## Distribution
-
-- GitHub: <https://github.com/VictoriaMetrics-Community/logql-to-logsql>

docs/playgrounds/sql-to-logsql.md

@@ -1,46 +0,0 @@
----
-weight: 7 
-title: SQL to LogsQL Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 7
-tags:
-- logsql
-- SQL
-- victorialogs
-- playground
-- monitoring
----
-
-- Try it: <https://play-sql.victoriametrics.com/>
-
-This playground enables you to query data from a VictoriaLogs instance or translate SQL to [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) without querying.
-
-![Screenshot of SQL to LogsQL](sql-to-logsql.webp)
-
-## What can you do here?
-
-First, run `SHOW TABLES;` to view all the existing tables in the SQL database and their equivalent query in [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/).
-
-Then, type your SQL query and press **Execute**. 
-
-For example, this query:
-
-```sql
-SELECT _time, _msg
-FROM logs
-WHERE _msg LIKE 'error'
-ORDER BY _time DESC
-LIMIT 100
-```
-
-Translates into:
-
-```text
-_msg:error | fields _time, _msg | sort by (_time desc) | limit 100
-```
-
-## Distribution
-
-- GitHub: <https://github.com/VictoriaMetrics/sql-to-logsql>

docs/playgrounds/victorialogs.md

@@ -1,44 +0,0 @@
----
-weight: 2 
-title: VictoriaLogs Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 2
-tags:
-- victorialogs
-- playground
-- logs
----
-
-- Try it: <https://play-vmlogs.victoriametrics.com/>
-- Query language reference: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/)
-
-This playground focuses on VictoriaLogs and lets you test the query engine on a demo log set. The playground demonstrates how VictoriaLogs handles high-volume log data with predictable performance and low operational overhead.
-
-![Screenshot of VMUI](vl-main-vmui.webp)
-<figcaption style="text-align: center; font-style: italic;">VictoriaLogs playground</figcaption>
-
-## What can you do here?
-
-The WebUI provides the following modes for displaying query results:
-- Group: results are displayed as a table with rows grouped by stream fields.
-- Table: displays query results as a table.
-- JSON: displays raw JSON response from `/select/logsql/query` HTTP API.
-- Live: displays live tailing results for the given query.
-
-As a starting point, you can type `collector: "otel-collector"` in the query field to search for entries collected by OpenTelemetry.
-
-![Screenshot of VMUI](vl-otel-collector.webp)
-<figcaption style="text-align: center; font-style: italic;">Log entries collected with the OpenTelemetry collector</figcaption>
-
-Typing `error AND _time:24h` shows you the entries containing the text "error" during the last 24 hours.
-![Screenshot of VMUI](vl-error.webp)
-
-The **Overview** provides a quick, high-level look at the logs stored in VictoriaLogs. It helps you understand the volume and structure of your log data before diving into detailed queries. You can see log ingestion trends, identify the most common fields and values, and quickly spot noisy or unusual streams. From here, you can click on fields or values to automatically apply filters and start exploring your data with LogsQL.
-
-![Screenshot of VMUI](vl-overview.webp)
-
-## Distribution
-
-- GitHub: <https://github.com/VictoriaMetrics/VictoriaLogs>

docs/playgrounds/victoriametrics.md

@@ -1,63 +0,0 @@
----
-weight: 1 
-title: VictoriaMetrics Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 1
-tags:
-- victoriametrics
----
-
-- Try it: <https://play.victoriametrics.com/>
-- Query language reference: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/)
-
-This is the primary playground for VictoriaMetrics, powered by VMUI and backed by a VictoriaMetrics cluster installation. Use it to experiment with the query engine, see available pages, or try tools such as the relabeling debugger. 
-
-This playground is the best starting point for understanding how VictoriaMetrics stores and queries metrics at scale.
-
-![Screenshot of VMUI](vm-main-vmui.webp)
-<figcaption style="text-align: center; font-style: italic;">VictoriaMetrics playground</figcaption>
-
-## What can you do here?
-
-The query tab provides a sandbox to experiment with [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/). Turn on Autocomplete and start typing to discover time series. You can add multiple queries and compare them.
-
-You can try these to get started:
-
-- Average CPU usage per job: `sum(rate(process_cpu_seconds_total[5m])) by (job)`
-- HTTP requests per-second rate: `sum(rate(vm_http_requests_total[5m]))`
-- Top 5 CPU intensive jobs `topk(5, sum(rate(process_cpu_seconds_total[5m])) by (job))`
-
-Below is an example of average CPU usage per job:
-
-```text
-sum(rate(process_cpu_seconds_total[5m])) by (job)
-```
-
-![Screenshot of VMUI](vm-process-cpu.webp)
-<figcaption style="text-align: center; font-style: italic;">Average CPU usage per job</figcaption>
-
-Here, we are requesting the per-second rate of HTTP requests:
-
-```text
-sum(rate(vm_http_requests_total[5m]))
-```
-
-![Screenshot of VMUI](vm-http-requests.webp)
-<figcaption style="text-align: center; font-style: italic;">HTTP requests per second</figcaption>
-
-And here is an example for obtaining the top 5 high CPU jobs:
-
-```text
-topk(5, sum(rate(process_cpu_seconds_total[5m])) by (job))  
-```
-
-![Screenshot of VMUI](vm-topk-process-cpu.webp)
-<figcaption style="text-align: center; font-style: italic;">Top 5 CPU intensive jobs</figcaption>
-
-For a deep dive into all the features of this playground, please visit the [VMUI](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) page.
-
-## Distribution
-
-- GitHub: <https://github.com/VictoriaMetrics/VictoriaMetrics>

docs/playgrounds/victoriatraces.md

@@ -1,36 +0,0 @@
----
-weight: 3 
-title: VictoriaTraces Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 3
-tags:
-- victoriatraces
-- playground
-- monitoring
----
-
-- Try it: <https://play-vtraces.victoriametrics.com/>
-- Query language reference: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/)
-
-> [!NOTE] Note
-> This playground is currently under development, as the main project it is correlated with, VictoriaTraces, is also under development.
-
-VictoriaTraces provides a UI for browsing raw data and Jaeger APIs/Grafana data source for trace visualization. This playground showcases VictoriaTraces, the VictoriaMetrics backend for distributed tracing, and enables trace searching, visualization, and service graph/dependency analysis.
-
-![Screenshot of Grafana](vt-grafana.webp)
-<figcaption style="text-align: center; font-style: italic;">Grafana playground showing VictoriaTraces/Jaeger datasource</figcaption>
-
-## What can you do here?
-
-The WebUI provides the following modes for displaying query results:
-- Group: results are displayed as a table with rows grouped by stream fields.
-- Table: displays query results as a table.
-- JSON: displays raw JSON response from the HTTP API.
-- Live: displays live tailing results for the given query.
-
-## Distribution
-
-- GitHub: <https://github.com/VictoriaMetrics/VictoriaTraces>
-

docs/playgrounds/vmanomaly.md

@@ -1,44 +0,0 @@
----
-weight: 5 
-title: Anomaly Detection Playground
-menu:
-  docs:
-    parent: "playgrounds"
-    weight: 5
-tags:
-- vmanomaly
-- playground
-- monitoring
----
-
-- Try it: <https://play-vmanomaly.victoriametrics.com/metrics/vmui/>
-- UI Guide: <https://docs.victoriametrics.com/anomaly-detection/ui/#example-usage>
-
-The playground demonstrates automatic [anomaly detection](https://docs.victoriametrics.com/anomaly-detection/).
-
-![Screenshot of VMUI](vmanomaly.webp)
-<figcaption style="text-align: center; font-style: italic;">Exploring model fit for CPU usage series</figcaption>
-
-The playground showcases anomaly detection data (native timeseries or converted to timeseries) using VictoriaMetrics, VictoriaLogs, or VictoriaTraces datasources, respectively:
-
-- <https://play-vmanomaly.victoriametrics.com/metrics/>
-- <https://play-vmanomaly.victoriametrics.com/logs/>
-- <https://play-vmanomaly.victoriametrics.com/traces/>
-
-## What can you do here?
-
-The Anomaly Detection playground lets you:
-- Understand how [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) are used to generate input data for anomaly detection.
-- Explore metrics data enriched with anomaly scores, predictions, and confidence intervals.
-- Visualize anomalies directly in VMUI, including consecutive anomalies that last over time rather than being a single point, to imitate how alerting rules trigger on such data.
-- Learn how anomaly scores can be used for alerting purposes by exploring generated alerting rules.
-
-## Distribution & setup
-
-VMAnomaly is distributed through various channels:
-
-- [Installation guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/)
-- Docker containers available in [Docker Hub](https://hub.docker.com/r/victoriametrics/vmanomaly) and [Quay.io](https://quay.io/repository/victoriametrics/vmanomaly)
-- [Helm charts](https://github.com/VictoriaMetrics/helm-charts) (including anomaly setups)
-- [VM Operator](https://docs.victoriametrics.com/operator/resources/vmanomaly/)
-

docs/victoriametrics/Articles.md

@@ -23,6 +23,7 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [CERN: The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf)
 * [Forbes: The (Almost) Infinitely Scalable Open Source Monitoring Dream](https://www.forbes.com/sites/adrianbridgwater/2022/08/16/the-almost-infinitely-scalable-open-source-monitoring-dream/)
 * [Forbes: The Agility In Cloud Observability](https://www.forbes.com/sites/adrianbridgwater/2023/07/05/the-agility-in-cloud-observability/)
+* [Airbnb: Building a high-volume metrics pipeline with OpenTelemetry and vmagent](https://medium.com/airbnb-engineering/building-a-high-volume-metrics-pipeline-with-opentelemetry-and-vmagent-c714d6910b45)
 * [Bedrock: Monitoring at scale with Victoria Metrics](https://tech.bedrockstreaming.com/2022/09/06/monitoring-at-scale-with-victoriametrics.html)
 * [TiDB by PingCap: Scaling Observability: Why TiDB Moved from Prometheus to VictoriaMetrics](https://www.pingcap.com/blog/tidb-observability-migrating-prometheus-victoriametrics/)
 * [TigrisData: We do our billing with Prometheus](https://www.tigrisdata.com/blog/billing-prometheus/)
@@ -112,6 +113,7 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [FreeBSD: monitoring with VictoriaMetrics and Grafana](https://setevoy.medium.com/freebsd-monitoring-with-victoriametrics-and-grafana-f789904f2628)
 * [QCon London 2026: Wrangling Telemetry at Scale, a Guide to Self-Hosted Observability](https://www.infoq.com/news/2026/03/self-hosted-observability/)
 * [How We Made Telemetry Queries 10x Faster: Chunk-Split Caching for Metrics, Logs, and Traces](https://mirastacklabs.ai/blog/chunk-split-caching/)
+* [Building a high-volume metrics pipeline with OpenTelemetry and vmagent](https://medium.com/airbnb-engineering/building-a-high-volume-metrics-pipeline-with-opentelemetry-and-vmagent-c714d6910b45)
 
 ## Third-party articles and slides about VictoriaLogs
 

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -556,6 +556,9 @@ See more details on [how to monitor VictoriaMetrics components](https://docs.vic
 - `-storage.maxHourlySeries` is the limit on the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) during the last hour.
 - `-storage.maxDailySeries` is the limit on the number of unique time series during the day. This limit can be used for limiting daily [time series churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate).
 
+It is possible to use `-1` as a value for these flags{{% available_from "v1.140.0" %}} in order to enable series tracking but set limit to maximum possible value.
+This is useful in order to estimate the number of unique series written to `vmstorage` without enforcing limits.
+
 Note that these limits are set and applied individually per each `vmstorage` node in the cluster. So, if the cluster has `N` `vmstorage` nodes, then the cluster-level limits will be `N` times bigger than the per-`vmstorage` limits.
 
 See more details about cardinality limiter in [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter).

docs/victoriametrics/LTS-releases.md

@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
 
 ## Currently supported LTS release lines
 
-- v1.136.x - the latest one is [v1.136.3 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.3)
-- v1.122.x - the latest one is [v1.122.18 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.18)
+- v1.136.x - the latest one is [v1.136.4 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.4)
+- v1.122.x - the latest one is [v1.122.19 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.19)

docs/victoriametrics/Quick-Start.md

@@ -58,9 +58,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.139.0
+docker pull victoriametrics/victoria-metrics:v1.140.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.139.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.140.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

docs/victoriametrics/README.md

@@ -795,6 +795,7 @@ curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -
 ```
 
 The exported CSV data can be imported to VictoriaMetrics via [/api/v1/import/csv](#how-to-import-csv-data).
+The first line of the file is a header row derived from the `format` parameter.
 
 The [deduplication](#deduplication) is applied for the data exported in CSV by default. It is possible to export raw data without de-duplication by passing `reduce_mem_usage=1` query arg to `/api/v1/export/csv`.
 
@@ -936,6 +937,8 @@ The `format` query arg must contain comma-separated list of parsing rules for CS
     * `rfc3339` - timestamp in [RFC3339](https://tools.ietf.org/html/rfc3339) format, i.e. `2006-01-02T15:04:05Z`.
     * `custom:<layout>` - custom layout for the timestamp. The `<layout>` may contain arbitrary time layout according to [time.Parse rules in Go](https://golang.org/pkg/time/#Parse).
 
+The first row is treated as a header but can be skipped if any `time` or `metric` column contains a non-numeric value.
+
 Each request to `/api/v1/import/csv` may contain arbitrary number of CSV lines.
 
 Example for importing CSV data via `/api/v1/import/csv`:
@@ -1976,6 +1979,9 @@ By default, VictoriaMetrics doesn't limit the number of stored time series. The
 
 Both limits can be set simultaneously. If any of these limits is reached, then incoming samples for new time series are dropped. A sample of dropped series is put in the log with `WARNING` level.
 
+It is possible to use `-1` as a value for these flags{{% available_from "v1.140.0" %}} in order to enable series tracking but set limit to maximum possible value.
+This is useful in order to estimate the number of unique series which is written to VictoriaMetrics single without enforcing limits.
+
 The exceeded limits can be [monitored](#monitoring) with the following metrics:
 
 * `vm_hourly_series_limit_rows_dropped_total` - the number of metrics dropped due to exceeded hourly limit on the number of unique time series.

docs/victoriametrics/Release-Guide.md

@@ -113,7 +113,7 @@ and the candidate is deployed to the sandbox environment.
 
 1. Make sure that the release branches have no security issues.
 1. Update release versions if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/SECURITY.md).
-1. Run `PKG_TAG=v1.xx.y make docs-update-version` command to update version help tooltips. 
+1. Run `TAG=v1.xx.y make docs-update-version` command to update version help tooltips.
 1. Cut new version in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md) and commit it. See example in this [commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b771152039d23b5ccd637a23ea748bc44a9511a7).
 1. Create the following release tags:
    * `git tag -s v1.xx.y` in `master` branch

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,11 +26,37 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
+
+Released at 2026-04-10
+
+**Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).
+
+* SECURITY: upgrade Go builder from Go1.26.1 to Go1.26.2. See [the list of issues addressed in Go1.26.2](https://github.com/golang/go/issues?q=milestone%3AGo1.26.2%20label%3ACherryPickApproved).
+
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add per-URL `-remoteWrite.disableMetadata` flag to disable metadata sending for specific remote storage URLs. See [#10711](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10711). Thanks to @evkuzin for the contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `profile` option to `ec2_sd_configs` for loading credentials from named AWS profiles in `~/.aws/credentials` and `~/.aws/config`, including `source_profile` chaining and `role_arn` resolution. See [ec2_sd_configs docs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#ec2_sd_configs). Issue [#1685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1685). Thanks to @andriibeee for the contribution.
 * FEATURE: introduce `vm_filestream_fsync_duration_seconds_total` and `vm_filestream_fsync_calls_total` metrics, which can be used for detecting slow storage if it cannot keep up with the current data ingestion rate. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432). Thanks to @mehrdadbn9 for the contribution.
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add dedicated `thanos` mode for [migrating data from Thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/thanos/). This mode supports both raw and downsampled Thanos blocks, including all aggregate types (count, sum, min, max, counter). Each aggregate is imported as a separate metric with resolution and aggregate type suffixes (e.g., `metric_name:5m:count`). The new mode uses `--thanos-*` prefixed flags: `--thanos-snapshot`, `--thanos-concurrency`, `--thanos-filter-time-start`, `--thanos-filter-time-end`, `--thanos-filter-label`, `--thanos-filter-label-value`, and `--thanos-aggr-types`. See [#9262](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9262).
+* FEATURE: [dashboards/alert-statistics](https://grafana.com/grafana/dashboards/24553): add pending and firing alerts stats; fix query in `FIRING over time by group` panel. See [#10571](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10571). Thanks to @sias32 for the contribution.
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add random jitter to concurrent periodical flushers targeting the remote write destination. This helps spread remote write flushes across the flush interval, avoiding congestion at the remote write destination and enhancing queue data consumption. See [#10729](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10729).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_remotewrite_sent_rows` and `vmalert_remotewrite_sent_bytes` histograms to provide better visibility into remote write request sizes. See [#10727](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10727).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow setting `-1` value for `-remoteWrite.maxHourlySeries` and `-remoteWrite.maxDailySeries` command-line flags. This automatically sets limits to the highest possible value in order to enable tracking without enforcing any limits. This is helpful for estimating current usage before applying real limits. See [#9614](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9614).
+* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): optimize vminsert buffer size per vmstorage node based on available CPU, memory and storage node count to reduce OOM risk. See [#10725](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10725).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): allow setting `-1` value for `-storage.maxHourlySeries` and `-storage.maxDailySeries` command-line flags. This automatically sets limits to the highest possible value in order to enable tracking without enforcing any limits. This is helpful for estimating current usage before applying real limits. See [#9614](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9614).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_remotewrite_queue_size` and `vmalert_remotewrite_queue_capacity` to facilitate monitoring of remote write queue usage. See [#10765](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10765).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): CSV export on the `Raw Query` tab now includes all labels from the executed query. VMUI no longer prepends a header row, as it is now provided by the backend. See [#10667](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10667) and [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666). Thanks to @lawrence3699 for the contribution.
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add header row to `/api/v1/export/csv` output and auto-detect header rows during import via `/api/v1/import/csv`. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666). Thanks to @andriibeee for the contribution.
+* FEATURE: all VictoriaMetrics components: expose operating system name and release version as metric `vm_os_info`. See [#10481](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10481).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): align group evaluation time with the `eval_offset` option to help manage group execution more effectively. See [#10772](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10772).
 
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry the requests that failed with unexpected EOF due to unstable network to S3 service. See [#10699](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10699).
+* BUGFIX: All VictoriaMetrics components: Fix an issue where `unsupported` metric metadata type was exposed for summaries and quantiles if a summary wasn't updated within a certain time window. See [metrics#120](https://github.com/VictoriaMetrics/metrics/issues/120) and [metrics#121](https://github.com/VictoriaMetrics/metrics/pull/121).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `scrape_series_added` metric to update only on successful scrapes, aligning its behavior with Prometheus. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent partial responses from second-level vmselect nodes in [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). Ensures response completeness and correctness, and avoids cache pollution in top-level vmselect. See [#10678](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Fix storage connection saturation spikes at 00:00 UTC and improve data ingestion when the storage is restarted during the first hour of the day. See [10698](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10698).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent rare panic during storage start-up at 100% disk usage. See [#10747](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10747) Thanks to @nmn3m for the contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent data ingestion from getting completely stuck when storage is under heavy load. See [#10784](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10784). Thanks to @fxrlv for the contribution.
 
 ## [v1.139.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.139.0)
 
@@ -54,7 +80,7 @@ Released at 2026-03-27
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic during directory deletion on `NFS`-based mounts. The bug was introduced in [83da33d8](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/83da33d8cfe8352fd0022d05a8b6346ebb48420d) and included in [v1.123.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11230). See [#9842](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9842).
 
-## [v1.139.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.139.0)
+## [v1.138.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.138.0)
 
 Released at 2026-03-13
 
@@ -109,6 +135,23 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.4](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.4)
+
+Released at 2026-04-10
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* SECURITY: upgrade Go builder from Go1.26.1 to Go1.26.2. See [the list of issues addressed in Go1.26.2](https://github.com/golang/go/issues?q=milestone%3AGo1.26.2%20label%3ACherryPickApproved).
+
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry the requests that failed with unexpected EOF due to unstable network to S3 service. See [#10699](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10699).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `scrape_series_added` metric to update only on successful scrapes, aligning its behavior with Prometheus. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent partial responses from second-level vmselect nodes in [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). Ensures response completeness and correctness, and avoids cache pollution in top-level vmselect. See [#10678](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Fix storage connection saturation spikes at 00:00 UTC and improve data ingestion when the storage is restarted during the first hour of the day. See [10698](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10698).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent rare panic during storage start-up at 100% disk usage. See [#10747](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10747) Thanks to @nmn3m for the contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent data ingestion from getting completely stuck when storage is under heavy load. See [#10784](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10784). Thanks to @fxrlv for the contribution.
+
 ## [v1.136.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.3)
 
 Released at 2026-03-27
@@ -133,7 +176,7 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
 
 * SECURITY: upgrade Go builder from Go1.26.0 to Go1.26.1. See [the list of issues addressed in Go1.26.1](https://github.com/golang/go/issues?q=milestone%3AGo1.26.1%20label%3ACherryPickApproved).
 
-FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add [histogram_fraction](https://docs.victoriametrics.com/victoriametrics/metricsql/#histogram_fraction) function to calculate the fraction of buckets falling between lowerLe and upperLe. See [#5346](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5346).
 
 * BUGFIX: all VictoriaMetrics components: replace `histogram` with `untyped` metric metadata type for [VictoriaMetrics histograms](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) when `-metrics.exposeMetadata` is set. See [#82](https://github.com/VictoriaMetrics/metrics/issues/82).
@@ -312,6 +355,21 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.19](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.19)
+
+Released at 2026-04-10
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* SECURITY: upgrade Go builder from Go1.25.8 to Go1.25.9. See [the list of issues addressed in Go1.25.9](https://github.com/golang/go/issues?q=milestone%3AGo1.25.9+label%3ACherryPickApproved).
+
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry the requests that failed with unexpected EOF due to unstable network to S3 service. See [#10699](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10699).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `scrape_series_added` metric to update only on successful scrapes, aligning its behavior with Prometheus. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent partial responses from second-level vmselect nodes in [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). Ensures response completeness and correctness, and avoids cache pollution in top-level vmselect. See [#10678](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678)
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent rare panic during storage start-up at 100% disk usage. See [#10747](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10747) Thanks to @nmn3m for the contribution.
+
 ## [v1.122.18](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.18)
 
 Released at 2026-03-27

docs/victoriametrics/enterprise.md

@@ -117,7 +117,7 @@ It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [
 
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
 and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -135,8 +135,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -151,7 +151,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.139.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.140.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](#binary-releases) section.
@@ -161,13 +161,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.139.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.140.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.139.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.140.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -177,7 +177,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     ports:
       - 8428:8428
     volumes:
@@ -209,7 +209,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -220,7 +220,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 
 license:
   secret:
@@ -270,7 +270,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -287,7 +287,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 ```
 
 Example secret with license key:
@@ -338,7 +338,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz`
 
 Includes:
 
@@ -347,7 +347,7 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.139.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.140.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## Monitoring license expiration
 

docs/victoriametrics/relabeling.md

@@ -24,6 +24,11 @@ VictoriaMetrics and vmagent support Prometheus-style relabeling with
 The following articles contain useful information about Prometheus relabeling:
 
 - [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
+- [Life of a label](https://www.robustperception.io/life-of-a-label)
+- [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
+- [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
+- [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
+- [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
 
 ## Relabeling Stages
 
@@ -352,9 +357,6 @@ see two types of targets:
   service discovery, before any relabeling rules are applied. This includes
   targets that may later be dropped.
 
-_This option is only available when the component is started with the
-`-promscrape.dropOriginalLabels=false` flag._
-
 {{% collapse name="How to use `/targets` page?" %}}
 
 This `/targets` page helps answer the following questions:
@@ -374,18 +376,12 @@ to all metrics scraped from that target.
 You can click the label column of the target to see the original labels
 **before** any relabeling was applied.
 
-_This option is only available when the component is started with the
-`-promscrape.dropOriginalLabels=false` flag._
-
 **3. Why does a target have a certain set of labels?**
 
 Click the `target` link in the `debug relabeling` column. This opens a
 step-by-step view of how the relabeling rules were applied to the original
 labels.
 
-_This option is only available when the component is started with the
-`-promscrape.dropOriginalLabels=false` flag._
-
 **4. How are metric relabeling rules applied to scraped metrics?**
 
 Click the `metrics` link in the `debug relabeling` column. This shows how the
@@ -408,9 +404,6 @@ Each column on the page shows important details:
 This page shows all
 [discovered targets](https://docs.victoriametrics.com/victoriametrics/sd_configs/).
 
-_This option is only available when the component is started with the
-`-promscrape.dropOriginalLabels=false` flag._
-
 It helps answer the following questions:
 
 **1. Why are some targets dropped during service discovery or showing unexpected

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/victoria-metrics-linux-amd64-v1.139.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.139.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.140.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/victoria-metrics-linux-amd64-v1.139.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.139.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.140.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/sd_configs.md

@@ -648,6 +648,12 @@ scrape_configs:
     #
     # role_arn: "..."
 
+    # profile is an optional named AWS profile from ~/.aws/config and ~/.aws/credentials.
+    # When set, credentials and role_arn are resolved from the profile, with source_profile
+    # chaining supported. See https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html
+    #
+    # profile: "..."
+
     # port is an optional port to scrape metrics from.
     # By default, port 80 is used.
     #

docs/victoriametrics/stream-aggregation/README.md

@@ -285,7 +285,7 @@ See also [quantiles over input metrics](#quantiles-over-input-metrics) and [aggr
 [Histogram](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) is a set of [counter](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter)
 metrics with different `vmrange` or `le` labels. Since typical usage of histograms is to calculate quantiles over the
 buckets change via [histogram_quantile](https://docs.victoriametrics.com/victoriametrics/metricsql/#histogram_quantile) function the
-appropriate aggregation output for this is [total](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_sum):
+appropriate aggregation output for this is [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_sum):
 
 ```yaml
 - match: 'http_request_duration_seconds_bucket'

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -548,9 +548,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
   -storage.idbPrefillStart duration
      Specifies how early VictoriaMetrics starts pre-filling indexDB records before indexDB rotation. Starting the pre-fill process earlier can help reduce resource usage spikes during rotation. In most cases, this value should not be changed. The maximum allowed value is 23h. (default 1h0m0s)
   -storage.maxDailySeries int
-     The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . See also -storage.maxHourlySeries
+     The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See also -storage.maxHourlySeries
   -storage.maxHourlySeries int
-     The maximum number of unique series can be added to the storage during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . See also -storage.maxDailySeries
+     The maximum number of unique series can be added to the storage during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See also -storage.maxDailySeries
   -storage.maxMetadataStorageSize size
      Overrides max size for metrics metadata entries in-memory storage. If set to 0 or a negative value, defaults to 1% of allowed memory.
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)

docs/victoriametrics/vmagent.md

@@ -146,6 +146,12 @@ and then it sends the buffered data to the remote storage in order to prevent da
 so there is no need to specify multiple `-remoteWrite.url` flags when writing data to the same cluster.
 See [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#replication-and-data-safety).
 
+### Relabeling and filtering
+
+`vmagent` can add, remove or update labels on the collected data before sending it to the remote storage.
+It can filter scrape targets or remove unwanted samples via Prometheus-like relabeling.
+Please see [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for details.
+
 ### Sharding among remote storages
 
 By default `vmagent` replicates data to remote storage systems via the `-remoteWrite.url` command-line flag.
@@ -188,12 +194,6 @@ except for the labels `instance` and `pod` must be routed to the same backend. I
 
 See also [how to scrape large number of targets](#scraping-big-number-of-targets).
 
-### Relabeling and filtering
-
-`vmagent` can add, remove or update labels on the collected data before sending it to the remote storage. Additionally,
-it can remove unwanted samples via Prometheus-like relabeling before sending the collected data to remote storage.
-Please see [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for details.
-
 ### Splitting data streams among multiple systems
 
 `vmagent` supports splitting the collected data between multiple destinations with the help of `-remoteWrite.urlRelabelConfig`,
@@ -613,6 +613,8 @@ and attaches `instance`, `job` and other target-specific labels to these metrics
   `vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line flag
   or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled.
 
+  Restarting `vmagent` can cause `scrape_series_added` to rise because all time series are new to a newly started `vmagent`.
+
 * `scrape_series_limit` - the limit on the number of unique [series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) the given target can expose according to [these docs](#cardinality-limiter).
   This metric is exposed only if the series limit is set.
 
@@ -939,6 +941,9 @@ The limit can be enforced by setting the following command-line flags:
 * `-remoteWrite.maxDailySeries` - limits the number of unique time series `vmagent` can write to remote storage systems during the last day.
   Useful for limiting daily churn rate.
 
+It is possible to use `-1` as a value for these flags{{% available_from "v1.140.0" %}} in order to enable series tracking but set limit to maximum possible value.
+This is useful in order to estimate the number of unique series which is written to remote storage systems without enforcing limits.
+
 Both limits can be set simultaneously. If any of these limits is reached, then samples for new time series are dropped instead of sending
 them to remote storage systems. A sample of dropped series is put in the log with `WARNING` level.
 

docs/victoriametrics/vmagent_common_flags.md

@@ -389,6 +389,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Optional path to bearer token file to use for the corresponding -remoteWrite.url. The token is re-read from the file every second
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
+  -remoteWrite.disableMetadata array
+     Whether to disable sending metadata to the corresponding -remoteWrite.url. By default, metadata sending is controlled by the global -enableMetadata flag
+     Supports array of values separated by comma or specified via multiple flags.
+     Empty values are set to false.
   -remoteWrite.disableOnDiskQueue array
      Whether to disable storing pending data to -remoteWrite.tmpDataPath when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload
      Supports array of values separated by comma or specified via multiple flags.
@@ -419,14 +423,14 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 8388608)
   -remoteWrite.maxDailySeries int
-     The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
+     The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
   -remoteWrite.maxDiskUsagePerURL array
      The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. Disk usage is unlimited if the value is set to 0
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB. (default 0)
      Supports array of values separated by comma or specified via multiple flags.
      Empty values are set to default value.
   -remoteWrite.maxHourlySeries int
-     The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
+     The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
   -remoteWrite.maxMetadataPerBlock int
      The maximum number of metadata to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 5000)
   -remoteWrite.maxRowsPerBlock int

docs/victoriametrics/vmalert.md

@@ -1236,8 +1236,8 @@ For example:
 ```yaml
 groups:
   - name: BaseGroup
-    interval: 1m
-    eval_offset: 10s
+    interval: 5m
+    eval_offset: 1m
     rules:
       - record: http_server_request_duration_seconds:sum_rate:5m:http_get
         expr: |
@@ -1258,8 +1258,8 @@ groups:
             )
           )
   - name: TopGroup
-    interval: 1m
-    eval_offset: 40s
+    interval: 5m
+    eval_offset: 3m
     rules:
       - record: http_server_request_duration_seconds:sum_rate:5m:merged
         expr: |
@@ -1271,20 +1271,20 @@ groups:
 This configuration ensures that rules in `BaseGroup` are executed at(assuming vmalert starts at `12:00:00`):
 
 ```
-[12:00:10, 12:01:10, 12:02:10, 12:03:10...]
+[12:01:00, 12:06:00, 12:11:00, 12:16:00...]
 ```
 
 while rules in group `TopGroup` are executed at:
 
 ```
-[12:00:40, 12:01:40, 12:02:40, 12:03:40...]
+[12:03:00, 12:08:00, 12:13:00, 12:18:00...]
 ```
 
-As a result, `TopGroup` always gets the latest results of `BaseGroup`.
+As a result, `TopGroup` can consistently obtain the latest results from `BaseGroup` if `BaseGroup` completes its evaluation and uploads its results to the datasource within 2 minutes.
 
 By default, the `eval_offset` values should be at least 30 seconds apart to accommodate the
 `-search.latencyOffset(default 30s)` command-line flag at vmselect or VictoriaMetrics single-node.
-The minimum `eval_offset` gap can be adjusted accordingly with `-search.latencyOffset`.
+The minimum `eval_offset` gap should be adjusted according to the sum of the execution duration of `BaseGroup` and `-search.latencyOffset`.
 
 ### Notifier configuration file
 

docs/victoriametrics/vmauth.md

@@ -1137,7 +1137,7 @@ The following [metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/
 * `vmauth_buffer_request_body_duration_seconds` - the [summary](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#summary) for the request body buffering duration.
   Use this metric to understand buffering performance and identify slow clients.
 
-See also [concurrency limits](https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limits).
+See also [concurrency limits](https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting).
 
 ## Backend TLS setup
 

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/vmutils-darwin-arm64-v1.139.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/vmutils-darwin-arm64-v1.140.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.139.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.140.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

docs/victoriametrics/vmctl/vmctl_flags.md

@@ -20,6 +20,7 @@ COMMANDS:
    influx        Migrate time series from InfluxDB
    remote-read   Migrate time series via Prometheus remote-read protocol
    prometheus    Migrate time series from Prometheus
+   thanos        Migrate time series from Thanos blocks (supports raw and downsampled data)
    vm-native     Migrate time series between VictoriaMetrics installations
    verify-block  Verifies exported block with VictoriaMetrics Native format
    help, h       Shows a list of commands or help for one command

docs/victoriametrics/vmctl/vmctl_prometheus_flags.md

@@ -29,7 +29,7 @@ OPTIONS:
    --prom-filter-time-end value                                       The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
    --prom-filter-label value                                          Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
    --prom-filter-label-value value                                    Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
-   --prom-tmp-dir-path value                                          Path to directory to be used for temporary files. (default: "/tmp")
+   --prom-tmp-dir-path value                                          Path to directory to be used for temporary files. (default: "/var/folders/9b/rj_f77q52w57vnxx236qh51m0000gn/T/")
    --vm-addr value                                                    VictoriaMetrics address to perform import requests. 
       Should be the same as --httpListenAddr value for single-node version or vminsert component. 
       When importing into the clustered version do not forget to set additionally --vm-account-id flag.