lib/promutil/labelscompressor: add rotation of labelscompressor

Fix app tests:

1. Sync code between vmsingle and vmcluster: it must be the same because
apptest does not differentiate between branches, it just runs pre-built
binaries
2. Simplify range queries in backup/restore test so that it does not
depend on the interval between samples to work correctly.

---------

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/test.yml

@@ -66,8 +66,8 @@ jobs:
     strategy:
       matrix:
         scenario:
-          - 'test-full'
-          - 'test-full-386'
+          - 'test'
+          - 'test-386'
           - 'test-pure'
 
     steps:
@@ -88,11 +88,6 @@ jobs:
       - name: Run tests
         run: make ${{ matrix.scenario}}
 
-      - name: Publish coverage
-        uses: codecov/codecov-action@v6
-        with:
-          files: ./coverage.txt
-
   apptest:
     name: apptest
     runs-on: apptest

Makefile

@@ -457,6 +457,9 @@ test:
 test-race:
 	go test -tags 'synctest' -race ./lib/... ./app/...
 
+test-386:
+	GOARCH=386 go test -tags 'synctest' ./lib/... ./app/...
+
 test-pure:
 	CGO_ENABLED=0 go test -tags 'synctest' ./lib/... ./app/...
 

README.md

@@ -4,7 +4,6 @@
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
-[![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)
 [![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)](https://x.com/VictoriaMetrics/)

app/vmalert/rule/group.go

@@ -381,7 +381,9 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 
 		if len(g.Rules) < 1 {
 			g.metrics.iterationDuration.UpdateDuration(start)
+			g.mu.Lock()
 			g.LastEvaluation = start
+			g.mu.Unlock()
 			return ts
 		}
 
@@ -395,7 +397,9 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 			}
 		}
 		g.metrics.iterationDuration.UpdateDuration(start)
+		g.mu.Lock()
 		g.LastEvaluation = start
+		g.mu.Unlock()
 		return ts
 	}
 
@@ -405,11 +409,11 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	g.mu.Unlock()
 	defer g.evalCancel()
 
-	realEvalTS := eval(evalCtx, evalTS)
-
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 
+	realEvalTS := eval(evalCtx, evalTS)
+
 	// restore the rules state after the first evaluation
 	// so only active alerts can be restored.
 	if rr != nil {

app/vmalert/web.go

@@ -52,7 +52,13 @@ var (
 		"alert":  rule.TypeAlerting,
 		"record": rule.TypeRecording,
 	}
-	ruleStates = []string{"ok", "nomatch", "inactive", "firing", "pending", "unhealthy"}
+
+	// The "recovering", "noData", "normal", "error" states are used by Grafana.
+	// Ignore "recovering" since it is not currently acknowledged by vmalert,
+	// treat "noData" as an alias for "nomatch",
+	// treat "normal" as an alias for "inactive",
+	// treat "error" as an alias for "unhealthy"
+	ruleStates = []string{"ok", "nomatch", "inactive", "firing", "pending", "unhealthy", "recovering", "noData", "normal", "error"}
 )
 
 type requestHandler struct {
@@ -363,6 +369,15 @@ func newRulesFilter(r *http.Request) (*rulesFilter, *httpserver.ErrorWithStatusC
 			if !slices.Contains(ruleStates, v) {
 				return nil, errResponse(fmt.Errorf(`invalid parameter "state": contains not supported value %q`, v), http.StatusBadRequest)
 			}
+			// Replace grafana states with supported internal states
+			switch v {
+			case "noData":
+				v = "nomatch"
+			case "normal":
+				v = "inactive"
+			case "error":
+				v = "unhealthy"
+			}
 			rf.states = append(rf.states, v)
 		}
 	}

app/vmauth/main.go

@@ -47,13 +47,12 @@ var (
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
 	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 
-	requestBufferSize = flagutil.NewBytes("requestBufferSize", defaultBodyBufferSize, "The size of the buffer for reading the request body before proxying the request to backends. "+
+	requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
 		"This allows reducing the consumption of backend resources when processing requests from clients connected via slow networks. "+
-		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering. "+
-		" Disabling request buffering also disables request retries configured with '-maxRequestBodySizeToRetry'")
-	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", defaultBodyBufferSize, "The maximum request body size in memory for potential retries at other backends. "+
-		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request retries. "+
-		"See also '-requestBufferSize' flag, which controlls a size of the buffer for potential retry.")
+		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
+	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
+		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
+		"See also -requestBufferSize")
 
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process simultaneously. "+
 		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
@@ -90,8 +89,6 @@ var (
 		"Recommended when vmauth is exposed to the internet.")
 )
 
-const defaultBodyBufferSize = 16 * 1024
-
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -352,26 +349,15 @@ func endConcurrencyLimit() {
 	<-concurrencyLimitCh
 }
 
-func getMaxRequestBufSize() int {
-	rbs := requestBufferSize.IntN()
-	rbstr := maxRequestBodySizeToRetry.IntN()
-	if rbs <= 0 {
-		// request buffering disabled
-		// it also disables request retries as a side effect
-		return 0
-	}
-	// for backward compatibility we must use max value for both flags
-	return max(rbs, rbstr)
-}
-
 func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (io.ReadCloser, error) {
 	if r == nil {
 		// This is a GET request with nil reader.
 		return nil, nil
 	}
 
-	maxBufSize := getMaxRequestBufSize()
+	maxBufSize := max(requestBufferSize.IntN(), maxRequestBodySizeToRetry.IntN())
 	if maxBufSize <= 0 {
+		// Request buffering is disabled.
 		return r, nil
 	}
 
@@ -401,7 +387,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 		}
 	}
 
-	bb := newBufferedBody(r, buf, maxBufSize, maxRequestBodySizeToRetry.IntN())
+	bb := newBufferedBody(r, buf, maxBufSize)
 	return bb, nil
 }
 
@@ -807,10 +793,11 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 }
 
 // bufferedBody serves two purposes:
-//  1. Enables request retries when the body size does not exceed maxBodySize
-//     by fully buffering the body in memory.
-//  2. Prevents slow clients from reducing effective server capacity by
-//     buffering the request body before acquiring a per-user concurrency slot.
+//
+//  1. It enables request retries when the request body size does not exceed maxBufSize
+//     by fully buffering the request body in memory.
+//  2. It prevents slow clients from reducing effective server capacity
+//     by buffering the request body before acquiring a per-user concurrency slot.
 //
 // See bufferRequestBody for details on how bufferedBody is used.
 type bufferedBody struct {
@@ -825,33 +812,29 @@ type bufferedBody struct {
 	// bufOffset is the offset at buf for already read bytes.
 	bufOffset int
 
-	// cannotRetry is set to true if buf size exceeds max retry body size
+	// cannotRetry is set to true after Close() call on non-nil r.
 	cannotRetry bool
-
-	// bodyWasClosed is set to true at Close if request retry is not possible
-	bodyWasClosed bool
 }
 
-func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize, maxBufRetrySize int) *bufferedBody {
+func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody {
 	// Do not use sync.Pool here, since http.RoundTrip may still use request body after return.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
 
 	if len(buf) < maxBufSize {
-		// Read the full request body into buf.
+		// The full request body has been already read into buf.
 		r = nil
 	}
 
 	return &bufferedBody{
-		r:           r,
-		buf:         buf,
-		cannotRetry: len(buf) > maxBufRetrySize || maxBufRetrySize == 0,
+		r:   r,
+		buf: buf,
 	}
 }
 
 // Read implements io.Reader interface.
 func (bb *bufferedBody) Read(p []byte) (int, error) {
-	if bb.bodyWasClosed {
-		return 0, fmt.Errorf("cannot read already closed body")
+	if bb.cannotRetry {
+		return 0, fmt.Errorf("cannot read already closed request body")
 	}
 	if bb.bufOffset < len(bb.buf) {
 		n := copy(p, bb.buf[bb.bufOffset:])
@@ -865,16 +848,14 @@ func (bb *bufferedBody) Read(p []byte) (int, error) {
 }
 
 func (bb *bufferedBody) canRetry() bool {
-	return !bb.cannotRetry
+	return bb.r == nil
 }
 
 // Close implements io.Closer interface.
 func (bb *bufferedBody) Close() error {
 	bb.resetReader()
-	if bb.cannotRetry {
-		bb.bodyWasClosed = true
-	}
 	if bb.r != nil {
+		bb.cannotRetry = true
 		return bb.r.Close()
 	}
 	return nil

app/vmauth/main_test.go

@@ -1714,6 +1714,13 @@ func TestBufferRequestBody_Failure(t *testing.T) {
 		}
 	}()
 
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
 	defaultMaxQueueDuration := *maxQueueDuration
 	defer func() {
 		*maxQueueDuration = defaultMaxQueueDuration
@@ -1722,6 +1729,9 @@ func TestBufferRequestBody_Failure(t *testing.T) {
 	f := func(body *mockBody, expectedResponse string) {
 		t.Helper()
 
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
 		if err := requestBufferSize.Set("2048"); err != nil {
 			t.Fatalf("cannot set requestBufferSize: %s", err)
 		}
@@ -1834,6 +1844,16 @@ func TestBufferedBody_RetrySuccess(t *testing.T) {
 			t.Fatalf("cannot set requestBufferSize: %s", err)
 		}
 
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
 		ctx := context.Background()
 		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
 		if err != nil {
@@ -1882,6 +1902,16 @@ func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 			t.Fatalf("cannot set requestBufferSize: %s", err)
 		}
 
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
 		ctx := context.Background()
 		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
 		if err != nil {
@@ -1934,6 +1964,16 @@ func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set("0"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+
 		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
 		defer func() {
 			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
@@ -2009,9 +2049,10 @@ func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		bb, ok := rb.(*bufferedBody)
-		canRetry := ok && bb.canRetry()
-		if canRetry {
-			t.Fatalf("canRetry() must return false before reading anything")
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
+			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		data, err := io.ReadAll(rb)
 		if err != nil {
@@ -2028,8 +2069,8 @@ func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error in io.ReadAll: %s", err)
 		}
-		if len(data) > 0 {
-			t.Fatalf("unexpected non-empty data read\ngot\n%s", data)
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
 	}
 

apptest/model.go

@@ -88,6 +88,7 @@ type QueryOpts struct {
 	MaxLookback    string
 	LatencyOffset  string
 	Format         string
+	NoCache        string
 }
 
 func (qos *QueryOpts) asURLValues() url.Values {
@@ -112,6 +113,7 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("max_lookback", qos.MaxLookback)
 	addNonEmpty("latency_offset", qos.LatencyOffset)
 	addNonEmpty("format", qos.Format)
+	addNonEmpty("nocache", qos.NoCache)
 
 	return uv
 }

apptest/tests/backup_restore_test.go

@@ -28,7 +28,6 @@ func TestSingleBackupRestore(t *testing.T) {
 			return tc.MustStartVmsingle("vmsingle", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
-				"-search.maxStalenessInterval=1m",
 			})
 		},
 		stopSUT: func() {
@@ -70,9 +69,7 @@ func TestClusterBackupRestore(t *testing.T) {
 				VminsertInstance: "vminsert",
 				VminsertFlags:    []string{},
 				VmselectInstance: "vmselect",
-				VmselectFlags: []string{
-					"-search.maxStalenessInterval=1m",
-				},
+				VmselectFlags:    []string{},
 			})
 		},
 		stopSUT: func() {
@@ -100,15 +97,14 @@ func TestClusterBackupRestore(t *testing.T) {
 func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	t := tc.T()
 
-	const msecPerMinute = 60 * 1000
-	genData := func(count int, prefix string, start int64) (recs []string, wantSeries []map[string]string, wantQueryResults []*apptest.QueryResult) {
+	genData := func(count int, prefix string, start, step int64) (recs []string, wantSeries []map[string]string, wantQueryResults []*apptest.QueryResult) {
 		recs = make([]string, count)
 		wantSeries = make([]map[string]string, count)
 		wantQueryResults = make([]*apptest.QueryResult, count)
 		for i := range count {
 			name := fmt.Sprintf("%s_%03d", prefix, i)
 			value := float64(i)
-			timestamp := start + int64(i)*msecPerMinute
+			timestamp := start + int64(i)*step
 
 			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
 			wantSeries[i] = map[string]string{"__name__": name}
@@ -148,15 +144,17 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 
 	// assertSeries retrieves all data from the storage and compares it with the
 	// expected result.
-	assertQueryResults := func(app apptest.PrometheusQuerier, query string, start, end int64, want []*apptest.QueryResult) {
+	assertQueryResults := func(app apptest.PrometheusQuerier, query string, start, end, step int64, want []*apptest.QueryResult) {
 		t.Helper()
 		tc.Assert(&apptest.AssertOptions{
 			Msg: "unexpected /api/v1/query_range response",
 			Got: func() any {
 				return app.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
-					Start: fmt.Sprintf("%d", start),
-					End:   fmt.Sprintf("%d", end),
-					Step:  "60s",
+					Start:       fmt.Sprintf("%d", start),
+					End:         fmt.Sprintf("%d", end),
+					Step:        fmt.Sprintf("%dms", step),
+					MaxLookback: fmt.Sprintf("%dms", step-1),
+					NoCache:     "1",
 				})
 			},
 			Want: &apptest.PrometheusAPIV1QueryResponse{
@@ -167,7 +165,6 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 				},
 			},
 			FailNow: true,
-			Retries: 300,
 		})
 	}
 
@@ -194,8 +191,9 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	// below.
 	const numMetrics = 1000
 	// With 1000 metrics (one per minute), the time range spans 2 months.
-	end := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
-	start := end - numMetrics*msecPerMinute
+	start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	step := (end - start) / numMetrics
 
 	// Verify backup/restore:
 	//
@@ -209,8 +207,8 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	// - Start vmsingle
 	// - Ensure that the queries return batch1 data only.
 
-	batch1Data, wantBatch1Series, wantBatch1QueryResults := genData(numMetrics, "batch1", start)
-	batch2Data, wantBatch2Series, wantBatch2QueryResults := genData(numMetrics, "batch2", start)
+	batch1Data, wantBatch1Series, wantBatch1QueryResults := genData(numMetrics, "batch1", start, step)
+	batch2Data, wantBatch2Series, wantBatch2QueryResults := genData(numMetrics, "batch2", start, step)
 	wantBatch12Series := slices.Concat(wantBatch1Series, wantBatch2Series)
 	wantBatch12QueryResults := slices.Concat(wantBatch1QueryResults, wantBatch2QueryResults)
 
@@ -219,13 +217,14 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	sut.PrometheusAPIV1ImportPrometheus(t, batch1Data, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
-	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, step, wantBatch1QueryResults)
+
 	createBackup(sut, "batch1")
 
 	sut.PrometheusAPIV1ImportPrometheus(t, batch2Data, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 	assertSeries(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12Series)
-	assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12QueryResults)
+	assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, step, wantBatch12QueryResults)
 	createBackup(sut, "batch12")
 
 	opts.stopSUT()
@@ -235,5 +234,5 @@ func testBackupRestore(tc *apptest.TestCase, opts testBackupRestoreOpts) {
 	sut = opts.startSUT()
 
 	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
-	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, step, wantBatch1QueryResults)
 }

apptest/vminsert.go

@@ -57,7 +57,7 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		extractREs = append(extractREs, regexp.MustCompile(logRecord))
 	}
 
-	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert", flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert-race", flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":                              "127.0.0.1:0",
 			"-clusternativeListenAddr":                     "127.0.0.1:0",
@@ -237,8 +237,22 @@ func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str
 	data := []byte(strings.Join(records, "\n"))
 	var recordsCount int
 	var metadataRecords int
+	uniqueMetadataMetricNames := make(map[string]struct{})
 	for _, record := range records {
-		if strings.HasPrefix(record, "#") {
+		// metric metadata has the following format:
+		//# HELP importprometheus_series
+		//# TYPE importprometheus_series
+		// it results into single metadata record
+		if strings.HasPrefix(record, "# ") {
+			metadataItems := strings.Split(record, " ")
+			if len(metadataItems) < 2 {
+				t.Fatalf("BUG: unexpected metadata format=%q", record)
+			}
+			metricName := metadataItems[2]
+			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
+				continue
+			}
+			uniqueMetadataMetricNames[metricName] = struct{}{}
 			metadataRecords++
 			continue
 		}

apptest/vmselect.go

@@ -25,7 +25,7 @@ type Vmselect struct {
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)
 func StartVmselect(instance string, flags []string, cli *Client, output io.Writer) (*Vmselect, error) {
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect", flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect-race", flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-clusternativeListenAddr": "127.0.0.1:0",

codecov.yml

@@ -1,9 +0,0 @@
-# see https://docs.codecov.com/docs/common-recipe-list#set-non-blocking-status-checks
-coverage:
-  status:
-    project:
-      default:
-        informational: true
-    patch:
-      default:
-        informational: true

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.139.0
+    image: victoriametrics/vmauth:v1.140.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -238,23 +238,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -306,7 +306,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.139.0
+    image: docker.io/victoriametrics/vmauth:v1.140.0
     ports:
       - 8427:8427
     volumes:

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.139.0-cluster
+    image: victoriametrics/vmstorage:v1.140.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.139.0-cluster
+    image: victoriametrics/vminsert:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.139.0-cluster
+    image: victoriametrics/vmselect:v1.140.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.139.0-enterprise
+    image: victoriametrics/vmauth:v1.140.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.139.0
+    image: victoriametrics/vmagent:v1.140.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.139.0
+    image: victoriametrics/vmalert:v1.140.0
     depends_on:
       - victoriametrics
       - alertmanager

docs/victoriametrics/Articles.md

@@ -23,6 +23,7 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [CERN: The CMS monitoring infrastructure and applications](https://arxiv.org/pdf/2007.03630.pdf)
 * [Forbes: The (Almost) Infinitely Scalable Open Source Monitoring Dream](https://www.forbes.com/sites/adrianbridgwater/2022/08/16/the-almost-infinitely-scalable-open-source-monitoring-dream/)
 * [Forbes: The Agility In Cloud Observability](https://www.forbes.com/sites/adrianbridgwater/2023/07/05/the-agility-in-cloud-observability/)
+* [Airbnb: Building a high-volume metrics pipeline with OpenTelemetry and vmagent](https://medium.com/airbnb-engineering/building-a-high-volume-metrics-pipeline-with-opentelemetry-and-vmagent-c714d6910b45)
 * [Bedrock: Monitoring at scale with Victoria Metrics](https://tech.bedrockstreaming.com/2022/09/06/monitoring-at-scale-with-victoriametrics.html)
 * [TiDB by PingCap: Scaling Observability: Why TiDB Moved from Prometheus to VictoriaMetrics](https://www.pingcap.com/blog/tidb-observability-migrating-prometheus-victoriametrics/)
 * [TigrisData: We do our billing with Prometheus](https://www.tigrisdata.com/blog/billing-prometheus/)

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -556,7 +556,7 @@ See more details on [how to monitor VictoriaMetrics components](https://docs.vic
 - `-storage.maxHourlySeries` is the limit on the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) during the last hour.
 - `-storage.maxDailySeries` is the limit on the number of unique time series during the day. This limit can be used for limiting daily [time series churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate).
 
-It is possible to use `-1` as a value for these flags{{% available_from "#" %}} in order to enable series tracking but set limit to maximum possible value.
+It is possible to use `-1` as a value for these flags{{% available_from "v1.140.0" %}} in order to enable series tracking but set limit to maximum possible value.
 This is useful in order to estimate the number of unique series written to `vmstorage` without enforcing limits.
 
 Note that these limits are set and applied individually per each `vmstorage` node in the cluster. So, if the cluster has `N` `vmstorage` nodes, then the cluster-level limits will be `N` times bigger than the per-`vmstorage` limits.

docs/victoriametrics/LTS-releases.md

@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
 
 ## Currently supported LTS release lines
 
-- v1.136.x - the latest one is [v1.136.3 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.3)
-- v1.122.x - the latest one is [v1.122.18 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.18)
+- v1.136.x - the latest one is [v1.136.4 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.4)
+- v1.122.x - the latest one is [v1.122.19 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.19)

docs/victoriametrics/Quick-Start.md

@@ -58,9 +58,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.139.0
+docker pull victoriametrics/victoria-metrics:v1.140.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.139.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.140.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

docs/victoriametrics/README.md

@@ -1979,7 +1979,7 @@ By default, VictoriaMetrics doesn't limit the number of stored time series. The
 
 Both limits can be set simultaneously. If any of these limits is reached, then incoming samples for new time series are dropped. A sample of dropped series is put in the log with `WARNING` level.
 
-It is possible to use `-1` as a value for these flags{{% available_from "#" %}} in order to enable series tracking but set limit to maximum possible value.
+It is possible to use `-1` as a value for these flags{{% available_from "v1.140.0" %}} in order to enable series tracking but set limit to maximum possible value.
 This is useful in order to estimate the number of unique series which is written to VictoriaMetrics single without enforcing limits.
 
 The exceeded limits can be [monitored](#monitoring) with the following metrics:

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,12 +26,16 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
+
+Released at 2026-04-10
+
 **Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).
 
 * SECURITY: upgrade Go builder from Go1.26.1 to Go1.26.2. See [the list of issues addressed in Go1.26.2](https://github.com/golang/go/issues?q=milestone%3AGo1.26.2%20label%3ACherryPickApproved).
 
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add per-URL `-remoteWrite.disableMetadata` flag to disable metadata sending for specific remote storage URLs. See [#10711](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10711). Thanks to @evkuzin for the contribution.
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `profile` option to `ec2_sd_configs` for loading credentials from named AWS profiles in `~/.aws/credentials` and `~/.aws/config`, including `source_profile` chaining and `role_arn` resolution. See [ec2_sd_configs docs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#ec2_sd_configs). Issue [#1685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1685).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `profile` option to `ec2_sd_configs` for loading credentials from named AWS profiles in `~/.aws/credentials` and `~/.aws/config`, including `source_profile` chaining and `role_arn` resolution. See [ec2_sd_configs docs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#ec2_sd_configs). Issue [#1685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1685). Thanks to @andriibeee for the contribution.
 * FEATURE: introduce `vm_filestream_fsync_duration_seconds_total` and `vm_filestream_fsync_calls_total` metrics, which can be used for detecting slow storage if it cannot keep up with the current data ingestion rate. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432). Thanks to @mehrdadbn9 for the contribution.
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add dedicated `thanos` mode for [migrating data from Thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/thanos/). This mode supports both raw and downsampled Thanos blocks, including all aggregate types (count, sum, min, max, counter). Each aggregate is imported as a separate metric with resolution and aggregate type suffixes (e.g., `metric_name:5m:count`). The new mode uses `--thanos-*` prefixed flags: `--thanos-snapshot`, `--thanos-concurrency`, `--thanos-filter-time-start`, `--thanos-filter-time-end`, `--thanos-filter-label`, `--thanos-filter-label-value`, and `--thanos-aggr-types`. See [#9262](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9262).
 * FEATURE: [dashboards/alert-statistics](https://grafana.com/grafana/dashboards/24553): add pending and firing alerts stats; fix query in `FIRING over time by group` panel. See [#10571](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10571). Thanks to @sias32 for the contribution.
@@ -44,13 +48,15 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): CSV export on the `Raw Query` tab now includes all labels from the executed query. VMUI no longer prepends a header row, as it is now provided by the backend. See [#10667](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10667) and [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666). Thanks to @lawrence3699 for the contribution.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add header row to `/api/v1/export/csv` output and auto-detect header rows during import via `/api/v1/import/csv`. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666). Thanks to @andriibeee for the contribution.
 * FEATURE: all VictoriaMetrics components: expose operating system name and release version as metric `vm_os_info`. See [#10481](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10481).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): align group evaluation time with the `eval_offset` option to help manage group execution more effectively. See [#10772](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10772).
 
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry the requests that failed with unexpected EOF due to unstable network to S3 service. See [#10699](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10699).
 * BUGFIX: All VictoriaMetrics components: Fix an issue where `unsupported` metric metadata type was exposed for summaries and quantiles if a summary wasn't updated within a certain time window. See [metrics#120](https://github.com/VictoriaMetrics/metrics/issues/120) and [metrics#121](https://github.com/VictoriaMetrics/metrics/pull/121).
-* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): align request body buffering flags - `maxRequestBodySizeToRetry` and `requestBufferSize` to the same `16KB` value. Allow disabling request buffering by setting `requestBufferSize=0`. See [#10675](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10675)
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `scrape_series_added` metric to update only on successful scrapes, aligning its behavior with Prometheus. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent partial responses from second-level vmselect nodes in [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). Ensures response completeness and correctness, and avoids cache pollution in top-level vmselect. See [#10678](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Fix storage connection saturation spikes at 00:00 UTC and improve data ingestion when the storage is restarted during the first hour of the day. See [10698](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10698).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent rare panic during storage start-up at 100% disk usage. See [#10747](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10747) Thanks to @nmn3m for the contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent data ingestion from getting completely stuck when storage is under heavy load. See [#10784](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10784). Thanks to @fxrlv for the contribution.
 
 ## [v1.139.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.139.0)
 
@@ -74,7 +80,7 @@ Released at 2026-03-27
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic during directory deletion on `NFS`-based mounts. The bug was introduced in [83da33d8](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/83da33d8cfe8352fd0022d05a8b6346ebb48420d) and included in [v1.123.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11230). See [#9842](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9842).
 
-## [v1.139.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.139.0)
+## [v1.138.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.138.0)
 
 Released at 2026-03-13
 
@@ -129,6 +135,23 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.4](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.4)
+
+Released at 2026-04-10
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* SECURITY: upgrade Go builder from Go1.26.1 to Go1.26.2. See [the list of issues addressed in Go1.26.2](https://github.com/golang/go/issues?q=milestone%3AGo1.26.2%20label%3ACherryPickApproved).
+
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry the requests that failed with unexpected EOF due to unstable network to S3 service. See [#10699](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10699).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `scrape_series_added` metric to update only on successful scrapes, aligning its behavior with Prometheus. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent partial responses from second-level vmselect nodes in [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). Ensures response completeness and correctness, and avoids cache pollution in top-level vmselect. See [#10678](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Fix storage connection saturation spikes at 00:00 UTC and improve data ingestion when the storage is restarted during the first hour of the day. See [10698](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10698).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent rare panic during storage start-up at 100% disk usage. See [#10747](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10747) Thanks to @nmn3m for the contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent data ingestion from getting completely stuck when storage is under heavy load. See [#10784](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10784). Thanks to @fxrlv for the contribution.
+
 ## [v1.136.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.3)
 
 Released at 2026-03-27
@@ -332,6 +355,21 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.19](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.19)
+
+Released at 2026-04-10
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* SECURITY: upgrade Go builder from Go1.25.8 to Go1.25.9. See [the list of issues addressed in Go1.25.9](https://github.com/golang/go/issues?q=milestone%3AGo1.25.9+label%3ACherryPickApproved).
+
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry the requests that failed with unexpected EOF due to unstable network to S3 service. See [#10699](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10699).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `scrape_series_added` metric to update only on successful scrapes, aligning its behavior with Prometheus. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent partial responses from second-level vmselect nodes in [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). Ensures response completeness and correctness, and avoids cache pollution in top-level vmselect. See [#10678](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10678)
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent rare panic during storage start-up at 100% disk usage. See [#10747](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10747) Thanks to @nmn3m for the contribution.
+
 ## [v1.122.18](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.18)
 
 Released at 2026-03-27

docs/victoriametrics/enterprise.md

@@ -117,7 +117,7 @@ It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [
 
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
 and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -135,8 +135,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -151,7 +151,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.139.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.140.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](#binary-releases) section.
@@ -161,13 +161,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.139.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.140.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.139.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.140.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -177,7 +177,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.139.0
+    image: victoriametrics/victoria-metrics:v1.140.0
     ports:
       - 8428:8428
     volumes:
@@ -209,7 +209,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -220,7 +220,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 
 license:
   secret:
@@ -270,7 +270,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -287,7 +287,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.139.0-enterprise
+    tag: v1.140.0-enterprise
 ```
 
 Example secret with license key:
@@ -338,7 +338,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.139.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz`
 
 Includes:
 
@@ -347,7 +347,7 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.139.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.140.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## Monitoring license expiration
 

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/victoria-metrics-linux-amd64-v1.139.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.139.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.140.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/victoria-metrics-linux-amd64-v1.139.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.139.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.140.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -548,9 +548,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
   -storage.idbPrefillStart duration
      Specifies how early VictoriaMetrics starts pre-filling indexDB records before indexDB rotation. Starting the pre-fill process earlier can help reduce resource usage spikes during rotation. In most cases, this value should not be changed. The maximum allowed value is 23h. (default 1h0m0s)
   -storage.maxDailySeries int
-     The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . See also -storage.maxHourlySeries
+     The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See also -storage.maxHourlySeries
   -storage.maxHourlySeries int
-     The maximum number of unique series can be added to the storage during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . See also -storage.maxDailySeries
+     The maximum number of unique series can be added to the storage during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See also -storage.maxDailySeries
   -storage.maxMetadataStorageSize size
      Overrides max size for metrics metadata entries in-memory storage. If set to 0 or a negative value, defaults to 1% of allowed memory.
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)

docs/victoriametrics/vmagent.md

@@ -941,7 +941,7 @@ The limit can be enforced by setting the following command-line flags:
 * `-remoteWrite.maxDailySeries` - limits the number of unique time series `vmagent` can write to remote storage systems during the last day.
   Useful for limiting daily churn rate.
 
-It is possible to use `-1` as a value for these flags{{% available_from "#" %}} in order to enable series tracking but set limit to maximum possible value.
+It is possible to use `-1` as a value for these flags{{% available_from "v1.140.0" %}} in order to enable series tracking but set limit to maximum possible value.
 This is useful in order to estimate the number of unique series which is written to remote storage systems without enforcing limits.
 
 Both limits can be set simultaneously. If any of these limits is reached, then samples for new time series are dropped instead of sending

docs/victoriametrics/vmagent_common_flags.md

@@ -389,6 +389,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Optional path to bearer token file to use for the corresponding -remoteWrite.url. The token is re-read from the file every second
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
+  -remoteWrite.disableMetadata array
+     Whether to disable sending metadata to the corresponding -remoteWrite.url. By default, metadata sending is controlled by the global -enableMetadata flag
+     Supports array of values separated by comma or specified via multiple flags.
+     Empty values are set to false.
   -remoteWrite.disableOnDiskQueue array
      Whether to disable storing pending data to -remoteWrite.tmpDataPath when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload
      Supports array of values separated by comma or specified via multiple flags.
@@ -419,14 +423,14 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 8388608)
   -remoteWrite.maxDailySeries int
-     The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
+     The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
   -remoteWrite.maxDiskUsagePerURL array
      The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. Disk usage is unlimited if the value is set to 0
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB. (default 0)
      Supports array of values separated by comma or specified via multiple flags.
      Empty values are set to default value.
   -remoteWrite.maxHourlySeries int
-     The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
+     The maximum number of unique series vmagent can send to remote storage systems during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
   -remoteWrite.maxMetadataPerBlock int
      The maximum number of metadata to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 5000)
   -remoteWrite.maxRowsPerBlock int

docs/victoriametrics/vmalert.md

@@ -1236,8 +1236,8 @@ For example:
 ```yaml
 groups:
   - name: BaseGroup
-    interval: 1m
-    eval_offset: 10s
+    interval: 5m
+    eval_offset: 1m
     rules:
       - record: http_server_request_duration_seconds:sum_rate:5m:http_get
         expr: |
@@ -1258,8 +1258,8 @@ groups:
             )
           )
   - name: TopGroup
-    interval: 1m
-    eval_offset: 40s
+    interval: 5m
+    eval_offset: 3m
     rules:
       - record: http_server_request_duration_seconds:sum_rate:5m:merged
         expr: |
@@ -1271,20 +1271,20 @@ groups:
 This configuration ensures that rules in `BaseGroup` are executed at(assuming vmalert starts at `12:00:00`):
 
 ```
-[12:00:10, 12:01:10, 12:02:10, 12:03:10...]
+[12:01:00, 12:06:00, 12:11:00, 12:16:00...]
 ```
 
 while rules in group `TopGroup` are executed at:
 
 ```
-[12:00:40, 12:01:40, 12:02:40, 12:03:40...]
+[12:03:00, 12:08:00, 12:13:00, 12:18:00...]
 ```
 
-As a result, `TopGroup` always gets the latest results of `BaseGroup`.
+As a result, `TopGroup` can consistently obtain the latest results from `BaseGroup` if `BaseGroup` completes its evaluation and uploads its results to the datasource within 2 minutes.
 
 By default, the `eval_offset` values should be at least 30 seconds apart to accommodate the
 `-search.latencyOffset(default 30s)` command-line flag at vmselect or VictoriaMetrics single-node.
-The minimum `eval_offset` gap can be adjusted accordingly with `-search.latencyOffset`.
+The minimum `eval_offset` gap should be adjusted according to the sum of the execution duration of `BaseGroup` and `-search.latencyOffset`.
 
 ### Notifier configuration file
 

docs/victoriametrics/vmauth.md

@@ -1137,7 +1137,7 @@ The following [metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/
 * `vmauth_buffer_request_body_duration_seconds` - the [summary](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#summary) for the request body buffering duration.
   Use this metric to understand buffering performance and identify slow clients.
 
-See also [concurrency limits](https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limits).
+See also [concurrency limits](https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting).
 
 ## Backend TLS setup
 

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.139.0/vmutils-darwin-arm64-v1.139.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/vmutils-darwin-arm64-v1.140.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.139.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.140.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

docs/victoriametrics/vmctl/vmctl_flags.md

@@ -20,6 +20,7 @@ COMMANDS:
    influx        Migrate time series from InfluxDB
    remote-read   Migrate time series via Prometheus remote-read protocol
    prometheus    Migrate time series from Prometheus
+   thanos        Migrate time series from Thanos blocks (supports raw and downsampled data)
    vm-native     Migrate time series between VictoriaMetrics installations
    verify-block  Verifies exported block with VictoriaMetrics Native format
    help, h       Shows a list of commands or help for one command

docs/victoriametrics/vmctl/vmctl_prometheus_flags.md

@@ -29,7 +29,7 @@ OPTIONS:
    --prom-filter-time-end value                                       The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
    --prom-filter-label value                                          Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
    --prom-filter-label-value value                                    Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
-   --prom-tmp-dir-path value                                          Path to directory to be used for temporary files. (default: "/tmp")
+   --prom-tmp-dir-path value                                          Path to directory to be used for temporary files. (default: "/var/folders/9b/rj_f77q52w57vnxx236qh51m0000gn/T/")
    --vm-addr value                                                    VictoriaMetrics address to perform import requests. 
       Should be the same as --httpListenAddr value for single-node version or vminsert component. 
       When importing into the clustered version do not forget to set additionally --vm-account-id flag. 

docs/victoriametrics/vmstorage_common_flags.md

@@ -224,9 +224,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -storage.idbPrefillStart duration
      Specifies how early VictoriaMetrics starts pre-filling indexDB records before indexDB rotation. Starting the pre-fill process earlier can help reduce resource usage spikes during rotation. In most cases, this value should not be changed. The maximum allowed value is 23h. (default 1h0m0s)
   -storage.maxDailySeries int
-     The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . See also -storage.maxHourlySeries
+     The maximum number of unique series can be added to the storage during the last 24 hours. Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See also -storage.maxHourlySeries
   -storage.maxHourlySeries int
-     The maximum number of unique series can be added to the storage during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . See also -storage.maxDailySeries
+     The maximum number of unique series can be added to the storage during the last hour. Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . Setting this flag to '-1' sets limit to maximum possible value (2147483647) which is useful in order to enable series tracking without enforcing limits. See also -storage.maxDailySeries
   -storage.maxMetadataStorageSize size
      Overrides max size for metrics metadata entries in-memory storage. If set to 0 or a negative value, defaults to 1% of allowed memory.
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)

lib/promutil/labelscompressor.go

@@ -3,6 +3,7 @@ package promutil
 import (
 	"sync"
 	"sync/atomic"
+	"time"
 	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -11,8 +12,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
-// LabelsCompressor compresses []prompb.Label into short binary strings
-type LabelsCompressor struct {
+const minRotationInterval = time.Hour
+
+// labelsCompressor compresses []prompb.Label into short binary strings.
+type labelsCompressor struct {
 	labelToIdx sync.Map
 	idxToLabel labelsMap
 
@@ -21,20 +24,18 @@ type LabelsCompressor struct {
 	totalSizeBytes atomic.Uint64
 }
 
-// SizeBytes returns the size of lc data in bytes
-func (lc *LabelsCompressor) SizeBytes() uint64 {
+func (lc *labelsCompressor) sizeBytes() uint64 {
 	return uint64(unsafe.Sizeof(*lc)) + lc.totalSizeBytes.Load()
 }
 
-// ItemsCount returns the number of items in lc
-func (lc *LabelsCompressor) ItemsCount() uint64 {
+func (lc *labelsCompressor) itemsCount() uint64 {
 	return lc.nextIdx.Load()
 }
 
-// Compress compresses labels, appends the compressed labels to dst and returns the result.
+// compress compresses labels, appends the compressed labels to dst and returns the result.
 //
-// It is safe calling Compress from concurrent goroutines.
-func (lc *LabelsCompressor) Compress(dst []byte, labels []prompb.Label) []byte {
+// It is safe calling compress from concurrent goroutines.
+func (lc *labelsCompressor) compress(dst []byte, labels []prompb.Label) []byte {
 	if len(labels) == 0 {
 		// Fast path
 		return append(dst, 0)
@@ -42,13 +43,13 @@ func (lc *LabelsCompressor) Compress(dst []byte, labels []prompb.Label) []byte {
 
 	a := encoding.GetUint64s(len(labels) + 1)
 	a.A[0] = uint64(len(labels))
-	lc.compress(a.A[1:], labels)
+	lc.compressInto(a.A[1:], labels)
 	dst = encoding.MarshalVarUint64s(dst, a.A)
 	encoding.PutUint64s(a)
 	return dst
 }
 
-func (lc *LabelsCompressor) compress(dst []uint64, labels []prompb.Label) {
+func (lc *labelsCompressor) compressInto(dst []uint64, labels []prompb.Label) {
 	if len(labels) == 0 {
 		return
 	}
@@ -98,10 +99,10 @@ func cloneLabel(label prompb.Label) prompb.Label {
 	}
 }
 
-// Decompress decompresses src into []prompb.Label, appends it to dst and returns the result.
+// decompress decompresses src into []prompb.Label, appends it to dst and returns the result.
 //
-// It is safe calling Decompress from concurrent goroutines.
-func (lc *LabelsCompressor) Decompress(dst []prompb.Label, src []byte) []prompb.Label {
+// It is safe calling decompress from concurrent goroutines.
+func (lc *labelsCompressor) decompress(dst []prompb.Label, src []byte) []prompb.Label {
 	labelsLen, nSize := encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
 		logger.Panicf("BUG: cannot unmarshal labels length from uvarint")
@@ -124,12 +125,12 @@ func (lc *LabelsCompressor) Decompress(dst []prompb.Label, src []byte) []prompb.
 	if len(tail) > 0 {
 		logger.Panicf("BUG: unexpected non-empty tail left: len(tail)=%d; tail=%X", len(tail), tail)
 	}
-	dst = lc.decompress(dst, a.A)
+	dst = lc.decompressInternal(dst, a.A)
 	encoding.PutUint64s(a)
 	return dst
 }
 
-func (lc *LabelsCompressor) decompress(dst []prompb.Label, src []uint64) []prompb.Label {
+func (lc *labelsCompressor) decompressInternal(dst []prompb.Label, src []uint64) []prompb.Label {
 	for _, idx := range src {
 		label, ok := lc.idxToLabel.Load(idx)
 		if !ok {
@@ -232,3 +233,143 @@ func (lm *labelsMap) moveMutableToReadOnlyLocked(pReadOnly *[]*prompb.Label) {
 	clear(lm.mutable)
 	lm.readOnly.Store(&labels)
 }
+
+// labelsCompressorState holds the current and previous labelsCompressor instances and generation byte that changes between rotations
+// and is used to pick a right compressor during decompression
+type labelsCompressorState struct {
+	gen      byte
+	current  *labelsCompressor
+	previous *labelsCompressor
+}
+
+// LabelsCompressor is a rotating compressor that maintains two labelsCompressor
+// instances to bound memory growth from stale label sets.
+//
+// Consumers must call Register on creation and Unregister on shutdown for a proper rotation period calculation.
+type LabelsCompressor struct {
+	state atomic.Pointer[labelsCompressorState]
+
+	rotationInterval atomic.Int64
+	startOnce        sync.Once
+
+	registryMu sync.Mutex
+	registry   []time.Duration
+}
+
+// getState returns current labelsCompressorState, which is initialized if needed.
+func (lc *LabelsCompressor) getState() *labelsCompressorState {
+	if s := lc.state.Load(); s != nil {
+		return s
+	}
+	s := &labelsCompressorState{gen: 0, current: &labelsCompressor{}}
+	// use CompareAndSwap to avoid overwriting pointer which could be stored by another thread
+	lc.state.CompareAndSwap(nil, s)
+	return lc.state.Load()
+}
+
+// rotate resets current compressor and moves its state to previous.
+func (lc *LabelsCompressor) rotate() {
+	old := lc.getState()
+	lc.state.Store(&labelsCompressorState{
+		gen:      old.gen ^ 1,
+		current:  &labelsCompressor{},
+		previous: old.current,
+	})
+}
+
+// Register records maxStaleness for a new consumer, recomputes the rotation
+// interval, starts the background rotation goroutine on the first call, and
+// returns an id that must be passed to Unregister when the consumer stops.
+func (lc *LabelsCompressor) Register(maxStaleness time.Duration) {
+	lc.registryMu.Lock()
+	lc.registry = append(lc.registry, maxStaleness)
+	max := lc.maxStaleness()
+	lc.registryMu.Unlock()
+
+	lc.rotationInterval.Store(int64(max * 2))
+	lc.startOnce.Do(func() {
+		lc.getState()
+		go func() {
+			for {
+				time.Sleep(time.Duration(lc.rotationInterval.Load()))
+				lc.rotate()
+			}
+		}()
+	})
+}
+
+// Unregister removes the given consumer ID from the registry and recomputes
+// the rotation interval from the remaining registered consumers.
+func (lc *LabelsCompressor) Unregister(maxStaleness time.Duration) {
+	lc.registryMu.Lock()
+	for i, s := range lc.registry {
+		if s == maxStaleness {
+			lc.registry = append(lc.registry[:i], lc.registry[i+1:]...)
+			break
+		}
+	}
+	max := lc.maxStaleness()
+	lc.registryMu.Unlock()
+	lc.rotationInterval.Store(int64(max * 2))
+}
+
+// maxStaleness returns the maximum staleness across all registered consumers.
+// Must be called with registryMu held.
+func (lc *LabelsCompressor) maxStaleness() time.Duration {
+	maxStaleness := time.Duration(0)
+	for _, d := range lc.registry {
+		if d > maxStaleness {
+			maxStaleness = d
+		}
+	}
+	return max(maxStaleness, minRotationInterval)
+}
+
+// Compress appends the generation byte followed by the compressed labels
+// to dst and returns the result.
+//
+// It is safe calling Compress from concurrent goroutines.
+func (lc *LabelsCompressor) Compress(dst []byte, labels []prompb.Label) []byte {
+	s := lc.getState()
+	dst = append(dst, s.gen)
+	return s.current.compress(dst, labels)
+}
+
+// Decompress reads the generation byte from key and decompresses the
+// remaining bytes using the corresponding labelsCompressor instance.
+func (lc *LabelsCompressor) Decompress(dst []prompb.Label, key []byte) []prompb.Label {
+	if len(key) == 0 {
+		logger.Panicf("BUG: unexpected empty key in Decompress")
+	}
+	gen := key[0]
+	s := lc.getState()
+	var c *labelsCompressor
+	if s.gen == gen {
+		c = s.current
+	} else if s.previous != nil {
+		c = s.previous
+	} else {
+		logger.Panicf("BUG: compressor for generation %d is not available; current generation is %d", gen, s.gen)
+	}
+	return c.decompress(dst, key[1:])
+}
+
+// SizeBytes returns the total memory used by the active compressor instances
+func (lc *LabelsCompressor) SizeBytes() uint64 {
+	s := lc.getState()
+	n := s.current.sizeBytes()
+	if s.previous != nil {
+		n += s.previous.sizeBytes()
+	}
+	return n
+}
+
+// ItemsCount returns the total number of label entries stored across the active
+func (lc *LabelsCompressor) ItemsCount() uint64 {
+	s := lc.getState()
+	n := s.current.itemsCount()
+	if s.previous != nil {
+		n += s.previous.itemsCount()
+	}
+	return n
+}

lib/protoparser/protoparserutil/compress_reader.go

@@ -26,7 +26,7 @@ import (
 // Later we could consider to make this limit configurable
 const maxSnappyBlockSize = 56_000_000
 
-// ReadUncompressedData reads uncompressed data from r using the given encoding and then passes it to the callback.
+// ReadUncompressedData reads uncompressed data from r using the given contentType and then passes it to the callback.
 //
 // The maxDataSize limits the maximum data size, which can be read from r.
 //

lib/storage/storage.go

@@ -268,7 +268,8 @@ func MustOpenStorage(path string, opts OpenOptions) *Storage {
 
 	// check for free disk space before opening the table
 	// to prevent unexpected part merges. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4023
-	s.startFreeDiskSpaceWatcher()
+	freeSpaceBytes := fs.MustGetFreeSpace(s.path)
+	s.isReadOnly.Store(freeSpaceBytes < freeDiskSpaceLimitBytes)
 
 	// Load data
 	tablePath := filepath.Join(path, dataDirname)
@@ -314,6 +315,7 @@ func MustOpenStorage(path string, opts OpenOptions) *Storage {
 	s.startCurrHourMetricIDsUpdater()
 	s.startNextDayMetricIDsUpdater()
 	s.startLegacyRetentionWatcher()
+	s.startFreeDiskSpaceWatcher()
 
 	return s
 }

lib/streamaggr/deduplicator.go

@@ -45,6 +45,7 @@ type Deduplicator struct {
 //
 // MustStop must be called on the returned deduplicator in order to free up occupied resources.
 func NewDeduplicator(pushFunc PushFunc, enableWindows bool, interval time.Duration, dropLabels []string, alias string) *Deduplicator {
+	lc.Register(2 * interval)
 	d := &Deduplicator{
 		da:            newDedupAggr(),
 		dropLabels:    dropLabels,
@@ -92,6 +93,8 @@ func (d *Deduplicator) MustStop() {
 	metrics.UnregisterSet(d.ms, true)
 	d.ms = nil
 
+	lc.Unregister(2 * d.interval)
+
 	close(d.stopCh)
 	d.wg.Wait()
 }

lib/streamaggr/streamaggr.go

@@ -53,10 +53,10 @@ var supportedOutputs = []string{
 	"unique_samples",
 }
 
-var (
-	// lc contains information about all compressed labels for streaming aggregation
-	lc promutil.LabelsCompressor
+// lc is the global rotating labels compressor shared across all aggregators.
+var lc promutil.LabelsCompressor
 
+var (
 	_ = metrics.NewGauge(`vm_streamaggr_labels_compressor_size_bytes`, func() float64 {
 		return float64(lc.SizeBytes())
 	})
@@ -310,12 +310,22 @@ func loadFromData(data []byte, filePath string, pushFunc PushFunc, opts *Options
 	}
 
 	metrics.RegisterSet(ms)
-	return &Aggregators{
+	a := &Aggregators{
 		as:         as,
 		configData: configData,
 		filePath:   filePath,
 		ms:         ms,
-	}, nil
+	}
+	lc.Register(a.maxStaleness())
+	return a, nil
+}
+
+func (a *Aggregators) maxStaleness() time.Duration {
+	maxStaleness := time.Duration(0)
+	for _, aggr := range a.as {
+		maxStaleness = max(aggr.stalenessInterval, maxStaleness)
+	}
+	return maxStaleness
 }
 
 // IsEnabled returns true if Aggregators has at least one configured aggregator
@@ -335,6 +345,8 @@ func (a *Aggregators) MustStop() {
 		return
 	}
 
+	lc.Unregister(a.maxStaleness())
+
 	metrics.UnregisterSet(a.ms, true)
 	a.ms = nil
 
@@ -1078,6 +1090,9 @@ func compressLabels(dst []byte, inputLabels, outputLabels []prompb.Label) []byte
 }
 
 func decompressLabels(dst []prompb.Label, key string) []prompb.Label {
+	if len(key) == 0 {
+		logger.Panicf("BUG: unexpected empty key in decompressLabels")
+	}
 	return lc.Decompress(dst, bytesutil.ToUnsafeBytes(key))
 }
 

lib/writeconcurrencylimiter/concurrencylimiter.go

@@ -32,7 +32,8 @@ var (
 //
 // The Reader must be obtained via GetReader() call.
 type Reader struct {
-	r io.Reader
+	r                    io.Reader
+	increasedConcurrency bool
 }
 
 // GetReader returns the Reader for r.
@@ -49,6 +50,7 @@ func GetReader(r io.Reader) (*Reader, error) {
 	}
 	rr := v.(*Reader)
 	rr.r = r
+	rr.increasedConcurrency = true
 
 	return rr, nil
 }
@@ -58,9 +60,11 @@ func GetReader(r io.Reader) (*Reader, error) {
 // It decreases the concurrency.
 func PutReader(r *Reader) {
 	r.r = nil
+	if r.increasedConcurrency {
+		DecConcurrency()
+		r.increasedConcurrency = false
+	}
 	readerPool.Put(r)
-
-	DecConcurrency()
 }
 
 var readerPool sync.Pool
@@ -68,12 +72,14 @@ var readerPool sync.Pool
 // Read implements io.Reader.
 func (r *Reader) Read(p []byte) (int, error) {
 	DecConcurrency()
+	r.increasedConcurrency = false
 
 	n, err := r.r.Read(p)
 
 	if errC := IncConcurrency(); errC != nil {
 		return n, errC
 	}
+	r.increasedConcurrency = true
 
 	if errors.Is(err, io.ErrUnexpectedEOF) {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8704