build(deps): bump github/codeql-action/init from 4.36.2 to 4.36.3

Bumps [github/codeql-action/init](https://github.com/github/codeql-action) from 4.36.2 to 4.36.3.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](8aad20d150...54f647b7e1)

---
updated-dependencies:
- dependency-name: github/codeql-action/init
  dependency-version: 4.36.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/build.yml

@@ -65,7 +65,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/changelog-linter.yml

@@ -13,7 +13,7 @@ jobs:
       contents: read
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           fetch-depth: 0  # we need full history for commit verification
           persist-credentials: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/codeql-analysis-go.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -54,14 +54,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           path: __vm
           persist-credentials: false
 
       - name: Checkout private code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: 'vm-runner'
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -79,7 +79,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -106,7 +106,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/vmui.yml

@@ -26,13 +26,11 @@ jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
     permissions:
-      checks: write
       contents: read
-      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -70,12 +68,6 @@ jobs:
         env:
           VMUI_SKIP_INSTALL: true
 
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
       - name: Check overall status
         run: |
           echo "Lint status: ${{ steps.lint.outcome }}"

Makefile

@@ -494,11 +494,13 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 apptest-mixed: victoria-metrics-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
-	VERSION=v1.145.0; \
+	VERSION=v1.147.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
 	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
 	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
 	DIR=/tmp/$${VERSION}; \
 	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
 		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
 	); \
 	VMSELECT_PATH=$${DIR}/vmselect-prod \

app/vmagent/main.go

@@ -462,7 +462,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+		rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
+		promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
 		return true
 	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmagent/opentelemetry/request_handler.go

@@ -63,6 +63,7 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
+	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range tss {
@@ -82,7 +83,19 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	var metadataTotal int
 	if prommetadata.IsEnabled() {
-		ctx.WriteRequest.Metadata = mms
+		for i := range mms {
+			mm := &mms[i]
+			mmsDst = append(mmsDst, prompb.MetricMetadata{
+				MetricFamilyName: mm.MetricFamilyName,
+				Help:             mm.Help,
+				Type:             mm.Type,
+				Unit:             mm.Unit,
+
+				AccountID: mm.AccountID,
+				ProjectID: mm.ProjectID,
+			})
+		}
+		ctx.WriteRequest.Metadata = mmsDst
 		metadataTotal = len(mms)
 	}
 

app/vmagent/remotewrite/client.go

@@ -151,17 +151,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
+	rwURL, err := url.Parse(remoteWriteURL)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	retryMaxIntervalFlag := retryMaxTime
 	if retryMaxInterval.String() != "" {
 		retryMaxIntervalFlag = retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
+		remoteWriteURL:   rwURL.String(),
 		authCfg:          authCfg,
 		awsCfg:           awsCfg,
 		fq:               fq,

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,6 +83,16 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
 	p := remoteWriteURLRelabelConfigData.Load()
@@ -108,6 +119,24 @@ func WriteURLRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(d)
 }
 
+// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
+func GetURLRelabelConfigString() []string {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
+	}
+	var ss []string
+	for i := range *remoteWriteURLs {
+		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
+		ss = append(ss, string(cfgDataBytes))
+	}
+	return ss
+}
+
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {

app/vmagent/remotewrite/remotewrite.go

@@ -1064,7 +1064,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 
 	copyTimeSeriesIfNeeded := func() {
 		if v == nil {
-			v := tssPool.Get().(*[]prompb.TimeSeries)
+			v = tssPool.Get().(*[]prompb.TimeSeries)
 			tss = append(*v, tss...)
 		}
 	}

app/vmalert-tool/unittest/recording.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"reflect"
 	"sort"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -94,10 +93,3 @@ Outer:
 	}
 	return
 }
-
-func durationToTime(pd *promutil.Duration) time.Time {
-	if pd == nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(pd.Duration().Milliseconds())
-}

app/vmalert-tool/unittest/unittest.go

@@ -44,12 +44,19 @@ import (
 var (
 	storagePath    string
 	httpListenAddr string
-	// insert series from 1970-01-01T00:00:00
-	testStartTime          = time.Unix(0, 0).UTC()
+	// Insert series from 2000-01-01T00:00:00.
+	testStartTime          = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
 
+func durationToTime(pd *promutil.Duration) time.Time {
+	if pd == nil {
+		return testStartTime
+	}
+	return testStartTime.Add(pd.Duration())
+}
+
 const (
 	testStoragePath = "vmalert-unittest"
 )

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -94,6 +95,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	hc := &http.Client{Transport: tr}
+	datasourceURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
+	}
+	hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
 
 	if extraParams == nil {
 		extraParams = url.Values{}
@@ -120,9 +127,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	}
 
 	return &Client{
-		c:                &http.Client{Transport: tr},
+		c:                hc,
 		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		datasourceURL:    strings.TrimSuffix(datasourceURL.String(), "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
 		extraParams:      extraParams,

app/vmalert/remoteread/init.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -76,7 +78,13 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	c := &http.Client{Transport: tr}
+	rrURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
+	}
 
+	c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +97,5 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
-	c := &http.Client{Transport: tr}
-	return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
+	return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Concurrency > 0 {
 		cc = cfg.Concurrency
 	}
+	hc := &http.Client{
+		Timeout:   *sendTimeout,
+		Transport: cfg.Transport,
+	}
+	rwURL, err := url.Parse(cfg.Addr)
+	if err != nil {
+		logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	c := &Client{
-		c: &http.Client{
-			Timeout:   *sendTimeout,
-			Transport: cfg.Transport,
-		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		c:             hc,
+		addr:          strings.TrimSuffix(rwURL.String(), "/"),
 		authCfg:       cfg.AuthCfg,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,

app/vmauth/main.go

@@ -386,7 +386,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 			return nil, &httpserver.ErrorWithStatusCode{
 				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
 					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
-				StatusCode: http.StatusBadRequest,
+				StatusCode: http.StatusRequestTimeout,
 			}
 		}
 

app/vmauth/main_synctest_test.go

@@ -0,0 +1,54 @@
+//go:build synctest
+
+package main
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+func TestBufferRequestBody_Timeout(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+
+		defaultMaxQueueDuration := *maxQueueDuration
+		defer func() {
+			*maxQueueDuration = defaultMaxQueueDuration
+		}()
+
+		*maxQueueDuration = 100 * time.Millisecond
+
+		ctx, cancel := context.WithTimeout(context.Background(), *maxQueueDuration)
+		defer cancel()
+
+		_, err := bufferRequestBody(ctx, &timeoutBody{delay: *maxQueueDuration + 100*time.Millisecond}, "foo")
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		esc, ok := err.(*httpserver.ErrorWithStatusCode)
+		if !ok {
+			t.Fatalf("unexpected error type: %s", err)
+		}
+		if esc.StatusCode != http.StatusRequestTimeout {
+			t.Fatalf("read request body timeout meet unexpected status code; got %d; want %d", esc.StatusCode, http.StatusRequestTimeout)
+		}
+	})
+}
+
+type timeoutBody struct {
+	delay time.Duration
+}
+
+func (r *timeoutBody) Read(_ []byte) (int, error) {
+	time.Sleep(r.delay)
+	return 0, context.DeadlineExceeded
+}
+
+func (r *timeoutBody) Close() error {
+	return nil
+}

app/vmselect/main.go

@@ -541,7 +541,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		promscrape.WriteMetricRelabelDebug(w, r, "", nil)
 		return true
 	case "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmselect/promql/binary_op.go

@@ -172,7 +172,13 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
-		if len(left) == 0 || len(right) == 0 {
+		if len(left) == 0 && len(right) == 0 {
+			return nil, nil
+		}
+		if len(left) == 0 && bfa.be.FillLeft == nil {
+			return nil, nil
+		}
+		if len(right) == 0 && bfa.be.FillRight == nil {
 			return nil, nil
 		}
 		left, right, dst, err := adjustBinaryOpTags(bfa.be, left, right)
@@ -183,6 +189,8 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			logger.Panicf("BUG: len(left) must match len(right) and len(dst); got %d vs %d vs %d", len(left), len(right), len(dst))
 		}
 		isBool := bfa.be.Bool
+		fillLeft := bfa.be.FillLeft
+		fillRight := bfa.be.FillRight
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -193,6 +201,19 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			}
 			for j, a := range leftValues {
 				b := rightValues[j]
+				leftIsNaN := math.IsNaN(a)
+				rightIsNaN := math.IsNaN(b)
+				// apply the fill value when either the left or right side is NaN, but not both.
+				if leftIsNaN && rightIsNaN {
+					dstValues[j] = bf(a, b, isBool)
+					continue
+				}
+				if leftIsNaN && fillLeft != nil {
+					a = fillLeft.N
+				}
+				if rightIsNaN && fillRight != nil {
+					b = fillRight.N
+				}
 				dstValues[j] = bf(a, b, isBool)
 			}
 		}
@@ -226,7 +247,7 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		}
 	}
 
-	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
+	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} {fill|fill_left|fill_right} b`
 	var rvsLeft, rvsRight []*timeseries
 	mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
 	joinOp := strings.ToLower(be.JoinModifier.Op)
@@ -239,10 +260,27 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		// Add __name__ to groupTags if metric name must be preserved.
 		groupTags = append(groupTags[:len(groupTags):len(groupTags)], "__name__")
 	}
+	// Add missing keys from mRight to mLeft when fill_left()/fill() modifier is used
+	if be.FillLeft != nil {
+		for k := range mRight {
+			if _, ok := mLeft[k]; !ok {
+				mLeft[k] = nil
+			}
+		}
+	}
 	for k, tssLeft := range mLeft {
 		tssRight := mRight[k]
+		if len(tssLeft) == 0 {
+			if be.FillLeft == nil {
+				logger.Panicf("BUG: unexpected empty tssLeft for key %q when FillLeft is nil", k)
+			}
+			tssLeft = []*timeseries{newFillTimeseries(be, tssRight[0])}
+		}
 		if len(tssRight) == 0 {
-			continue
+			if be.FillRight == nil {
+				continue
+			}
+			tssRight = []*timeseries{newFillTimeseries(be, tssLeft[0])}
 		}
 		switch joinOp {
 		case "group_left":
@@ -287,6 +325,28 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 	return rvsLeft, rvsRight, dst, nil
 }
 
+// newFillTimeseries returns a time series filled with NaN values for the fill_left()/fill_right()/fill() modifiers.
+// These NaN values will be replaced later with the fill value if needed.
+func newFillTimeseries(be *metricsql.BinaryOpExpr, src *timeseries) *timeseries {
+	var ts timeseries
+	ts.CopyFromShallowTimestamps(src)
+	if !be.KeepMetricNames {
+		ts.MetricName.ResetMetricGroup()
+	}
+	groupTags := be.GroupModifier.Args
+	switch strings.ToLower(be.GroupModifier.Op) {
+	case "on":
+		ts.MetricName.RemoveTagsOn(groupTags)
+	default:
+		ts.MetricName.RemoveTagsIgnoring(groupTags)
+	}
+	values := ts.Values
+	for i := range values {
+		values[i] = math.NaN()
+	}
+	return &ts
+}
+
 func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
 	if len(tss) == 0 {
 		logger.Panicf("BUG: tss must contain at least one value")

app/vmselect/promql/eval.go

@@ -424,18 +424,7 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	if bf == nil {
 		return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
 	}
-	var err error
-	var tssLeft, tssRight []*timeseries
-	switch strings.ToLower(be.Op) {
-	case "and", "if":
-		// Fetch right-side series at first, since it usually contains
-		// lower number of time series for `and` and `if` operator.
-		// This should produce more specific label filters for the left side of the query.
-		// This, in turn, should reduce the time to select series for the left side of the query.
-		tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
-	default:
-		tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
-	}
+	tssLeft, tssRight, err := execBinaryOpArgs(qt, ec, be)
 	if err != nil {
 		return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
 	}
@@ -451,6 +440,29 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	return rv, nil
 }
 
+// binaryOpEvalOrder might change the order of evaluation of the left and right sides of a binary operation,
+// when there is chance to push down common label filters from exprFirst to exprSecond in the following executions.
+func binaryOpEvalOrder(be *metricsql.BinaryOpExpr) (exprFirst, exprSecond metricsql.Expr) {
+	exprFirst, exprSecond = be.Left, be.Right
+	switch strings.ToLower(be.Op) {
+	case "and", "if":
+		// For `and` and `if`, fetch the right-side series first, since it usually contains
+		// fewer time series and yields more specific filters for the left side.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	if be.FillLeft != nil && be.FillRight == nil {
+		// For `fill_left(<value>)`, the unmatched series can only come from the right side, so evaluate it first.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	return exprFirst, exprSecond
+}
+
+// canPushdownCommonFilters decides if common label filters can be pushed down from one side of a binary operation to the other.
+//
+// Common filters cannot be pushed down when:
+// - the operator is `or` or `default`;
+// - either side is an aggregation function without explicit grouping;
+// - fill(<value>) modifier is used.
 func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	switch strings.ToLower(be.Op) {
 	case "or", "default":
@@ -459,6 +471,10 @@ func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	if isAggrFuncWithoutGrouping(be.Left) || isAggrFuncWithoutGrouping(be.Right) {
 		return false
 	}
+	// Filters cannot be propagated when fill(<value>) modifier is used.
+	if be.FillLeft != nil && be.FillRight != nil {
+		return false
+	}
 	return true
 }
 
@@ -470,7 +486,15 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
 	return len(afe.Modifier.Args) == 0
 }
 
-func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+	exprFirst, exprSecond := binaryOpEvalOrder(be)
+	firstIsLeft := exprFirst == be.Left
+	sortResult := func(tssFirst, tssSecond []*timeseries) ([]*timeseries, []*timeseries, error) {
+		if firstIsLeft {
+			return tssFirst, tssSecond, nil
+		}
+		return tssSecond, tssFirst, nil
+	}
 	if canPushdownCommonFilters(be) {
 		// Execute binary operation in the following way:
 		//
@@ -512,7 +536,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		if err != nil {
 			return nil, nil, err
 		}
-		return tssFirst, tssSecond, nil
+		return sortResult(tssFirst, tssSecond)
 	}
 
 	// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
@@ -568,7 +592,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 	if errSecond != nil {
 		return nil, nil, errSecond
 	}
-	return tssFirst, tssSecond, nil
+	return sortResult(tssFirst, tssSecond)
 }
 
 func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {

app/vmselect/promql/exec_test.go

@@ -4006,6 +4006,275 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`vector + vector fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill(0) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill() both sides NaN case`, func(t *testing.T) {
+		t.Parallel()
+		q := `(
+			label_set(time() <= 1200, "foo", "common")
+		) + fill(10) (
+			label_set(time() >= 1600, "foo", "common")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1010, 1210, nan, 1610, 1810, 2010},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_left(10) fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_right() only`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common", "extra", "l")
+			or label_set(2, "foo", "left_only", "extra", "l")
+		) + on(foo) fill(0) (
+			label_set(3, "foo", "common", "extra", "r")
+			or label_set(4, "foo", "right_only", "extra", "r")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() group_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "method", "get", "code", "500")
+			or label_set(2, "method", "get", "code", "404")
+			or label_set(3, "method", "put", "code", "501")
+		) + on(method) group_left() fill_right(0) (
+			label_set(10, "method", "get")
+		), "method", "code")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12, 12, 12, 12, 12, 12},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("404"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11, 11, 11, 11, 11, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("500"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("501"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector / vector ignoring() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(6, "method", "get", "code", "500")
+			or label_set(1, "method", "put", "code", "500")
+		) / ignoring(code) fill(0) (
+			label_set(12, "method", "get")
+			or label_set(5, "method", "post")
+			or label_set(10, "method", "put")
+		), "method")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("post"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(scalar)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, time())`

app/vmselect/vmui/assets/rolldown-runtime-CNC7AqOf.js

@@ -0,0 +1 @@
+var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(e){throw n=[e],e}},s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/assets/rolldown-runtime-Cyuzqnbw.js

@@ -1 +0,0 @@
-var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t)=>()=>(e&&(t=e(e=0)),t),s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-xYKUiOTH.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Cyuzqnbw.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-B83wxFqK.js">
+  <script type="module" crossorigin src="./assets/index-D5egN2id.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BBUnmLOr.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -30,7 +30,15 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
 		"The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
-	vmselectAddr                  = flag.String("vmselectAddr", "", "TCP address to accept connections from vmselect services")
+	maxBackfillAge = flagutil.NewRetentionDuration("maxBackfillAge", "0", "The maximum allowed age for the ingested samples with historical timestamps. "+
+		"Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. "+
+		"By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. "+
+		"This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	vmselectAddr = flag.String("vmselectAddr", "", "TCP address to listen for incoming connections from vmselect. "+
+		"When set, the node will be able to accept cluster-native vmselect RPC requests as if it were vmstorage. "+
+		"The tenant ID assigned to this node's data is controlled by -accountID and -projectID flags. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy")
 	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
 		"This reduces CPU usage at the cost of higher network bandwidth usage")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
@@ -146,6 +154,7 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
 	opts := storage.OpenOptions{
 		Retention:                   retentionPeriod.Duration(),
 		FutureRetention:             futureRetention.Duration(),
+		MaxBackfillAge:              maxBackfillAge.Duration(),
 		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
 		MaxHourlySeries:             getMaxHourlySeries(),
 		MaxDailySeries:              getMaxDailySeries(),
@@ -467,8 +476,10 @@ func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/inmemory"}`, tm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/small"}`, tm.SmallSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/big"}`, tm.BigSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/metaindex"}`, tm.MetaindexSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/inmemory"}`, idbm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/file"}`, idbm.FileSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/metaindex"}`, idbm.MetaindexSizeBytes)
 
 	metrics.WriteCounterUint64(w, `vm_rows_received_by_storage_total`, m.RowsReceivedTotal)
 	metrics.WriteCounterUint64(w, `vm_rows_added_to_storage_total`, m.RowsAddedTotal)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.4 AS build-web-stage
+FROM golang:1.26.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/package.json

@@ -9,7 +9,7 @@
     "start": "vite",
     "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
+    "lint": "eslint --format stylish 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
     "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
@@ -23,40 +23,40 @@
     "classnames": "^2.5.1",
     "dayjs": "^1.11.21",
     "lodash.debounce": "^4.0.8",
-    "marked": "^18.0.5",
-    "preact": "^10.29.2",
-    "qs": "^6.15.2",
+    "marked": "^18.0.6",
+    "preact": "^10.29.7",
+    "qs": "^6.15.3",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.17.0",
+    "react-router-dom": "^7.18.1",
     "uplot": "^1.6.32",
-    "vite": "^8.0.16",
+    "vite": "^8.1.4",
     "web-vitals": "^5.3.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.5",
+    "@eslint/eslintrc": "^3.3.6",
     "@eslint/js": "^9.39.2",
     "@preact/preset-vite": "^2.10.5",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^25.9.2",
+    "@types/node": "^26.1.1",
     "@types/qs": "^6.15.1",
     "@types/react": "^19.2.17",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.61.0",
-    "@typescript-eslint/parser": "^8.61.0",
+    "@typescript-eslint/eslint-plugin": "^8.64.0",
+    "@typescript-eslint/parser": "^8.64.0",
     "cross-env": "^10.1.0",
     "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-unused-imports": "^4.4.1",
-    "globals": "^17.6.0",
-    "http-proxy-middleware": "^4.1.0",
+    "globals": "^17.7.0",
+    "http-proxy-middleware": "^4.2.0",
     "jsdom": "^29.1.1",
-    "postcss": "^8.5.15",
+    "postcss": "^8.5.19",
     "sass-embedded": "^1.100.0",
     "typescript": "^6.0.3",
-    "vitest": "^4.1.8"
+    "vitest": "^4.1.10"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseAlert/index.tsx

@@ -98,7 +98,7 @@ const BaseAlert = ({ item, group }: BaseAlertProps) => {
       {!!Object.keys(item.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/index.tsx

@@ -121,7 +121,7 @@ const BaseRule = ({ item, group }: BaseRuleProps) => {
       {!!Object.keys(item?.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/HeaderNav.tsx

@@ -17,10 +17,12 @@ interface HeaderNavProps {
 const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
   const { pathname } = useLocation();
   const [activeMenu, setActiveMenu] = useState(pathname);
+  const [openMenu, setOpenMenu] = useState<string | null>(null);
   const menu = useNavigationMenu();
 
   useEffect(() => {
     setActiveMenu(pathname);
+    setOpenMenu(null);
   }, [pathname]);
 
   return (
@@ -41,6 +43,8 @@ const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
               color={color}
               background={background}
               direction={direction}
+              openMenu={openMenu}
+              setOpenMenu={setOpenMenu}
             />
           )
           : (

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/NavSubItem.tsx

@@ -1,11 +1,8 @@
-import { FC, useRef, useState } from "preact/compat";
-import { useLocation } from "react-router-dom";
+import { FC, useRef, useState, Dispatch, SetStateAction } from "preact/compat";
 import classNames from "classnames";
 import { ArrowDropDownIcon } from "../../../components/Main/Icons";
 import Popper from "../../../components/Main/Popper/Popper";
 import NavItem from "./NavItem";
-import { useEffect } from "react";
-import useBoolean from "../../../hooks/useBoolean";
 import { NavigationItem, NavigationItemType } from "../../../router/navigation";
 
 interface NavItemProps {
@@ -15,6 +12,8 @@ interface NavItemProps {
   color?: string
   background?: string
   direction?: "row" | "column"
+  openMenu: string | null,
+  setOpenMenu: Dispatch<SetStateAction<string | null>>,
 }
 
 const NavSubItem: FC<NavItemProps> = ({
@@ -23,21 +22,18 @@ const NavSubItem: FC<NavItemProps> = ({
   color,
   background,
   submenu,
-  direction = "row"
+  direction = "row",
+  openMenu,
+  setOpenMenu,
 }) => {
-  const { pathname } = useLocation();
-
   const [menuTimeout, setMenuTimeout] = useState<NodeJS.Timeout | null>(null);
   const buttonRef = useRef<HTMLDivElement>(null);
 
-  const {
-    value: openSubmenu,
-    setFalse: handleCloseSubmenu,
-    setTrue: setOpenSubmenu,
-  } = useBoolean(false);
+  const openSubmenu = openMenu === label;
+  const handleCloseSubmenu = () => setOpenMenu(prev => (prev === label ? null : prev));
 
   const handleOpenSubmenu = () => {
-    if (direction === "row" || !openSubmenu) setOpenSubmenu();
+    if (direction === "row" || !openSubmenu) setOpenMenu(label);
     if (direction === "column" && openSubmenu) handleCloseSubmenu();
     if (direction === "row" && menuTimeout) clearTimeout(menuTimeout);
   };
@@ -52,10 +48,6 @@ const NavSubItem: FC<NavItemProps> = ({
     if (menuTimeout) clearTimeout(menuTimeout);
   };
 
-  useEffect(() => {
-    handleCloseSubmenu();
-  }, [pathname]);
-
   return (
     <div
       className={classNames({

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -32,7 +32,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   const match = searchParams.get("match");
   const focusLabel = searchParams.get("focusLabel");
   const isMetric = /__name__/.test(match || "");
-
+  const showMetricNameStats = !(match || focusLabel);
   const progress = totalSeries / totalSeriesAll * 100;
   const diff = totalSeries - totalSeriesPrev;
   const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
@@ -56,7 +56,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
     },
   ].filter(t => t.display);
 
-  if (!totals.length) {
+  if (!totals.length && !showMetricNameStats) {
     return null;
   }
 
@@ -75,7 +75,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           <h4 className="vm-cardinality-totals-card__title">
             {info && (
               <Tooltip title={<p className="vm-cardinality-totals-card__tooltip">{info}</p>}>
-                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon/></div>
+                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon /></div>
               </Tooltip>
             )}
             {title}
@@ -99,7 +99,10 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           )}
         </div>
       ))}
-      <CardinalityMetricNameStats metricNameStats={metricNameStats}/>
+      {
+        showMetricNameStats &&
+        <CardinalityMetricNameStats metricNameStats={metricNameStats} />
+      }
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreAlerts/style.scss

@@ -62,6 +62,13 @@
     padding-right: 40px;
   }
 
+  .vm-annotations-table {
+    tbody > tr > td {
+      vertical-align: top;
+      white-space: pre-line;
+    }
+  }
+
   display: flex;
   flex-direction: column;
   align-items: stretch;

apptest/tests/max_backfill_age_test.go

@@ -0,0 +1,222 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestSingleMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle", []string{
+				"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+				"-retentionPeriod=" + retentionPeriod,
+				"-maxBackfillAge=" + maxBackfillAge,
+			})
+		},
+		stop: func() {
+			tc.StopApp("vmsingle")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+func TestClusterMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&apptest.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage1"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				Vmstorage2Instance: "vmstorage2",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage2"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags:    []string{},
+			})
+		},
+		stop: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1")
+			tc.StopApp("vmstorage2")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+type maxBackfillAgeOpts struct {
+	start func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier
+	stop  func()
+}
+
+func testMaxBackfillAge(tc *apptest.TestCase, opts maxBackfillAgeOpts) {
+	t := tc.T()
+
+	assertSeries := func(app apptest.PrometheusQuerier, prefix string, start, end int64, want []map[string]string) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, apptest.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &apptest.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want,
+			},
+			FailNow: true,
+		})
+	}
+
+	assertQueryResults := func(app apptest.PrometheusQuerier, prefix string, start, end, step int64, want []*apptest.QueryResult) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
+					Start:       fmt.Sprintf("%d", start),
+					End:         fmt.Sprintf("%d", end),
+					Step:        fmt.Sprintf("%dms", step),
+					MaxLookback: fmt.Sprintf("%dms", step-1),
+					NoCache:     "1",
+				})
+			},
+			Want: &apptest.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &apptest.QueryData{
+					ResultType: "matrix",
+					Result:     want,
+				},
+			},
+			FailNow: true,
+		})
+	}
+
+	const numMetrics = 1000
+	now := time.Now().UTC()
+	var start, end, step int64
+	emptySeries := []map[string]string{}
+	emptyQueryResults := []*apptest.QueryResult{}
+
+	// Start sut with the same -retentionPeriod and -maxBackfillAge.
+	sut := opts.start("1y", "1y")
+
+	// Verify that samples older than the retention period are rejected.
+	start = now.Add(-365 * 24 * time.Hour).Add(-time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	outsideRetention := genMaxBackfillAgeData("outside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, outsideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "outside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "outside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that samples within the retention period are accepted and
+	// searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention := genMaxBackfillAgeData("inside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Restart sut with -maxBackfillAge shorter than the -retentionPeriod.
+	opts.stop()
+	sut = opts.start("1y", "6M")
+
+	// Verify that new samples older than max backfill age but still within the
+	// retention period are rejected but existing samples are still searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention2 := genMaxBackfillAgeData("inside_retention2", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention2.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention2", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention2", start, end, step, emptyQueryResults)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Verify that the metrics that are outside the backfill window can still
+	// be deleted.
+	sut.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*inside_retention.*"}`, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that the samples that are within the backfill window are accepted
+	// and searchable.
+	start = now.Add(-180 * 24 * time.Hour).UnixMilli()
+	end = now.Add(-180 * 24 * time.Hour).Add(1 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideMaxBackfillAge := genMaxBackfillAgeData("inside_max_backfill_age", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideMaxBackfillAge.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_max_backfill_age", start, end, insideMaxBackfillAge.wantSeries)
+	assertQueryResults(sut, "inside_max_backfill_age", start, end, step, insideMaxBackfillAge.wantQueryResults)
+
+	opts.stop()
+}
+
+type maxBackfillAgeData struct {
+	samples          []string
+	wantSeries       []map[string]string
+	wantQueryResults []*apptest.QueryResult
+}
+
+func genMaxBackfillAgeData(prefix string, numMetrics, start, step int64) maxBackfillAgeData {
+	samples := make([]string, numMetrics)
+	wantSeries := make([]map[string]string, numMetrics)
+	wantQueryResults := make([]*apptest.QueryResult, numMetrics)
+	for i := range numMetrics {
+		metricName := fmt.Sprintf("metric_%s_%04d", prefix, i)
+		labelName := fmt.Sprintf("label_%s_%04d", prefix, i)
+		labelValue := fmt.Sprintf("value_%s_%04d", prefix, i)
+		value := i
+		timestamp := start + i*step
+		samples[i] = fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
+		wantSeries[i] = map[string]string{
+			"__name__": metricName,
+			labelName:  "value",
+			"label":    labelValue,
+		}
+		wantQueryResults[i] = &apptest.QueryResult{
+			Metric: map[string]string{
+				"__name__": metricName,
+				labelName:  "value",
+				"label":    labelValue,
+			},
+			Samples: []*apptest.Sample{{Timestamp: timestamp, Value: float64(value)}},
+		}
+	}
+	return maxBackfillAgeData{samples, wantSeries, wantQueryResults}
+}

deployment/docker/Makefile

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.24.1
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.24.1
 
-GO_BUILDER_IMAGE := golang:1.26.4
+GO_BUILDER_IMAGE := golang:1.26.5
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.148.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.148.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.146.0
+    image: victoriametrics/vmauth:v1.148.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts-vmagent.yml

@@ -10,7 +10,7 @@ groups:
     concurrency: 2
     rules:
       - alert: PersistentQueueIsDroppingData
-        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path) > 0
+        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path,name) > 0
         for: 10m
         labels:
           severity: critical
@@ -201,4 +201,4 @@ groups:
         annotations:
           summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours."
           description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
-            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
+            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -10,9 +10,9 @@ sitemap:
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.146.0)
-  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.146.0)
-  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.146.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.148.0)
+  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.148.0)
+  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.148.0)
 - [Grafana](https://grafana.com/) (v12.2.0)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
@@ -323,7 +323,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -340,7 +340,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
     ports:
       - 8428:8428
     volumes:
@@ -373,7 +373,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/connecting-vm-components-to-cloud-storage/README.md

@@ -0,0 +1,426 @@
+Several VictoriaMetrics components can connect to cloud storage to read or write object data.
+
+The following table shows the supported types of storage for each component:
+
+| Component | AWS S3 and S3-compatible | Google Cloud Storage | Azure Blob Storage |
+|-----------|----|----------------------|--------------------|
+| [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) | ✅ | ✅ | ✅ |
+| [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/) | ✅ | ✅ | ✅ |
+| [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) | ✅ | ✅ | ✅ |
+| [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) |  ✅ | ✅ | ❌ |
+
+All these components use the same underlying libraries, so the authentication setup is largely the same. The main difference is in the command-line flags: 
+
+- vmalert uses `-s3.*` prefixed flags (e.g., `-s3.credsFilePath`)
+- backup and restore tools use unprefixed flags (e.g., `-credsFilePath`)
+
+See the [component reference](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/#per-component-flag-reference) for details.
+
+## Obtaining credentials
+
+You need to supply credentials so the component can connect to the cloud storage. The setup differs by provider; the sections below cover AWS S3, S3‑compatible systems, GCS, and Azure Blob Storage.
+
+### AWS S3
+
+1. In AWS, [create an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) or role with the minimum permissions for the target bucket (see table below).
+1. [Create an access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) for that IAM identity.
+1. Copy the **Access key ID** and **Secret access key** values. You will use them in the credentials file or environment variables.
+
+| Component       | S3 usage                                                    | Minimum S3 permissions |
+|-----------------|-------------------------------------------------------------|------------------------|
+| vmalert (Enterprise) | Reads alerting/recording rules from bucket.    | `s3:GetObject`, `s3:ListBucket` on the rules bucket/prefix.|
+| vmrestore       | Restores backups from S3 buckets                            | `s3:GetObject`, `s3:ListBucket` on the backup bucket/prefix.|
+| vmbackup        | Uploads backups to S3 and may delete old backup objects during maintenance.| `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject` on the backup bucket/prefix.|
+| vmbackupmanager (Enterprise) | Automates backups using vmbackup behavior.                  | Same as vmbackup. |
+
+
+### S3-compatible storage (MinIO, Ceph)
+
+Generate access keys using your storage system's admin interface or CLI. The credentials follow the same format as AWS S3.
+
+### Google Cloud Storage
+
+1. Open the Google Cloud Console and go to **IAM & Admin > Service Accounts**.
+1. Click **Create service account**, enter a name, and assign a Storage role (for example, Storage Object Admin).
+1. Open the service account, go to **Keys**, then click **Add key > Create new key**.
+1. Choose **JSON** as the key type and click **Create**.
+1. Store the JSON file on the machine running the VictoriaMetrics component.
+
+### Azure Blob Storage
+
+> Azure does not use credential files.
+
+1. Log in to the Azure Portal.
+1. Search for and select **Storage accounts**.
+1. Click on your specific storage account name. (This is your `AZURE_STORAGE_ACCOUNT_NAME`).
+1. In the left menu under **Security + networking**, click **Access keys**.
+1. Copy the key value from either **key1** or **key2** (this is your `AZURE_STORAGE_ACCOUNT_KEY`).
+1. Define the access keys as environment variables.
+    ```sh
+    export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
+    export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
+    ```
+
+## Authenticating with the cloud provider
+
+Provide the credentials as a file or with environment variables, along with the path to the cloud storage bucket. The syntax for the bucket name depends on the cloud provider:
+
+- `s3://`: for AWS S3 and self-hosted S3-compatible storage (MinIO, Ceph)
+- `gs://`: Google Cloud Storage
+- `azblob://`: Azure Blob Storage
+
+### vmbackup and vmrestore
+
+The following example backups to an AWS S3 bucket using a credentials file:
+
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=s3://victoriametrics-backup/backup01 \
+  -credsFilePath=/etc/credentials
+```
+
+In order to restore from the same backup from AWS S3:
+
+```sh
+vmrestore \
+  -src=s3://victoriametrics-backup/backup01 \
+  -storageDataPath=/data \
+  -credsFilePath=/etc/credentials
+
+```
+
+> To use non-AWS S3 buckets such as MinIO or Ceph, you must [supply the `-customS3Endpoint` argument](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/#s3-compatible).
+
+Alternatively, you can set the access keys as environment variables instead of using a credential file:
+
+```sh
+export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
+
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=s3://victoriametrics-backup/backup01
+
+vmrestore \
+  -src=s3://victoriametrics-backup/backup01 \
+  -storageDataPath=/data
+```
+
+
+Backups on Google Cloud Storage use the `gs://` prefix in the destination:
+
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=gs://victoriametrics-backup/backup01 \
+  -credsFilePath=/etc/credentials
+```
+
+You can restore this backup with:
+
+```sh
+vmrestore \
+  -src=gs://victoriametrics-backup/backup01 \
+  -storageDataPath=/data \
+  -credsFilePath=/etc/credentials
+```
+
+On Google Cloud, you can define the path to the JSON credential file with `GOOGLE_APPLICATION_CREDENTIALS`. For example:
+
+```sh
+export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
+
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=gs://victoriametrics-backup/backup01
+
+vmrestore \
+  -src=gs://victoriametrics-backup/backup01 \
+  -storageDataPath=/data
+```
+
+For Azure Blob Storage, use the `azblob://` prefix and rely on environment variables instead of `-credsFilePath`.
+
+```sh
+export AZURE_STORAGE_ACCOUNT_NAME=myaccount
+export AZURE_STORAGE_ACCOUNT_KEY=mykey
+
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=azblob://victoriametrics-backup/backup01
+
+vmrestore \
+  -src=azblob://victoriametrics-backup/backup01 \
+  -storageDataPath=/data
+```
+
+### vmbackupmanager 
+
+> vmbackupmanager only works in the [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) edition.
+
+To manage backups with vmbackupmanager on AWS S3, add the credentials with the `-credsFilePath` flag:
+
+```sh
+vmbackupmanager \
+  -dst=s3://vmstorage-data/backups \
+  -credsFilePath=/etc/credentials \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+Or define the access keys using environment variables:
+
+```sh
+export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
+
+vmbackupmanager \
+  -dst=s3://vmstorage-data/backups \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+> To use non-AWS S3 buckets, you must [supply the `-customS3Endpoint` argument](#s3-compatible).
+
+Automated backups on Google Cloud Storage take the following form:
+
+```sh
+vmbackupmanager \
+  -dst=gs://vmstorage-data/backups \
+  -credsFilePath=/etc/credentials \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+As with vmbackup and vmrestore, you can also define the path to the JSON credential file with `GOOGLE_APPLICATION_CREDENTIALS`:
+
+```sh
+export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
+
+vmbackupmanager \
+  -dst=gs://vmstorage-data/backups \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+vmbackupmanager can also use Azure Blob Storage by defining environment variables:
+
+```sh
+export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
+export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
+
+vmbackupmanager \
+  -dst=azblob://vmstorage-data/backups \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+### vmalert
+
+> - vmalert cloud storage command line flags are prefixed with `-s3.` for S3 buckets *and* Google Cloud Storage.
+> - Cloud storage only works in the [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) edition.
+
+Read alerting rules from an S3 bucket. The `-rule` flag accepts a prefix, so it matches all files starting with `alerts_` in the `rules` folder:
+
+```sh
+vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+Instead of a credential file, you can supply the access keys using environment variables:
+
+```sh
+export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
+
+vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+> To use non-AWS S3 buckets, you must [supply the `-s3.customEndpoint` argument](#s3-compatible).
+
+To read rules from Google Cloud Storage:
+
+```sh
+vmalert \
+  -rule=gs://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/gcp-service-account.json \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+If you prefer, you can supply the path to the JSON credential file with the `GOOGLE_APPLICATION_CREDENTIALS` environment variable:
+
+```sh
+export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
+
+vmalert \
+  -rule=gs://my-alert-bucket/rules/alerts_ \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+## Credentials files format
+
+The file format depends on the storage provider.
+
+### S3 credentials
+
+The file uses the standard AWS shared credentials format used by the [AWS CLI](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) and [AWS SDKs](https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html):
+
+```ini
+[default]
+aws_access_key_id=YOUR_AWS_ACCESS_KEY
+aws_secret_access_key=YOUR_AWS_SECRET_ACCESS_KEY
+```
+
+You can define multiple profiles in a single file:
+
+```ini
+[default]
+aws_access_key_id=DEFAULT_ACCESS_KEY
+aws_secret_access_key=DEFAULT_SECRET_KEY
+
+[monitoring]
+aws_access_key_id=MONITORING_ACCESS_KEY
+aws_secret_access_key=MONITORING_SECRET_KEY
+
+[alerts]
+aws_access_key_id=ALERTS_ACCESS_KEY
+aws_secret_access_key=ALERTS_SECRET_KEY
+```
+
+Use the `-configProfile` flag (or `-s3.configProfile` in vmalert) to select a non-default profile:
+
+```sh
+-configProfile=alerts
+```
+
+You can separate credentials from other configuration settings. Put credentials in one file:
+
+```ini
+[default]
+aws_access_key_id=DEFAULT_ACCESS_KEY
+aws_secret_access_key=DEFAULT_SECRET_KEY
+```
+
+And non-sensitive settings in another:
+
+```ini
+[default]
+region=us-east-1
+```
+
+Then pass both files:
+
+```sh
+-configFilePath=/etc/aws-config \
+-credsFilePath=/etc/credentials
+```
+
+### GCS credentials file format
+
+The file is the JSON key downloaded from Google Cloud Console. Its content looks like this:
+
+```json
+{
+  "type": "service_account",
+  "project_id": "project-id",
+  "private_key_id": "key-id",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+  "client_email": "service-account-email",
+  "client_id": "client-id",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+}
+```
+
+This is the standard service account key format defined by [Google Cloud IAM](https://developers.google.com/workspace/guides/create-credentials).
+
+### Azure Blob Storage
+
+Azure does not support credentials via file. Use environment variables instead.
+
+```sh
+export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
+export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
+```
+
+## Self-hosted and S3-compatible endpoints
+
+For S3-compatible storage such as MinIO or Ceph, set a custom endpoint with the `-customS3Endpoint` flag  for vmbackup, vmrestore, and vmbackupmanager. For example:
+
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=s3://victoriametrics-backup/backup01 \
+  -customS3Endpoint=http://minio.example.local:9000
+```
+
+On vmalert, use the `-s3.customEndpoint` flag instead:
+
+```sh
+vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -s3.customEndpoint=http://minio.example.local:9000 \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+### Addressing S3-compatible buckets {#s3-compatible}
+
+When connecting to non-AWS S3-compatible buckets, there is an additional flag you might need to configure:
+
+- `-s3ForcePathStyle`: on vmbackupmanager, vmbackup, and vmrestore.
+- `-s3.forcePathStyle`: on vmalert.
+
+The flag changes the expected URL pattern for a bucket.
+
+| Flag value | Address-style | Example | Use with |
+|------------|---------------|---------|----------|
+| `true` (default) | Path-style | `https://endpoint/bucket/key` |  MinIO, Ceph, most S3-compatible storages |
+| `false`        | Virtual host-style | `https://bucket.endpoint/key` | [Aliyun OSS](https://www.aliyun.com/product/oss) and other endpoints that require it |
+
+> The flag only takes effect when you use a custom endpoint (`-customS3Endpoint` or `-s3.customEndpoint` on vmalert). When connecting to real AWS S3, the SDK handles addressing automatically.
+
+## Per-component flag reference
+
+The table below shows how the same concept maps to different flag names across components.
+
+| Concept | vmalert | vmbackup, vmrestore, and vmbackupmanager |
+|---|---|---|---|---|
+| Credentials file | `-s3.credsFilePath` | `-credsFilePath` |
+| Config file | `-s3.configFilePath` | `-configFilePath` |
+| Profile selection | `-s3.configProfile` | `-configProfile` |
+| Custom endpoint | `-s3.customEndpoint` | `-customS3Endpoint` |
+| Force path style | `-s3.forcePathStyle` | `-s3ForcePathStyle` |
+| TLS insecure | N/A | `-s3TLSInsecureSkipVerify` |
+| Storage class | N/A | `-s3StorageClass` |
+
+
+

docs/guides/connecting-vm-components-to-cloud-storage/_index.md

@@ -0,0 +1,12 @@
+---
+weight: 5
+title: Connecting VictoriaMetrics components to cloud storage
+menu:
+  docs:
+    parent: guides
+    weight: 5
+tags:
+  - metrics
+  - guide
+---
+{{% content "README.md" %}}

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -240,23 +240,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.148.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.148.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.148.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -308,7 +308,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.146.0
+    image: docker.io/victoriametrics/vmauth:v1.148.0
     ports:
       - 8427:8427
     volumes:

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.148.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.148.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.148.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.146.0-enterprise
+    image: victoriametrics/vmauth:v1.148.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - victoriametrics
       - alertmanager

docs/victoriametrics/Articles.md

@@ -114,6 +114,8 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [QCon London 2026: Wrangling Telemetry at Scale, a Guide to Self-Hosted Observability](https://www.infoq.com/news/2026/03/self-hosted-observability/)
 * [How We Made Telemetry Queries 10x Faster: Chunk-Split Caching for Metrics, Logs, and Traces](https://mirastacklabs.ai/blog/chunk-split-caching/)
 * [Claude Code: creating Kubernetes debugging AI Agent for VictoriaMetrics](https://rtfm.co.ua/en/claude-code-creating-kubernetes-debugging-ai-agent-for-victoriametrics/)
+* [OpenTelemetry: OTel Collectors in Kubernetes and VictoriaMetrics Stack integration](https://itnext.io/opentelemetry-otel-collectors-in-kubernetes-and-victoriametrics-stack-integration-d907ed0a15a0)
+* [VictoriaMetrics vs Prometheus: my default, and when I still pick Prometheus](https://jorijn.com/en/blog/victoriametrics-vs-prometheus/)
 
 ## Third-party articles and slides about VictoriaLogs
 

docs/victoriametrics/FAQ.md

@@ -338,13 +338,15 @@ File bugs and feature requests in our [GitHub Issues](https://github.com/Victori
 ## Where can I find information about multi-tenancy?
 
 See [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).
-Multitenancy is supported only by the [cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) of VictoriaMetrics.
+Multitenancy is fully supported only by the [cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) of VictoriaMetrics.
+Single-node provides limited [multitenancy](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy) support to facilitate
+the migration [from single-node to cluster](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#from-single-node-to-cluster).
 
 ## How to set a memory limit for VictoriaMetrics components?
 
 All VictoriaMetrics components provide command-line flags to control the size of internal buffers and caches:
 `-memory.allowedPercent` and `-memory.allowedBytes` (pass `-help` to any VictoriaMetrics component in order to see the description for these flags).
-These limits don't take into account additional memory, which may be needed for processing incoming queries.
+These limits don't account for additional memory that may be needed to process incoming queries.
 Hard limits may be enforced only by the OS via [cgroups](https://en.wikipedia.org/wiki/Cgroups),
 Docker (see [these docs](https://docs.docker.com/config/containers/resource_constraints)) or
 Kubernetes (see [these docs](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers)).
@@ -465,9 +467,13 @@ Cluster version of VictoriaMetrics may be preferred over single-node VictoriaMet
 
 ## How to migrate data from single-node VictoriaMetrics to cluster version?
 
-The single-node version of VictoriaMetrics stores data on disk in slightly different format compared to the cluster version of VictoriaMetrics.
-This makes it impossible to just copy the on-disk data from `-storageDataPath` directory from single-node VictoriaMetrics to a `vmstorage` node in VictoriaMetrics cluster.
-If you need to migrate data from a single-node VictoriaMetrics to the cluster version, then [follow these instructions](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
+The single-node version of VictoriaMetrics stores data on disk in a slightly different format compared to the cluster version of VictoriaMetrics.
+This makes it impossible to just copy the on-disk data from the `-storageDataPath` directory from single-node VictoriaMetrics to a `vmstorage` node in a VictoriaMetrics cluster.
+
+There are two options, however:
+
+1. Deploy a new cluster next to the existing single-node and let them co-exist until the cluster is filled with new data and the old data in vmsingle becomes outside of the retention period. This option requires no data migration or downtime. See instructions [here](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#from-single-node-to-cluster).
+2. If you need to actually migrate data from a single-node VictoriaMetrics to the cluster version (and/or possibly modify it), then follow [these vmctl instructions](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
 ## Why isn't MetricsQL 100% compatible with PromQL?
 

docs/victoriametrics/LTS-releases.md

@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
 
 ## Currently supported LTS release lines
 
-- v1.136.x - the latest one is [v1.136.4 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.4)
-- v1.122.x - the latest one is [v1.122.19 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.19)
+- v1.148.x - the latest one is [v1.148.0 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
+- v1.136.x - the latest one is [v1.136.14 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.14)

docs/victoriametrics/Quick-Start.md

@@ -14,6 +14,178 @@ aliases:
 - /quick-start/index.html
 - /quick-start/
 ---
+There are two ways to get started with VictoriaMetrics:
+
+* [Try it locally](https://docs.victoriametrics.com/victoriametrics/quick-start/#try-it-locally) - if you just want to see how VictoriaMetrics works,
+  go with the single binary: download it, start it with one command, and see live metrics
+  in the built-in UI in a couple of minutes. No Docker, no configuration files
+  and no extra components are required;
+* [Install it](https://docs.victoriametrics.com/victoriametrics/quick-start/#how-to-install) - if you want to set up VictoriaMetrics for real use,
+  pick a distribution (single-node, cluster or cloud) and an installation method
+  (Docker, Helm, binary releases, etc.).
+
+If you'd rather not install anything at all, try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) -
+a list of publicly available playgrounds for VictoriaMetrics software.
+
+Whichever way you choose, you may also find interesting the other sections of this page,
+like how to [write](https://docs.victoriametrics.com/victoriametrics/quick-start/#write-data) and [query](https://docs.victoriametrics.com/victoriametrics/quick-start/#query-data) data,
+[alerting](https://docs.victoriametrics.com/victoriametrics/quick-start/#alerting),
+[data migration](https://docs.victoriametrics.com/victoriametrics/quick-start/#data-migration) from other TSDBs,
+and [productionization](https://docs.victoriametrics.com/victoriametrics/quick-start/#productionization)
+best practices for running VictoriaMetrics in production.
+
+## Try it locally
+
+The fastest way to try VictoriaMetrics on your own machine is its binary - the only thing needed to run it.
+
+### Step 1: Download the binary
+
+Create a directory for this test drive, so all the files created along the way stay in one place:
+
+```sh
+mkdir vm-quick-start && cd vm-quick-start
+```
+
+Download the `victoria-metrics-<os>-<arch>-<version>.tar.gz` archive for your OS and architecture
+from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+and unpack it. It contains a single `victoria-metrics-prod` binary.
+
+For example, on Linux with `amd64` architecture:
+
+```sh
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.148.0.tar.gz
+```
+
+The binary is self-contained and requires no installation - it is ready to run as is.
+
+### Step 2: Start VictoriaMetrics
+
+Starting VictoriaMetrics is as simple as executing the binary, with no arguments at all.
+But since it is nicer to have some data to explore right after the start, let's also enable self-scraping
+via the `-selfScrapeInterval` command-line flag, so VictoriaMetrics collects metrics about itself:
+
+```sh
+./victoria-metrics-prod -selfScrapeInterval=10s
+```
+
+VictoriaMetrics prints a couple of dozen log lines on start, describing the storage, caches
+and memory limits it sets up. Look for these two lines confirming that it is up and running:
+
+```sh
+2026-07-10T16:55:06.615Z    info    app/victoria-metrics/main.go:102    started VictoriaMetrics in 0.019 seconds
+...
+2026-07-10T16:55:06.615Z    info    lib/httpserver/httpserver.go:148    started server at http://0.0.0.0:8428/
+```
+
+That's it - VictoriaMetrics is running, listening on port `8428` and scraping its own metrics
+(CPU and memory usage, the number of stored metrics, request rates, etc.) every 10 seconds.
+If you list the `vm-quick-start` directory, you can see a new `victoria-metrics-data` directory
+created next to the binary - this is where the collected data is stored.
+
+Now that metrics are being collected, it's time to look at them.
+
+### Step 3: Explore the metrics
+
+Open [http://localhost:8428/vmui](http://localhost:8428/vmui) in your browser to access [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) -
+the built-in UI for querying and graphing metrics. You should see its query page:
+
+![vmui query page](quick-start-vmui.webp)
+
+Self-scraped metrics become queryable within ~30 seconds after the start.
+
+Try the following:
+
+* Open the [metrics explorer](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#metrics-explorer)
+  at [http://localhost:8428/vmui/#/metrics](http://localhost:8428/vmui/#/metrics) to browse all collected metrics;
+* Enter a query in the input field at [http://localhost:8428/vmui](http://localhost:8428/vmui) and press `Enter`. For example:
+  * `process_resident_memory_bytes` - memory used by VictoriaMetrics;
+  * `rate(process_cpu_seconds_total)` - its CPU usage;
+  * `vm_rows{type=~"storage/.+"}` - the number of stored [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples).
+
+Queries are written in [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) -
+a PromQL-compatible query language, so any PromQL query works here as well.
+
+So far the only available metrics are the ones VictoriaMetrics reports about itself - let's collect something more interesting.
+
+### Step 4 (optional): Collect metrics from your system
+
+Self-scraped metrics only describe VictoriaMetrics itself. To monitor your machine (CPU, memory, disk, network),
+run [node_exporter](https://github.com/prometheus/node_exporter) - the standard Prometheus exporter for host metrics -
+and let VictoriaMetrics scrape it. Single-node VictoriaMetrics has a built-in
+[Prometheus-compatible scraper](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter),
+so no other components are needed.
+
+1. Download the `node_exporter-<version>.<os>-<arch>.tar.gz` archive for your OS and architecture
+   from the [releases page](https://github.com/prometheus/node_exporter/releases/latest) and unpack it.
+   For example, on Linux with `amd64` architecture:
+
+   ```sh
+   wget https://github.com/prometheus/node_exporter/releases/download/v1.12.0/node_exporter-1.12.0.linux-amd64.tar.gz
+   tar xzf node_exporter-1.12.0.linux-amd64.tar.gz
+   ```
+
+   Unlike VictoriaMetrics, it unpacks into its own directory. Start it from there:
+
+   ```sh
+   ./node_exporter-1.12.0.linux-amd64/node_exporter
+   ```
+
+   It exposes host metrics at [http://localhost:9100/metrics](http://localhost:9100/metrics).
+
+1. Create a `scrape.yaml` file with the following contents:
+
+   ```yaml
+   scrape_configs:
+   - job_name: node-exporter
+     static_configs:
+     - targets:
+       - localhost:9100
+   ```
+
+1. Restart VictoriaMetrics with the `-promscrape.config` command-line flag pointing to this file:
+
+   ```sh
+   ./victoria-metrics-prod -selfScrapeInterval=10s -promscrape.config=scrape.yaml
+   ```
+
+Check [http://localhost:8428/targets](http://localhost:8428/targets) - the `node-exporter` target should have `state: up`.
+The target shows up as `state: down` until the first scrape happens, which can take some seconds -
+just reload the page a bit later.
+Now query host metrics in [vmui](http://localhost:8428/vmui). For example:
+
+* `node_memory_MemAvailable_bytes` - available memory on your machine;
+* `100 - avg(rate(node_cpu_seconds_total{mode="idle"})) * 100` - overall CPU usage in percent.
+
+See [scrape config examples](https://docs.victoriametrics.com/victoriametrics/scrape_config_examples/) for more advanced scrape configurations.
+
+Scraping pulls metrics from targets, but it is not the only way to get data in - metrics can also be pushed directly.
+
+### Step 5 (optional): Push your own metrics
+
+VictoriaMetrics also accepts metrics pushed via [many popular protocols](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#push-model).
+For example, insert a measurement using the InfluxDB line protocol with a plain `curl` command:
+
+```sh
+curl -d 'room_temperature,room=kitchen value=21.5' http://localhost:8428/write
+```
+
+Then query `room_temperature_value` in [vmui](http://localhost:8428/vmui) to see it.
+Take into account that freshly ingested data can take up to 30 seconds to show up in query results,
+so don't worry if it doesn't appear immediately - just retry the query a bit later.
+
+Once you're done experimenting, tidying everything up takes a single command.
+
+### Cleanup
+
+Stop VictoriaMetrics and node_exporter with `Ctrl+C` in their respective terminals. All the collected data lives in the `victoria-metrics-data` directory -
+delete it if you want to start from scratch. To remove all traces of this test drive,
+delete the whole `vm-quick-start` directory created at [step 1](https://docs.victoriametrics.com/victoriametrics/quick-start/#step-1-download-the-binary).
+
+This test drive only scratches the surface of what VictoriaMetrics can do. Ready for a real setup?
+Continue with the installation options below, or learn the [key concepts](https://docs.victoriametrics.com/victoriametrics/keyconcepts/)
+of writing and querying data first.
+
 ## How to install
 
 VictoriaMetrics is available in the following distributions:
@@ -42,9 +214,6 @@ Just download VictoriaMetrics and follow [these instructions](https://docs.victo
 See [available integrations](https://docs.victoriametrics.com/victoriametrics/integrations/) with other systems like
 [Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/) or [Grafana](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/).
 
-> Want to see VictoriaMetrics in action, but without installing anything?
-> Try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) - a list of publicly available playgrounds for VictoriaMetrics software.
-
 VictoriaMetrics is developed at a fast pace, so it is recommended to periodically check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/)
 and perform [regular upgrades](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
 
@@ -60,9 +229,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.146.0
+docker pull victoriametrics/victoria-metrics:v1.148.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.146.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.148.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
@@ -465,6 +634,15 @@ It is recommended to read [Replication and data safety](https://docs.victoriamet
 
 For backup configuration, please refer to [vmbackup documentation](https://docs.victoriametrics.com/victoriametrics/vmbackup/).
 
+### Graceful shutdown
+
+To gracefully shut down a VictoriaMetrics process, send SIGTERM or SIGINT signal and wait until the process exits.
+See [how to send signals to processes](https://stackoverflow.com/questions/33239959/send-signal-to-process-from-command-line).
+
+Graceful shutdown is required for data safety. A successful graceful shutdown guarantees that pending in-memory data and 
+ongoing writes are flushed on disk before the process exits. During graceful shutdown, VictoriaMetrics stops accepting
+new HTTP connections and waits for in-flight requests to finish until `-http.maxGracefulShutdownDuration` expires.
+
 ### Configuring limits
 
 To avoid excessive resource usage or performance degradation, limits must be in place:

docs/victoriametrics/README.md

@@ -403,6 +403,9 @@ Resources:
 * [Cardinality explorer blog post](https://victoriametrics.com/blog/cardinality-explorer/).
 * [skills/victoriametrics-cardinality-analysis](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/victoriametrics-cardinality-analysis/SKILL.md) for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
 
+For monitoring or alerting on cardinality, use [vmestimator](https://docs.victoriametrics.com/victoriametrics/vmestimator/).
+vmestimator measures metrics cardinality across [arbitrary label dimensions](https://docs.victoriametrics.com/victoriametrics/vmestimator/#basic) in real time and exposes the [results as metrics](https://docs.victoriametrics.com/victoriametrics/vmestimator/#cardinality-metrics).
+
 ### Cardinality explorer statistic inaccuracy
 
 In [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) each vmstorage tracks the stored time series individually.
@@ -1548,6 +1551,18 @@ For example, the following command starts VictoriaMetrics, which accepts samples
 /path/to/victoria-metrics -futureRetention=1y
 ```
 
+By default, VictoriaMetrics accepts samples with timestamps as old as the configured `-retentionPeriod` allows, e.g. it accepts backfilled
+historical data as long as it fits into the retention. If you need rejecting samples with historical timestamps older than the specified
+duration, then specify the desired duration via the `-maxBackfillAge` command-line flag. This can be useful for limiting ingestion of
+historical samples, for example, when older data has been moved to another storage tier (nvme/hdd, hot/cold).
+`-maxBackfillAge` cannot exceed the configured `-retentionPeriod` - bigger values are automatically clamped to `-retentionPeriod`.
+
+For example, the following command starts VictoriaMetrics, which rejects ingested samples with timestamps older than 2 days:
+
+```sh
+/path/to/victoria-metrics -maxBackfillAge=2d
+```
+
 ### Multiple retentions
 
 Distinct retentions for distinct time series can be configured via [retention filters](#retention-filters)
@@ -1682,9 +1697,46 @@ See also [retention filters](#retention-filters).
 The downsampling can be evaluated for free by downloading and using enterprise binaries from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
 See how to request a [free trial license](https://victoriametrics.com/products/enterprise/trial/).
 
-## Multi-tenancy
+## Multitenancy {#multi-tenancy}
 
-Single-node VictoriaMetrics doesn't support multi-tenancy. Use the [cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) instead.
+Single-node VictoriaMetrics has limited
+[multitenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
+support {{% available_from "v1.147.0" %}}. Specifically, a single-node can serve
+multitenant queries as if it were a `vmstorage`. The write path is not supported.
+
+The functionality is disabled by default and can be enabled by setting the
+`-vmselectAddr` flag. This will start the `vmselect RPC server` that accepts
+requests and serves responses in cluster format.
+
+Cluster data format assumes the presence of a `tenantID`. Single-node data
+format still does not support multitenancy, but it is possible to configure the
+single-node to specify which `tenantID` the single-node's data corresponds to with the
+`-accountID` and `-projectID` flags. Both are `0` by default, which means that
+`"0:0"` `tenantID` is used by default.
+
+For example, the following command will start a single-node that listens for
+vmselect RPC requests on the `8401` port. The requests must be either `multitenant`
+(i.e., want data for all tenants) or for `"12:34"` tenant. Otherwise, the
+single-node will return an empty result:
+
+```shell
+./victoria-metrics -storageDataPath=/data -vmselectAddr=:8401 -accountID=12 -projectID=34
+```
+
+The `tenantID` configuration is not persisted in any way and is enforced only at
+runtime. Thus, it is safe to change the `-accountID` and `-projectID` flag
+values at any time.
+
+Note that the single-node's HTTP handlers still do not support multitenancy.
+
+The purpose of this limited multitenancy support is enabling the single-node to
+operate in VictoriaMetrics cluster setups. I.e., one or more single-nodes that
+contain data for different tenants can be a part of a cluster, and the entire
+non-homogeneous deployment can be queried with a higher-level `vmselect`.
+
+This, in turn, enables easy [migrations from single-node to cluster](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#from-single-node-to-cluster).
+Previously, the only option was the use of
+[vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
 ## Scalability and cluster version
 
@@ -2294,6 +2346,30 @@ Things to consider when copying data:
 For scenarios like single-to-cluster, cluster-to-single, re-sharding or migrating only a fraction of data: 
 [see how to migrate data from VictoriaMetrics via vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
+### From Single-node to Cluster
+
+When, for some reason, the deployment needs to be switched from single-node to
+cluster (such as the single-node can't be scaled vertically anymore, or
+multitenancy becomes a requirement, etc.) the migration can be as simple as:
+
+1. Restart the existing single-node with multitenancy support enabled as
+   described in [Multitenancy](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy) section.
+1. Deploy an empty cluster next to the existing single-node.
+1. Deploy higher-level `vmselect` and configure it to query both the existing
+   single-node and the new cluster.
+1. Start writing data to the cluster.
+1. Stop writing data to the single-node.
+
+This approach requires no data migration nor downtime (apart from restarting the
+single-node at step 1). And once the single-node data becomes outside the
+retention period, the single-node can be removed from the deployment.
+
+Note that if you need to actually migrate data to cluster and/or modify it, you
+will need to use
+[vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/)
+instead.
+
+
 ### From other systems
 
 Use [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/) to migrate data from other systems to VictoriaMetrics.

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,9 +26,38 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
+
+## [v1.148.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
+
+Released at 2026-07-20
+
+* SECURITY: upgrade Go builder from Go1.26.4 to Go1.26.5. See [the list of issues addressed in Go1.26.5](https://github.com/golang/go/issues?q=milestone%3AGo1.26.5%20label%3ACherryPickApproved).
+
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support `fill` modifiers to allow missing series on either side of a binary operation to be filled with a provided default value. See [#10598](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10598).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): support scraping metrics over Unix domain sockets. The socket path can be configured via the `__unix_socket__` target label. See [#11156](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11156). Thanks to @vinyas-bharadwaj for contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Improve background discovery performance for [http_sd](https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs) discovery. See [#8838](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8838).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add client side least-loaded load-balancing with `DNS` discovery. See [#2388](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2388) and these [vmagent DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmagent/#dns-urls), [vmalert DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmalert/#dns-urls).
+* FEATURE: [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-maxBackfillAge` command-line flag for limiting ingestion of samples with historical timestamps, for example, when older data has been moved between storage tiers (nvme/hdd, hot/cold). See [#11199](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11199). Thanks to @AshwinRamaniPsg for contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): automatically preload relabeling rules configured via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` in the [metrics relabel debug UI](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabel-debugging). See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918).
+
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Now drops metadata blocks when communicating with vmstorage nodes over the legacy RPC protocol. To avoid this limitation, upgrade `vmstorage` to a version that supports the new RPC protocol (>= [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11370)). See [#11146](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11146).
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry S3 requests failing with `HTTP 429` status code or `TooManyRequests` error code. Previously such requests were not retried, so a short burst of rate limiting would fail the whole backup. See [#11218](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218). Thanks to @gautamrizwani for contribution.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply limit to metrics metadata response. See [#11139](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11139).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a possible data race when processing OpenTelemetry metadata. See [#11238](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11238). Thanks to @nevgeny for contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): flush pending persistent queue data to chunk file before updating the metadata. This prevents the metadata writer offset from getting ahead of the chunk file size and avoids losing the persistent queue after an unclean shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): atomically write persistent queue metainfo to prevent possible file corruption on ungraceful shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix increased CPU and memory usage when `-remoteWrite.urlRelabelConfig` or `-remoteWrite.streamAggr.config` flags are used. The bug was introduced in [#10854](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10854) and existed since [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0). See [#11250](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11250).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve newline formatting in alert and rule annotations on the Alerting page. See [#11171](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11171).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): return `408 Request Timeout` instead of `400 Bad Request` when the request body isn't received within `-maxQueueDuration`. This prevents `vmagent` from incorrectly downgrading the remote write protocol and dropping data when `vmauth` is used as a proxy for a remote write endpoint. See [#11272](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11272).
 ## [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0)
 
-Release candidate
+Released at 2026-07-06
+
+**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): contains a bug that causes increased CPU and memory usage when `-remoteWrite.urlRelabelConfig` or `-remoteWrite.streamAggr.config` flags are used. The bug was introduced in [#10854](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10854). Upgrade to v1.148.0 or rollback to v1.146.0. See [#11250](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11250).
 
 * SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
 
@@ -36,15 +65,17 @@ Release candidate
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduces CPU usage by 10% at [sharding among remote storages](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages). See [#11113](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11113). Thanks to @bennf for contribution.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce `64KiB` size limit for `metric metadata` fields - `Unit`, `Help` and `MetricFamilyName`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `vm_data_size_bytes{type="storage/metaindex"}` and `vm_data_size_bytes{type="indexdb/metaindex"}` metrics for tracking memory occupied by metaindex data. See [#11204](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11204). Thanks to @SamarthBagga for contribution.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
-* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [#4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328), [#10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926), and the [documentation](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy).
 * FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
-* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow log requests with missing or invalid auth tokens to [access log](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). This is useful for identifying `remote_addr` IPs performing brute-force attacks. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fall through to `unauthorized_user` when a [JWT token](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) has no `vm_access` claim and no `default_vm_access_claim` is configured. Previously, vmauth returned `401 Unauthorized` immediately in this case, which prevented `unauthorized_user` from handling such requests. See [#5740](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5740).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): improve the selection algorithm of [buckets_limit](https://docs.victoriametrics.com/victoriametrics/metricsql/#buckets_limit) to remove consecutive empty buckets at the beginning and end to obtain more accurate min and max values. See [#10417](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_group_rule_results_limit` metric to indicate the number of alerts or recording results that a single rule within the group can produce. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
-* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
 
 * BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
@@ -67,7 +98,6 @@ Released at 2026-06-22
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `-vm-headers` and `-vm-bearer-token` flags for authenticating requests to the VictoriaMetrics import destination. The flags are available in `opentsdb`, `influx`, `remote-read`, `prometheus`, `mimir`, and `thanos` vmctl sub-commands. See [#8897](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8897).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): log calls to [/api/v1/admin/tsdb/delete_series](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series) API handler. This should help to identify events of metrics deletion from the database. See [#11104](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11104).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the `last` value to graph legend statistics. See [#10759](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10759).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
 
 * BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly expose metric `vm_retention_filters_partitions_scheduled_rows`. See [#11138](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11138)
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808).
@@ -77,8 +107,8 @@ Released at 2026-06-22
 * BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): disallow restoring parts outside the configured `-storageDataPath` directory. See [710c920d](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/710c920d6083327042a309e449fae4383617d817).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly apply long tenant filters. Previously, such filters could be truncated, causing tenants to be matched incorrectly. See [#11096](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11096). Thanks for @fxrlv for the contribution.
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix corrupted metrics metadata when a response contains multiple rows. See [#11115](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11115). Thanks for @fxrlv for the contribution.
-* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8)
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): don't cache empty responses for tenant IDs discovery during [multitenant queries](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenant-reads). This problem was visible during integration tests when multitenant queries were executed before the first ingestion happened. See [#10982](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10982)
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8)
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
 
@@ -103,9 +133,9 @@ Released at 2026-06-08
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly calculate number of loaded users to be printed in startup log. Previously, it was only accounting for static users and skipped JWT configuration entries. See [#11050](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11050/).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): `integrate()` no longer extrapolates the last sample's value past the end of the time series. Previously, querying `integrate(metric[1h])` at a timestamp where the series had already ended would keep accruing area as if the last value continued indefinitely, producing values much larger than the true integral. See [#9474](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474). Thanks to @wtfashwin for contribution.
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid returning HTTP 503 for queries with partial results when a storage group is unavailable and `-search.denyPartialResponse` is disabled. See [#11009](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11009). Thanks to @fxrlv for the contribution.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `utf-8` label names for [/federate](https://docs.victoriametrics.com/victoriametrics/#federation) API requests. See [#10968](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10968).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the `Disable deduplication` toggle under its own local storage key. Before this fix, the toggle state was lost after reload and could overwrite the `Compact view` table setting. See [#11004](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11004). Thanks to @immanuwell for the contribution.
-* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
 
 ## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
 
@@ -113,28 +143,28 @@ Released at 2026-05-22
 
 * FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
 * FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Improve [slowness-based rerouting](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) to prevent rerouting storms under high cluster load. Previously, rerouting could cascade across storage nodes when the whole cluster was saturated, making the situation worse. Now rerouting only activates when the cluster p90 saturation is below 60%, and the slowest node is more than 20% slower than p90. See [#10876](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10876).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `{{.MetricsAccountID}}` and `{{.MetricsProjectID}}` [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) for use in `headers` and `url_prefix` config fields. Previously, only the combined `{{.MetricsTenant}}` (`accountID:projectID`) JWT placeholder was supported, making it impossible to configure [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers). See [#10927](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10927). Thanks to @Vinayak9769 for the contribution.
-* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive credentials.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): hide values passed to `-remoteWrite.headers`,`remoteRead.headers`, `datasource.headers` and `notifier.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly establish [mtls](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection) connection between vmstorage and vminsert. Regression was introduced in v1.130.0 release for the enterprise version of vmstorage. See [#10972](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10972).
 * BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
-* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
-* BUGFIX: all VictoriaMetrics components: fix int64 overflow when parsing [timestamp parameters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats) with relative durations. See [#10880](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
+* BUGFIX: all VictoriaMetrics components: fix int64 overflow when parsing [timestamp parameters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats) with relative durations. See [#10880](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880).
 
 ## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)
 
@@ -144,10 +174,10 @@ Released at 2026-05-08
 
 * FEATURE: all VictoriaMetrics components: suppress TCP health check errors when `-tls` flag is set. See [#10538](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10538).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `__meta_hetzner_robot_datacenter` label for `robot` role in [hetzner_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs). See [#10909](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10909). Thanks to @juliusrickert for contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-rule.stripFilePath` to support stripping rule file paths in logs and all API responses, including /metrics. See [#5625](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5625).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `formatTime` template function for formatting a Unix timestamp using the provided layout. For example, `{{ now | formatTime "2006-01-02T15:04:05Z07:00" }}` returns the current time in RFC3339 format. See issue [#10624](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10624). Thanks to @andriibeee for the contribution.
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
 * FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Kafka (Enterprise)` row with panels for monitoring traffic (bytes), messages in/out, producer and consumer errors. See [#10728](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10728).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support sending data to the configured `-remoteWrite.url` via [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol). See [#10929](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10929).
 
@@ -320,6 +350,27 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.14)
+
+Released at 2026-07-17
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* SECURITY: upgrade Go builder from Go1.26.4 to Go1.26.5. See [the list of issues addressed in Go1.26.5](https://github.com/golang/go/issues?q=milestone%3AGo1.26.5%20label%3ACherryPickApproved).
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): flush pending persistent queue data to chunk file before updating the metadata. This prevents the metadata writer offset from getting ahead of the chunk file size and avoids losing the persistent queue after an unclean shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a possible data race when processing OpenTelemetry metadata. See [#11238](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11238). Thanks to @nevgeny for contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): atomically write persistent queue metainfo to prevent possible file corruption on ungraceful shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry S3 requests failing with `HTTP 429` status code or `TooManyRequests` error code. Previously such requests were not retried, so a short burst of rate limiting would fail the whole backup. See [#11218](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218). Thanks to @gautamrizwani for contribution.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply limit to metrics metadata response. See [#11139](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11139).
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Now drops metadata blocks when communicating with vmstorage nodes over the legacy RPC protocol. To avoid this limitation, upgrade `vmstorage` to a version that supports the new RPC protocol (>= [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11370)). See [#11146](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11146).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve newline formatting in alert and rule annotations on the Alerting page. See [#11171](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11171).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): return `408 Request Timeout` instead of `400 Bad Request` when the request body isn't received within `-maxQueueDuration`. This prevents `vmagent` from incorrectly downgrading the remote write protocol and dropping data when `vmauth` is used as a proxy for a remote write endpoint. See [#11272](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11272).
+
 ## [v1.136.13](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.13)
 
 Released at 2026-07-03
@@ -718,6 +769,18 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.27](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.27)
+
+Released at 2026-07-17
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): flush pending persistent queue data to chunk file before updating the metadata. This prevents the metadata writer offset from getting ahead of the chunk file size and avoids losing the persistent queue after an unclean shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry S3 requests failing with `HTTP 429` status code or `TooManyRequests` error code. Previously such requests were not retried, so a short burst of rate limiting would fail the whole backup. See [#11218](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218). Thanks to @gautamrizwani for contribution.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
+
 ## [v1.122.26](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.26)
 
 Released at 2026-07-03

docs/victoriametrics/enterprise.md

@@ -121,7 +121,7 @@ It is allowed to run Enterprise components in [cases listed here](https://docs.v
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
 [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
 and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -139,8 +139,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.146.0/victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -155,7 +155,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.146.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.148.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
@@ -165,13 +165,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.146.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.148.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.146.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.148.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -181,7 +181,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
     ports:
       - 8428:8428
     volumes:
@@ -213,7 +213,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.148.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -224,7 +224,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.148.0-enterprise
 
 license:
   secret:
@@ -274,7 +274,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.148.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -291,7 +291,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.148.0-enterprise
 ```
 
 Example secret with license key:
@@ -342,7 +342,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz`
 
 Includes:
 
@@ -351,7 +351,7 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.146.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.148.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## What Happens to Licensed Components When a License Expires
 

docs/victoriametrics/relabeling.md

@@ -513,6 +513,7 @@ URLs for scrape targets are composed of the following parts:
     [scrape_config](https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs)
     or by updating/setting the corresponding `__param_*` labels during
     relabeling.
+- Unix domain socket path (e.g. `/var/run/node_exporter.sock`) can be configured during target relabeling via a special label - `__unix_socket__`. If this label is set, `vmagent` tunnels the scrape request through the specified Unix domain socket instead of connecting to a TCP socket. The `__address__` label is still used to populate the `instance` label and construct the HTTP request URL, but the actual network connection goes through the Unix socket.
 
 The resulting scrape URL looks like the following:
 

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.146.0/victoria-metrics-linux-amd64-v1.146.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.146.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.148.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.146.0/victoria-metrics-linux-amd64-v1.146.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.146.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.148.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/sd_configs.md

@@ -1923,6 +1923,9 @@ scrape_configs:
   # Example values:
   # - "10MiB" - 10 * 1024 * 1024 bytes
   # - "100MB" - 100 * 1000 * 1000 bytes
+  # The max_scrape_size can be set on a per-target basis by specifying `__max_scrape_size__`
+  # label during target relabeling phase.
+  # See https://docs.victoriametrics.com/victoriametrics/relabeling/
   #
   # max_scrape_size: <size>
 

docs/victoriametrics/stream-aggregation/README.md

@@ -645,9 +645,9 @@ See also [why you shouldn't put an aggregator behind a load balancer](https://do
 
 # Troubleshooting
 
-- [Unexpected spikes for `total` or `increase` outputs](#data-delay-and-staleness).
+- [Unexpected spikes for `total` or `increase` outputs](#staleness).
 - [Excessively large values for `total*`, `increase*`, and `rate*` outputs](#counter-resets).
-- [Lower than expected values for `total_prometheus` and `increase_prometheus` outputs](#data-delay-and-staleness).
+- [Lower than expected values for `total_prometheus` and `increase_prometheus` outputs](#staleness).
 - [High memory usage and CPU usage](#high-resource-usage).
 - [Unexpected results in vmagent cluster mode](#cluster-mode).
 - [Inaccurate aggregation results for histograms](#aggregation-windows)

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -13,6 +13,8 @@ victoria-metrics is a time series database and monitoring solution.
 
 See the docs at https://docs.victoriametrics.com/victoriametrics/
 
+  -accountID uint
+     The accountID of the stored data
   -bigMergeConcurrency int
      Deprecated: this flag does nothing
   -blockcache.missesBeforeCaching int
@@ -104,7 +106,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
@@ -186,6 +188,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxBackfillAge value
+     The maximum allowed age for the ingested samples with historical timestamps. Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention
+     The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 0)
   -maxConcurrentInserts int
      The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 2*cgroup.AvailableCPUs())
   -maxIngestionRate int
@@ -194,11 +199,11 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
   -maxLabelNameLen int
-     The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_name"} metric at /metrics page is incremented (default 256)
+     The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_name"} metric at /metrics page is incremented. Value must be in range 1..65535. (default 256)
   -maxLabelValueLen int
-     The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented (default 4096)
+     The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented. Value must be in range 1..65535. (default 4096)
   -maxLabelsPerTimeseries int
-     The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented (default 40)
+     The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented. (default 40)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -261,6 +266,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss (default 64)
   -prevCacheRemovalPercent float
      Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1)
+  -projectID uint
+     The projectID of the stored data
   -promscrape.azureSDCheckInterval duration
      Interval for checking for changes in Azure. This works only if azure_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#azure_sd_configs for details (default 1m0s)
   -promscrape.cluster.memberLabel string
@@ -401,6 +408,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 1M)
   -retentionTimezoneOffset duration
      The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
+  -rpc.disableCompression
+     Whether to disable compression of the data sent from vmstorage to vmselect. This reduces CPU usage at the cost of higher network bandwidth usage
+  -rpc.handshakeTimeout duration
+     Timeout for RPC handshake between vminsert/vmselect and vmstorage. Increase this value if transient handshake failures occur. See https://docs.victoriametrics.com/victoriametrics/troubleshooting/#cluster-instability section for more details. (default 5s)
   -search.cacheTimestampOffset duration
      The maximum duration since the current time for response data, which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset (default 5m0s)
   -search.disableAutoCacheReset
@@ -628,6 +639,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Show VictoriaMetrics version
   -vmalert.proxyURL string
      Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules . See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert
+  -vmselectAddr string
+     TCP address to accept connections from vmselect services
   -vmui.customDashboardsPath string
      Optional path to vmui dashboards. See https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards
   -vmui.defaultTimezone string

docs/victoriametrics/victoria_metrics_enterprise_flags.md

@@ -21,7 +21,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -45,7 +45,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmagent.md

@@ -466,6 +466,43 @@ and `-remoteWrite.streamAggr.config`:
 
 There is also the `-promscrape.configCheckInterval` command-line flag, which can be used to automatically reload configs from the updated `-promscrape.config` file.
 
+## DNS URLs
+
+If `vmagent` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses
+via [DNS A records](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1). The port from the original URL is appended to each discovered IP address.
+Each discovered IP address is used for least-loaded balancing of write requests.
+
+DNS URLs are supported in the following places:
+
+* In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
+  `192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428/api/v1/write` is automatically resolved into
+  `-remoteWrite.url=http://192.168.1.15:8428/api/v1/write`.
+
+DNS URLs are useful when client-side HTTP load balancing is needed. A good example
+is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
+which returns multiple IP addresses for a single hostname.
+
+### DNS URLs and HTTPS
+
+When a `dns+` URL uses the `https` scheme, `vmagent` connects to the discovered
+IP addresses directly. This affects [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security)
+in two ways:
+
+* No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) is sent in the TLS handshake,
+  since the connection target is an IP address rather than a hostname.
+* The server certificate is verified against the IP address, so the verification fails
+  unless the certificate contains the corresponding
+  [IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
+
+To use `dns+` URLs with HTTPS, pass the original hostname via the `-remoteWrite.tlsServerName`
+command-line flag. It is used both as SNI and as the name the server certificate
+is verified against:
+
+```sh
+-remoteWrite.url=https://dns+victoria-metrics:8428/api/v1/write
+-remoteWrite.tlsServerName=victoria-metrics
+```
+
 ## SRV URLs
 
 If `vmagent` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
@@ -476,7 +513,7 @@ SRV URLs are supported in the following places:
 * In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
   `victoria-metrics-host:8428` TCP address, then `-remoteWrite.url=http://srv+victoria-metrics/api/v1/write` is automatically resolved into
   `-remoteWrite.url=http://victoria-metrics-host:8428/api/v1/write`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmagent`
-  uses a randomly chosen address for each connection it establishes to the remote storage.
+   performs per request least-loaded load-balancing.
 
 * In scrape target addresses aka `__address__` label. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#how-to-modify-scrape-urls-in-targets) for details.
 

docs/victoriametrics/vmagent_common_flags.md

@@ -83,7 +83,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
@@ -170,6 +170,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented
   -maxLabelsPerTimeseries int
      The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented
+  -mdx.label -remoteWrite.url
+     Optional label value in the form 'name=value' used to identify VictoriaMetrics metrics for MDX. Metrics containing the specified label are forwarded to -remoteWrite.url endpoints configured with `-remoteWrite.mdx.enable=true`.
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -465,6 +467,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      The maximum number of metadata to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 5000)
   -remoteWrite.maxRowsPerBlock int
      The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 10000)
+  -remoteWrite.mdx.enable array
+     Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange
+     Supports array of values separated by comma or specified via multiple flags.
+     Empty values are set to false.
   -remoteWrite.oauth2.clientID array
      Optional OAuth2 clientID to use for the corresponding -remoteWrite.url
      Supports an array of values separated by comma or specified via multiple flags.

docs/victoriametrics/vmagent_enterprise_flags.md

@@ -89,7 +89,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -103,7 +103,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmalert.md

@@ -546,7 +546,7 @@ tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags) and
 
 ## Reading rules from object storage
 
-[Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
+The [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
 from object storage:
 
 * `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket
@@ -563,6 +563,48 @@ The following [command-line flags](#flags) can be used for fine-tuning access to
 * `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
 * `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
 
+### S3 (AWS and S3-compatible)
+
+The following example reads rules from an S3 bucket:
+
+```sh
+./bin/vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+For S3-compatible backends such as [MinIO](https://www.min.io/) or [Ceph](https://ceph.io/), add `-s3.customEndpoint`:
+
+```sh
+./bin/vmalert \
+  -rule=s3://victoriametrics-alert-rules/rules_ \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -s3.customEndpoint=http://minio.example.local:9000 \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for details on creating IAM users, credentials file format, environment variables, and IAM roles.
+
+### Google Cloud Storage (GCS)
+
+The following example reads rules from a GCS bucket:
+
+```sh
+./bin/vmalert \
+  -rule=gs://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/gcp-service-account.json \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for details on creating service accounts and downloading JSON keys.
+
 ## Topology examples
 
 The following sections are showing how `vmalert` may be used and configured
@@ -1470,6 +1512,59 @@ alert_relabel_configs:
 
 The configuration file can be [hot-reloaded](#hot-config-reload).
 
+## DNS URLs
+
+If `vmalert` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses via DNS A/AAAA records.
+The port from the original URL is appended to each discovered IP address.
+Each discovered IP address is used for least-loaded balancing of write requests.
+
+DNS URLs are supported in the following places:
+
+* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
+  `192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428` is automatically resolved into
+  `-remoteWrite.url=http://192.168.1.15:8428`.
+
+DNS URLs are useful when client-side HTTP load balancing is needed. A good example
+is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
+which returns multiple IP addresses for a single hostname.
+
+### DNS URLs and HTTPS
+
+When a `dns+` URL uses the `https` scheme, `vmalert` connects to the discovered
+IP addresses directly. No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication)
+is sent in the TLS handshake, and the server certificate is verified against the IP address,
+which fails unless the certificate contains the corresponding
+[IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
+
+To use `dns+` URLs with HTTPS, pass the original hostname via the corresponding
+`tlsServerName` command-line flag - `-datasource.tlsServerName`, `-remoteRead.tlsServerName`
+or `-remoteWrite.tlsServerName`. It is used both as SNI and as the name the server
+certificate is verified against:
+
+```sh
+-datasource.url=https://dns+victoria-metrics:8428
+-datasource.tlsServerName=victoria-metrics
+```
+
+Alternatively, issue server certificates with IP SAN entries for every backend IP address.
+Avoid `tlsInsecureSkipVerify` flags for working around this, since they disable
+server certificate verification completely.
+
+## SRV URLs
+
+If `vmalert` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
+record into TCP address with hostname and TCP port, and then use the resulting URL when it needs to connect to it.
+
+SRV URLs are supported in the following places:
+
+* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
+  `victoria-metrics-host:8085`, then `-remoteWrite.url=http://srv+victoria-metrics:8428` is automatically resolved into
+  `-remoteWrite.url=http://victoria-metrics-host:8085`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmalert`
+   performs per request round-robin load-balancing.
+
+SRV URLs are useful when HTTP services run on different TCP ports or when their TCP ports can change over time (for instance, after a restart).
+
+
 ## Contributing
 
 `vmalert` is mostly designed and built by VictoriaMetrics community.

docs/victoriametrics/vmalert_common_flags.md

@@ -124,7 +124,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration

docs/victoriametrics/vmalert_enterprise_flags.md

@@ -23,7 +23,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -49,7 +49,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmauth_common_flags.md

@@ -68,7 +68,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration

docs/victoriametrics/vmauth_enterprise_flags.md

@@ -19,7 +19,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -33,7 +33,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmbackup.md

@@ -204,61 +204,45 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 ### Providing credentials as a file
 
-Obtaining credentials from a file.
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for instructions on obtaining credentials and providing them via credential files, environment variables, cloud provider metadata service, or Kubernetes secrets and IAM roles.
 
-Add flag `-credsFilePath=/etc/credentials` with the following content:
+The following examples show the most common authentication patterns for each storage provider.
 
-* for S3 (AWS, MinIO or other S3 compatible storages):
+#### S3 (AWS and S3-compatible)
 
-     ```sh
-     [default]
-     aws_access_key_id=theaccesskey
-     aws_secret_access_key=thesecretaccesskeyvalue
-    ```
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=s3://victoriametrics-backup/backup01 \
+  -credsFilePath=/etc/credentials
+```
 
-* for GCP cloud storage:
+#### Google Cloud Storage (GCS)
 
-    ```json
-    {
-           "type": "service_account",
-           "project_id": "project-id",
-           "private_key_id": "key-id",
-           "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
-           "client_email": "service-account-email",
-           "client_id": "client-id",
-           "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-           "token_uri": "https://accounts.google.com/o/oauth2/token",
-           "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-           "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
-    }
-    ```
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=gs://victoriametrics-backup/backup01 \
+  -credsFilePath=/etc/credentials
+```
 
-### Providing credentials via env variables
+#### Azure Blob Storage
 
-Obtaining credentials from env variables.
+```sh
+export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
+export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
 
-* For AWS S3 compatible storages set env variable `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
-  Also you can set env variable `AWS_SHARED_CREDENTIALS_FILE` with path to credentials file.
-* For GCE cloud storage set env variable `GOOGLE_APPLICATION_CREDENTIALS` with path to credentials file.
-* For Azure storage use one of these env variables:
-  * `AZURE_STORAGE_ACCOUNT_CONNECTION_STRING`: use a connection string (must be either SAS Token or Account/Key)
-  * `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY`: use a specific account name and key (either primary or secondary)
-  * `AZURE_USE_DEFAULT_CREDENTIAL` and `AZURE_STORAGE_ACCOUNT_NAME`: use the `DefaultAzureCredential` to allow the Azure library
-     to search for multiple options (for example, managed identity related variables). Note that if multiple credentials are available,
-     it is required to specify the `AZURE_CLIENT_ID` to select specific credentials.
-
-  The `AZURE_STORAGE_DOMAIN` can be used for optionally overriding the default domain for the Azure storage service.
-
-Please, note that `vmbackup` will use credentials provided by cloud providers metadata service [when applicable](https://docs.victoriametrics.com/victoriametrics/vmbackup/#using-cloud-providers-metadata-service).
-
-### Using cloud providers metadata service
-
-`vmbackup` and `vmbackupmanager` will automatically use cloud providers metadata service in order to obtain credentials if they are running in cloud environment and credentials are not explicitly provided via flags or env variables.
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=azblob://victoriametrics-backup/backup01
+```
 
 ### Providing credentials in Kubernetes
 
-The simplest way to provide credentials in Kubernetes is to use [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/)
-and inject them into the pod as environment variables. For example, the following secret can be used for AWS S3 credentials:
+The simplest way to provide credentials in Kubernetes is to use [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) and inject them into the pod as environment variables. For example, the following secret can be used for AWS S3 credentials:
 
 ```yaml
 apiVersion: v1
@@ -266,14 +250,13 @@ kind: Secret
 metadata:
   name: vmbackup-credentials
 data:
-  access_key: key
-  secret_key: secret
+  access_key: <base64-encoded-key>
+  secret_key: <base64-encoded-secret>
 ```
 
 And then it can be injected into the pod as environment variables:
 
 ```yaml
-...
 env:
 - name: AWS_ACCESS_KEY_ID
   valueFrom:
@@ -285,7 +268,6 @@ env:
     secretKeyRef:
       key: secret_key
       name: vmbackup-credentials
-...
 ```
 
 A more secure way is to use IAM roles to provide tokens for pods instead of managing credentials manually.

docs/victoriametrics/vmbackupmanager.md

@@ -88,34 +88,21 @@ There are two flags which could help with performance tuning:
 
 ### Example of Usage
 
-GCS and cluster version. You need to have a credentials file in json format with following structure:
-
-credentials.json
-
-```json
-{
-  "type": "service_account",
-  "project_id": "<project>",
-  "private_key_id": "",
-  "private_key": "-----BEGIN PRIVATE KEY-----\-----END PRIVATE KEY-----\n",
-  "client_email": "test@<project>.iam.gserviceaccount.com",
-  "client_id": "",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%40<project>.iam.gserviceaccount.com"
-}
-
-```
-
-Backup manager launched with the following configuration:
+The following command backs up data to a GCS bucket:
 
 ```sh
 export NODE_IP=192.168.0.10
 export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
-./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -licenseFile=/path/to/vm-license
+./vmbackupmanager \
+  -dst=gs://vmstorage-data/$NODE_IP \
+  -credsFilePath=credentials.json \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create \
+  -licenseFile=/path/to/vm-license
 ```
 
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for instructions on obtaining credentials and configuring access to S3, GCS, and Azure Blob Storage.
+
 Expected logs in vmbackupmanager:
 
 ```sh
@@ -147,8 +134,7 @@ objects. Typical object storage systems implement server-side copy by creating n
 This is very fast and efficient. Unfortunately there are systems such as [S3 Glacier](https://aws.amazon.com/s3/storage-classes/glacier/),
 which perform full object copy during server-side copying. This may be slow and expensive.
 
-Please, see [vmbackup docs](https://docs.victoriametrics.com/victoriametrics/vmbackup/#advanced-usage) for more examples of authentication with different
-storage types.
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for more examples of authentication with different storage types.
 
 ### Backup Retention Policy
 

docs/victoriametrics/vmctl/victoriametrics.md

@@ -8,9 +8,9 @@ menu:
     weight: 9
 ---
 
-The simplest way to migrate data between VictoriaMetrics installations is [to copy data between instances](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#data-migration).
-But when simple copy isn't possible (migration between single-node and cluster, or re-sharding) or when data need to be
-modified - use `vmctl vm-native` to migrate data.
+The simplest way to migrate data between VictoriaMetrics installations is [to copy data between instances](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#data-migration) or [to run single-node and cluster together](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#from-single-node-to-cluster).
+But when simple copy isn't possible (migration between single-node and cluster, or re-sharding) or when data needs to be
+modified, use `vmctl vm-native` to migrate data.
 
 See `./vmctl vm-native --help` for details and full list of flags.
 
@@ -190,4 +190,4 @@ See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametr
 
 See `./vmctl vm-native --help` for details and full list of flags:
 
-{{% content "vmctl_vm-native_flags.md" %}}
+{{% content "vmctl_vm-native_flags.md" %}}

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.146.0/vmutils-darwin-arm64-v1.146.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/vmutils-darwin-arm64-v1.148.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.146.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.148.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

docs/victoriametrics/vmestimator.md

@@ -1,584 +0,0 @@
----
-weight: 12
-menu:
-  docs:
-    parent: victoriametrics
-    weight: 12
-title: vmestimator
-tags:
-  - metrics
-  - cardinality
-aliases:
-  - /vmestimator.html
-  - /vmestimator/index.html
-  - /vmestimator/
----
-
-`vmestimator` measures metrics cardinality across arbitrary label dimensions and exposes the results as metrics.
-
-## Why measure?
-
-Consider a setup where metrics are scraped from dozens of Prometheus targets.
-One day, a team deploys a new version of their service with a `trace_id` or `user_id` label.
-Overnight, that job's cardinality explodes from 500 to 500,000 time series.
-Suddenly, VictoriaMetrics consumes 100x more memory and disk.
-Ingestion slows down, storage struggles to keep up, and in the worst case becomes unavailable.
-
-By the time someone gets paged, the damage is already done: indexes are bloated, caches are oversized, and observability across the entire system is affected.
-
-`vmestimator` continuously tracks cardinality and exposes the estimation results as [metrics](https://github.com/VictoriaMetrics/vmestimator/blob/main/README.md#cardinality-metrics).
-This allows alerting on cardinality spikes within minutes and identifying the offending job directly from the alert.
-Instead of discovering the problem after it impacts the infrastructure, it becomes possible to react before it turns into an outage.
-
-Per-job cardinality tracking is the most actionable use case, but it’s not the only one (see [use cases](https://github.com/VictoriaMetrics/vmestimator/#use-cases)).
-`vmestimator` can measure cardinality across arbitrary label dimensions,
-enabling use cases such as per-tenant usage analysis, long-term trend tracking, and capacity planning.
-
-## Design
-
-We recommend deploying `vmestimator` close to the metrics source, ideally alongside `vmagent` instances that scrape targets.
-Each `vmagent` mirrors all ingested metrics into the estimator.
-
-To reduce overhead, persistent queueing and metadata ingestion can be disabled for the estimator remote write path.
-It is safe to send metrics from multiple independent `vmagent` instances into a single `vmestimator`.
-
-Run vmestimator (see [configuration](https://github.com/VictoriaMetrics/vmestimator#configuration)):
-```bash
-/path/to/vmestimator -config=streams.yaml # -httpListenAddr=:8490
-```
-
-Run vmagent:
-```bash
-/path/to/vmagent \
-  -remoteWrite.url=http://127.0.0.1:8428/api/v1/write \
-  -remoteWrite.url=http://127.0.0.1:8490/cardinality/api/v1/write \
-  -remoteWrite.disableOnDiskQueue=false,true \
-  -remoteWrite.disableMetadata=false,true
-```
-
-The next step is to expose cardinality estimates as metrics.
-For this, `vmagent` should scrape the estimator `/metrics` endpoint and forward those metrics to a `vmsingle` instance (or another VictoriaMetrics storage).
-
-<img style="min-width:0;width: 100%" src="https://github.com/user-attachments/assets/e52d9210-b6f9-457b-8d8f-1d6ff6ba1416" />
-
-This setup is straightforward and introduces minimal overhead.
-The main drawback is that cardinality data shares the same storage with production metrics.
-If that storage becomes unavailable, the visibility into cardinality is lost precisely when it may be most needed.
-
-To mitigate this, we recommend running a separate `vmsingle` instance dedicated to scraping and storing VictoriaMetrics-related monitoring signals only.
-This pattern is commonly referred to as a monitoring-of-monitoring (MoM) setup.
-In this architecture, `vmestimator` metrics are isolated from production observability storage,
-ensuring cardinality visibility remains available even during incidents affecting the primary monitoring system.
-
-The resulting topology looks like this:
-<img style="min-width:0;width: 100%" src="https://github.com/user-attachments/assets/e2ca4a69-e931-47a1-9d91-99749382d4a9" />
-
-## Install
-
-Create a `streams.yaml` from [example config](https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml).
-Run the Docker image from [Docker Hub](https://hub.docker.com/r/victoriametrics/vmestimator) or [Quay](https://quay.io/repository/victoriametrics/vmestimator), mounting your config file:
-```bash
-docker run --rm \
-  -p 8490:8490 \
-  -v /path/to/streams.yaml:/streams.yaml \
-  docker.io/victoriametrics/vmestimator:latest \
-  -config=/streams.yaml
-```
-
-See [Use Cases](https://github.com/VictoriaMetrics/vmestimator#use-cases) for more configuration examples and
-[Command-line flags](https://github.com/VictoriaMetrics/vmestimator#command-line-flags) for all available options.
-
-To build from sources, see [How to build from sources](https://github.com/VictoriaMetrics/vmestimator#how-to-build-from-sources).
-
-## Configuration
-
-To run vmestimator a `streams.yaml` config has to be provided (see [example config](https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml)):
-
-```bash
-/path/to/vmestimator -config=streams.yaml # -httpListenAddr=:8490
-```
-
-Config reference:
-```yaml
-streams:
-  -
-    # The measurement window: how long unique series are retained before the HLL sketch resets.
-    # Increases are always reflected immediately. Interval only controls how fast the estimate
-    # drops after previously seen series disappear.
-    #
-    # Running two streams with different intervals (e.g. 5m and 1h) lets you derive churn rate
-    # by comparing their estimates. See Use Cases -> Churn Rate
-    #
-    # default: 5m
-    interval: 'golang duration'
-
-    # Label names used to split the cardinality estimate into per-combination groups.
-    # Each distinct combination of values for these labels gets its own estimate metric.
-    # Omit entirely for a single global estimate across all series.
-    # Examples:
-    #  - ["job"]
-    #  - ["__name__"] 
-    #  - ["vm_account_id","vm_project_id"]
-    #
-    # default: none (single global estimate)
-    group_by: 'string array'
-
-    # Maximum number of distinct groups (HLL sketches) to track.
-    # Once the limit is reached, excess groups are counted in a single shared "rejected" sketch
-    # rather than getting their own entry. Acts as a memory cap and a safeguard against OOM
-    # when the group_by label values grow unboundedly.
-    # Memory upper bound per stream: 
-    #   group_limit * 2^hll_precision bytes. 
-    #
-    # default: 10000
-    group_limit: 'integer'
-
-    # Number of shards used to reduce lock contention during parallel ingestion.
-    # Slightly increases memory for global streams (no group_by); negligible otherwise.
-    # Leave at the default unless you have profiled lock contention or have a specific reason to change it.
-    #
-    # default: min(64, 2*availableCPUs)
-    buckets: 'integer'
-
-    # HyperLogLog precision p, in range [4..18].
-    # Determines the number of registers m = 2^p and the relative error 1.04 / sqrt(m):
-    #   p=14 → m=16 384, error ~0.81%, memory ~16 KB per sketch  (default, suits most cases)
-    #   p=18 → m=262 144, error ~0.20%, memory ~256 KB per sketch (billing-grade accuracy)
-    #   p=10 → m=1 024,   error ~3.25%, memory ~1 KB per sketch   (thousands of groups, memory-tight)
-    # See more in https://research.google.com/pubs/archive/40671.pdf
-    #
-    # default: 14
-    hll_precision: 'integer'
-
-    # Whether to use the sparse HyperLogLog representation for low-cardinality groups.
-    # Sparse mode uses far less memory until a group's cardinality reaches ~2^(p-1),
-    # at which point it automatically promotes to the dense representation.
-    # See more in https://research.google.com/pubs/archive/40671.pdf
-    #
-    # default: true
-    hll_sparse: 'boolean'
-
-    # Static labels attached to every output metric produced by this stream entry.
-    # Useful when multiple vmestimator instances feed the same storage and you need
-    # to distinguish their estimates in dashboards and alerts.
-    labels: 'map key string: value string'
-```
-
-## Cardinality Metrics
-
-Cardinality estimates are exposed as the `cardinality_estimate` metric.
-All metrics include `interval`, `group_by_keys`, `group_by_values`, and any static labels defined in the stream config.
-
-For global estimates (no `group_by` configured), `group_by_keys` is `__global__` and `group_by_values` is omitted:
-```
-cardinality_estimate{interval="1h0m0s",group_by_keys="__global__"} 142300
-```
-
-For grouped estimates, one summary line shows the total number of distinct groups `group_by_keys="__group__"`, followed by one line per distinct label value combination.
-Each per-group line also includes individual `by_{key}="{val}"` labels:
-```
-cardinality_estimate{interval="5m0s",group_by_keys="__group__",group_by_values="instance,job"} 2
-cardinality_estimate{interval="5m0s",group_by_keys="instance,job",group_by_values="host1:9090,prometheus",by_instance="host1:9090",by_job="prometheus"} 312
-cardinality_estimate{interval="5m0s",group_by_keys="instance,job",group_by_values="host2:9100,node",by_instance="host2:9100",by_job="node"} 87
-```
-
-Note: the total distinct group count in the summary line may exceed the number of per-group lines when `group_limit` is reached
-and excess groups are counted in a single shared "rejected" sketch rather than getting their own entry.
-
-By default, cardinality estimates are merged with the estimator's operational metrics and exposed at `/metrics`.
-This is controlled by the `-cardinalityMetrics.exposeAt` flag:
-- `-cardinalityMetrics.exposeAt=/metrics` (default): cardinality metrics merged with operational metrics at `/metrics`
-- `-cardinalityMetrics.exposeAt=/cardinality/metrics`: cardinality metrics exposed at separate path
-- `-cardinalityMetrics.exposeAt=`: cardinality metrics not exposed via HTTP
-
-Computing cardinality estimates is expensive, so results are cached.
-Cache duration is controlled by `-cardinalityMetrics.cacheTTL` (default: `30s`).
-Set to `0` to disable caching entirely.
-
-## Use Cases
-
-### Basic
-
-Global cardinality:
-```yaml
-# streams.yaml
-
-- interval: '5m'
-```
-
-Per metric name cardinality:
-```yaml
-# streams.yaml
-
-- interval: '5m'
-  group_by: ['__name__']
-```
-
-Per job label cardinality:
-```yaml
-# streams.yaml
-
-- interval: '5m'
-  group_by: ['job']
-```
-
-Per tenant cardinality:
-```yaml
-# streams.yaml
-
-- interval: '5m'
-  group_by: ['vm_account_id', 'vm_project_id']
-```
-
-### Churn calculation
-
-[Churn rate](https://valyala.medium.com/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48#churn-rate) measures how quickly time series are created and disappear.
-[High churn](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) means many series appear briefly and are replaced by new ones.
-This puts pressure on storage, because each new series must be indexed regardless of how short its lifetime is.
-
-To measure churn, configure two streams with the same `group_by` but different intervals. A short one (`15m`) and a long one (`30m`):
-```yaml
-# streams.yaml
-
-- interval: '15m'
-  group_by: ['job']
-
-- interval: '30m'
-  group_by: ['job']
-```
-
-When churn is low, both estimates are roughly equal.
-When churn is high, the `30m` estimate grows significantly larger than the `15m` estimate, because the long window accumulates series that have already disappeared.
-
-The following query computes the churn ratio per job:
-```
-(
-    sum(
-        max(cardinality_estimate{group_by_keys="job",interval="30m0s"}) without (instance)
-    ) by (group_by_keys,group_by_values)
-    -
-    sum(
-        max(cardinality_estimate{group_by_keys="job",interval="15m0s"}) without (instance)
-    ) by (group_by_keys,group_by_values)
-)
-/
-sum(
-    max(cardinality_estimate{group_by_keys="job",interval="30m0s"}) without (instance)
-) by (group_by_keys,group_by_values) * 100
-```
-
-A result near `0` means the series set is stable. The same series were active throughout the entire hour.
-A result near `1` means complete churn. Entirely different series appeared each 5-minute window.
-Values in between indicate the fraction of maximum possible churn that is occurring.
-
-This helps identify jobs that create the most indexing pressure on storage, even when their current active cardinality appears moderate.
-
-### Alerting
-
-Pre-built alert rules for cardinality monitoring are available in
-[deployment/docker/rules/alerts-cardinality.yml](https://github.com/VictoriaMetrics/vmestimator/blob/main/deployment/docker/rules/alerts-cardinality.yml).
-
-They require two streams with the same `group_by` but different intervals to also support churn detection:
-```yaml
-# streams.yaml
-# or use example config:
-# https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml
-
-- interval: '15m'
-  group_by: ['job']
-
-- interval: '30m'
-  group_by: ['job']
-```
-
-The included alerts are:
-
-- **JobTooHighCardinality** — fires when any job exceeds 20,000 estimated active series over the last 30 minutes.
-  The threshold is a starting point and should be calibrated to reflect the expected cardinality of your largest jobs.
-
-- **JobTooHighChurnRate** — fires when more than 10% of a job's series churned between the 15m and 30m windows.
-  Catches jobs that generate continuous indexing pressure even when their active series count looks moderate.
-
-- **CardinalityGroupLimitNearlyReached** — fires when the number of tracked groups exceeds 80% of the configured `group_limit`.
-  Acts as an early warning that some label value combinations may soon be dropped from individual tracking.
-
-- **CardinalityGroupLimitReached** — fires when groups are actively rejected because `group_limit` is full.
-  At this point, some label combinations are being counted in a shared "rejected" sketch rather than tracked individually.
-
-All alerts link to the [Cardinality Explorer dashboard](https://play-grafana.victoriametrics.com/d/mktd5h8/).
-
-## Alternative solutions
-
-### PromQL
-
-Cardinality can be estimated with PromQL.
-
-Global cardinality:
-```
-count({__name__=~".*"})
-```
-
-Top ten metric names by cardinality:
-```
-topk(10, count({__name__=~".*"}) by (__name__))
-```
-
-Top ten jobs by cardinality:
-```
-topk(10, count({__name__=~".*"}) by (job))
-```
-
-This approach works for small setups but does not scale well, because these queries scan the entire time series set.
-Most critically, if the storage is overloaded or unavailable, these queries could not be executed.
-
-### Cardinality Explorer
-
-VictoriaMetrics includes a built-in [cardinality explorer](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-explorer).
-It provides per-metric detail beyond raw series counts: query frequency, last access time, day-over-day change, and share of total cardinality.
-It is well suited for in-depth, ad-hoc investigation.
-For example, finding metrics that are high-cardinality but rarely queried,
-so they can be [dropped via relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/#how-to-drop-metrics-during-scrape) or reduce cardinality with [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/).
-
-Both tools serve different purposes and work well together.
-Use `vmestimator` for continuous monitoring, alerting, and cross-cluster cardinality tracking.
-Use the cardinality explorer when you need to drill into a specific metric or label and understand what is driving its cardinality.
-
-## Cluster
-
-`vmestimator` supports a clustered deployment for high availability or when CPU on a single instance becomes a limiting factor.
-
-Instances are split into two roles: **storage nodes** accept Prometheus remote write and maintain local HyperLogLog sketches; **selector nodes** query all storage nodes, merge their sketches, and expose a unified cardinality estimate. Cardinality estimate results should be scraped from selector nodes.
-
-<img style="min-width:0;width: 100%" src="https://github.com/user-attachments/assets/846e5f77-378a-44dc-a4c8-2a1c64eca9d8" />
-
-**Storage nodes:**
-```
-vmestimator -config=streams.yaml -httpListenAddr=:8491 -cardinalityMetrics.exposeAt=/cardinality/metrics
-vmestimator -config=streams.yaml -httpListenAddr=:8492 -cardinalityMetrics.exposeAt=/cardinality/metrics
-vmestimator -config=streams.yaml -httpListenAddr=:8493 -cardinalityMetrics.exposeAt=/cardinality/metrics
-```
-
-**Selector nodes:**
-```
-vmestimator -storageNode=http://vmestimator-storage-1:8491 \
-            -storageNode=http://vmestimator-storage-2:8492 \
-            -storageNode=http://vmestimator-storage-3:8493 \
-            -httpListenAddr=:8490
-```
-
-Setting `-cardinalityMetrics.exposeAt=/cardinality/metrics` on storage nodes keeps per-node estimates off the default `/metrics` path. The `/metrics` endpoint then returns only operational metrics, while `/cardinality/metrics` exposes the node's local estimate — useful for inspecting or debugging a specific node.
-
-A selector with `-storageNode` flags and no `-config` runs without local estimators and only merges remote data.
-
-When multiple selector nodes are scraped, each returns a fully merged estimate.
-Deduplicate at query time to avoid overcounting:
-```
-max(cardinality_estimate) without (instance)
-```
-
-## Operational metrics
-
-When grouping is enabled, vmestimator exposes per-bucket operational metrics at `/metrics`:
-
-- `vmestimator_estimator_group_size{group_by_keys, bucket}` — number of active groups in this bucket after the last rotation
-- `vmestimator_estimator_group_rejected_size{group_by_keys}` — estimated number of distinct group values rejected since the last rotation because `group_limit` was reached
-- `vmestimator_estimator_group_limit{group_by_keys, bucket}` — configured `group_limit` for this bucket
-
-
-## Dashboards
-
-Two Grafana dashboards are available in the [dashboards](https://github.com/VictoriaMetrics/vmestimator/tree/main/dashboards) directory:
-
-- [VictoriaMetrics - vmestimator](https://play-grafana.victoriametrics.com/d/mkv22l4/victoriametrics-vmestimator) — application health: CPU, memory, ingestion rates, concurrent inserts, and group key saturation.
-  <img width="1507" height="801" alt="Screenshot 2026-06-29 at 19 06 46" src="https://github.com/user-attachments/assets/cbfd979d-f403-4270-b098-2d2f0b392172" />
-
-- [VictoriaMetrics - Cardinality Explorer](https://play-grafana.victoriametrics.com/d/mktd5h8/victoriametrics-cardinality-explorer) — cardinality analysis: global estimates, per-group-key series counts, and top-10 highest-cardinality label value combinations.
-  <img width="1510" height="796" alt="Screenshot 2026-06-29 at 19 05 47" src="https://github.com/user-attachments/assets/a1aea6e1-8714-4d5a-a629-8bdee978f1c6" />
-
-## How to build from sources
-
-It is recommended to use the [docker images](https://hub.docker.com/r/victoriametrics/vmestimator).
-
-Development build:
-1. [Install Go](https://golang.org/doc/install).
-1. Run `make vmestimator` from the root folder of [the repository](https://github.com/VictoriaMetrics/vmestimator).
-   It builds `vmestimator` binary and places it into the `bin` folder.
-
-Production build:
-1. [Install docker](https://docs.docker.com/install/).
-1. Run `make vmestimator-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/vmestimator).
-   It builds `vmestimator-prod` binary and puts it into the `bin` folder.
-
-Building docker images:
-
-Run `make package-vmestimator`. It builds `victoriametrics/vmestimator:<PKG_TAG>` docker image locally.
-`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
-The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmestimator`.
-
-The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image by setting it via `<ROOT_IMAGE>` environment variable.
-For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
-
-```sh
-ROOT_IMAGE=scratch make package-vmestimator
-```
-
-You can build and publish to your own registry and namespace:
-```
-DOCKER_REGISTRIES=ghcr.io DOCKER_NAMESPACE=foo make publish-vmestimator
-```
-
-## Command-line flags
-
-Run `vmestimator -help` in order to see all the available options:
-
-```
-Usage of ./bin/vmestimator:
-  -cardinalityMetrics.cacheTTL duration
-        Duration for caching cardinality metrics response (default 30s)
-  -cardinalityMetrics.exposeAt string
-        HTTP path for exposing cardinality metrics. If set to the default /metrics, cardinality metrics are merged with regular metrics and exposed together. If set to a different path, only cardinality metrics are exposed at that endpoint. If set to an empty value, cardinality metrics are not exposed via HTTP at all. (default "/metrics")
-  -config string
-        Path to YAML configuration file. Must be set unless -storageNode is specified. See https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml for config example
-  -enableTCP6
-        Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
-  -envflag.enable
-        Whether to enable reading flags from environment variables in addition to the command line. Command line flag values have priority over values from environment vars. Flags are read only from the command line if this flag isn't set. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#environment-variables for more details
-  -envflag.prefix string
-        Prefix for environment variables if -envflag.enable is set
-  -filestream.disableFadvise
-        Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU
-  -flagsAuthKey value
-        Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
-        Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file.
-        Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
-  -fs.maxConcurrency int
-        The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph (default 160)
-  -http.connTimeout duration
-        Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
-  -http.disableCORS
-        Disable CORS for all origins (*)
-  -http.disableKeepAlive
-        Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
-  -http.disableResponseCompression
-        Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-  -http.header.csp string
-        Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-  -http.header.disableServerHostname
-        Whether to disable 'X-Server-Hostname' header in HTTP responses
-  -http.header.frameOptions string
-        Value for 'X-Frame-Options' header
-  -http.header.hsts string
-        Value for 'Strict-Transport-Security' header, recommended: 'max-age=31536000; includeSubDomains'
-  -http.idleConnTimeout duration
-        Timeout for incoming idle http connections (default 1m0s)
-  -http.maxGracefulShutdownDuration duration
-        The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
-  -http.pathPrefix string
-        An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
-  -http.shutdownDelay duration
-        Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
-  -httpAuth.password value
-        Password for HTTP server's Basic Auth. The authentication is disabled if -httpAuth.username is empty
-        Flag value can be read from the given file when using -httpAuth.password=file:///abs/path/to/file or -httpAuth.password=file://./relative/path/to/file.
-        Flag value can be read from the given http/https url when using -httpAuth.password=http://host/path or -httpAuth.password=https://host/path
-  -httpAuth.username string
-        Username for HTTP server's Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
-  -httpListenAddr array
-        TCP address to listen for incoming HTTP requests
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -insert.maxQueueDuration duration
-        The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
-  -internStringCacheExpireDuration duration
-        The expiry duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
-  -internStringDisableCache
-        Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
-  -internStringMaxLen int
-        The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
-  -loggerDisableTimestamps
-        Whether to disable writing timestamps in logs
-  -loggerErrorsPerSecondLimit int
-        Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
-  -loggerFormat string
-        Format for logs. Possible values: default, json (default "default")
-  -loggerJSONFields string
-        Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
-  -loggerLevel string
-        Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
-  -loggerMaxArgLen int
-        The maximum length of a single logged argument. Longer arguments are replaced with 'arg_start..arg_end', where 'arg_start' and 'arg_end' is prefix and suffix of the arg with the length not exceeding -loggerMaxArgLen / 2 (default 5000)
-  -loggerOutput string
-        Output for the logs. Supported values: stderr, stdout (default "stderr")
-  -loggerTimezone string
-        Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
-  -loggerWarnsPerSecondLimit int
-        Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-  -maxConcurrentInserts int
-        The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 20)
-  -maxInsertRequestSize size
-        The maximum size in bytes of a single Prometheus remote_write API request
-        Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
-  -memory.allowedBytes size
-        Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
-        Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
-  -memory.allowedPercent float
-        Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
-  -metrics.exposeMetadata
-        Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
-  -metricsAuthKey value
-        Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
-        Flag value can be read from the given file when using -metricsAuthKey=file:///abs/path/to/file or -metricsAuthKey=file://./relative/path/to/file.
-        Flag value can be read from the given http/https url when using -metricsAuthKey=http://host/path or -metricsAuthKey=https://host/path
-  -pprofAuthKey value
-        Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides -httpAuth.*
-        Flag value can be read from the given file when using -pprofAuthKey=file:///abs/path/to/file or -pprofAuthKey=file://./relative/path/to/file.
-        Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path
-  -pushmetrics.disableCompression
-        Whether to disable request body compression when pushing metrics to every -pushmetrics.url
-  -pushmetrics.extraLabel array
-        Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -pushmetrics.header array
-        Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -pushmetrics.interval duration
-        Interval for pushing metrics to every -pushmetrics.url (default 10s)
-  -pushmetrics.url array
-        Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -secret.flags array
-        Comma-separated list of flag names with secret values. Values for these flags are hidden in logs and on /metrics page
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -storageNode array
-        HTTP URLs of remote vmestimator nodes to query for cardinality snapshots, e.g. http://vmestimator-2:8490
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -tls array
-        Whether to enable TLS for incoming HTTP requests at the given -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set. See also -mtls
-        Supports array of values separated by comma or specified via multiple flags.
-        Empty values are set to false.
-  -tlsCertFile array
-        Path to file with TLS certificate for the corresponding -httpListenAddr if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. See also -tlsAutocertHosts
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -tlsCipherSuites array
-        Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -tlsKeyFile array
-        Path to file with TLS key for the corresponding -httpListenAddr if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated. See also -tlsAutocertHosts
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -tlsMinVersion array
-        Optional minimum TLS version to use for the corresponding -httpListenAddr if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-        Supports an array of values separated by comma or specified via multiple flags.
-        Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -version
-        Show VictoriaMetrics version
-```
-
-

docs/victoriametrics/vminsert_common_flags.md

@@ -85,7 +85,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
@@ -165,9 +165,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
   -maxLabelNameLen int
-     The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_name"} metric at /metrics page is incremented (default 256)
+     The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_name"} metric at /metrics page is incremented. Value must be in range 1..65535. (default 256)
   -maxLabelValueLen int
-     The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented (default 4096)
+     The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented. Value must be in range 1..65535. (default 4096)
   -maxLabelsPerTimeseries int
      The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented (default 40)
   -memory.allowedBytes size

docs/victoriametrics/vminsert_enterprise_flags.md

@@ -41,7 +41,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -61,7 +61,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmrestore.md

@@ -47,13 +47,13 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
 
 ## Troubleshooting
 
-* See [how to setup credentials via environment variables](https://docs.victoriametrics.com/victoriametrics/vmbackup/#providing-credentials-via-env-variables).
+* See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for instructions on configuring credentials.
 * If `vmrestore` consumes all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
 * If `vmrestore` has been interrupted due to temporary error, then just restart it with the same args. It will resume the restore process.
 
 ## Advanced usage
 
-Please, see [vmbackup docs](https://docs.victoriametrics.com/victoriametrics/vmbackup/#advanced-usage) for examples of authentication
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for examples of authentication
 with different storage types.
 
 Run `vmrestore -help` in order to see all the available options:

docs/victoriametrics/vmselect_common_flags.md

@@ -41,6 +41,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Flag value can be read from the given http/https url when using -deleteAuthKey=http://host/path or -deleteAuthKey=https://host/path
   -denyQueryTracing
      Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing
+  -enableMetadata
+     Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
   -enableMultitenancyViaHeaders
      Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers
   -enableTCP6
@@ -84,7 +86,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
@@ -103,6 +105,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Whether to use proxy protocol for connections accepted at the given -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing
      Supports array of values separated by comma or specified via multiple flags.
      Empty values are set to false.
+  -insert.maxQueueDuration duration
+     The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
   -internStringCacheExpireDuration duration
      The expiry duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
   -internStringDisableCache
@@ -127,6 +131,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxConcurrentInserts int
+     The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 24)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -148,6 +154,119 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path
   -prevCacheRemovalPercent float
      Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1)
+  -promscrape.azureSDCheckInterval duration
+     Interval for checking for changes in Azure. This works only if azure_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#azure_sd_configs for details (default 1m0s)
+  -promscrape.cluster.memberLabel string
+     If non-empty, then the label with this name and the -promscrape.cluster.memberNum value is added to all the scraped metrics. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info
+  -promscrape.cluster.memberNum string
+     The number of vmagent instance in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name. See also -promscrape.cluster.memberLabel . See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info (default "0")
+  -promscrape.cluster.memberURLTemplate string
+     An optional template for URL to access vmagent instance with the given -promscrape.cluster.memberNum value. Every %d occurrence in the template is substituted with -promscrape.cluster.memberNum at urls to vmagent instances responsible for scraping the given target at /service-discovery page. For example -promscrape.cluster.memberURLTemplate='http://vmagent-%d:8429/targets'. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more details
+  -promscrape.cluster.membersCount int
+     The number of members in a cluster of scrapers. Each member must have a unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default, cluster scraping is disabled, i.e. a single scraper scrapes all the targets. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info (default 1)
+  -promscrape.cluster.name string
+     Optional name of the cluster. If multiple vmagent clusters scrape the same targets, then each cluster must have unique name in order to properly de-duplicate samples received from these clusters. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info
+  -promscrape.cluster.replicationFactor int
+     The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 1, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info (default 1)
+  -promscrape.cluster.shardByLabels array
+     Optional list of target labels, which will be used for sharding targets among cluster members if -promscrape.cluster.membersCount is greater than 1. If none of the specified labels are found in a target, then all the target labels will be used for sharding. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info
+     Supports an array of values separated by comma or specified via multiple flags.
+     Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
+  -promscrape.config string
+     Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
+  -promscrape.config.dryRun
+     Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
+  -promscrape.config.strictParse
+     Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true)
+  -promscrape.configCheckInterval duration
+     Interval for checking for changes in -promscrape.config file. By default, the checking is disabled. See how to reload -promscrape.config file at https://docs.victoriametrics.com/victoriametrics/vmagent/#configuration-update
+  -promscrape.consul.waitTime duration
+     Wait time used by Consul service discovery. Default value is used if not set
+  -promscrape.consulSDCheckInterval duration
+     Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#consul_sd_configs for details (default 30s)
+  -promscrape.consulagentSDCheckInterval duration
+     Interval for checking for changes in Consul Agent. This works only if consulagent_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#consulagent_sd_configs for details (default 30s)
+  -promscrape.digitaloceanSDCheckInterval duration
+     Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#digitalocean_sd_configs for details (default 1m0s)
+  -promscrape.disableCompression
+     Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine-grained control
+  -promscrape.disableKeepAlive
+     Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine-grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets
+  -promscrape.discovery.concurrency int
+     The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
+  -promscrape.discovery.concurrentWaitTime duration
+     The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
+  -promscrape.dnsSDCheckInterval duration
+     Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#dns_sd_configs for details (default 30s)
+  -promscrape.dockerSDCheckInterval duration
+     Interval for checking for changes in docker. This works only if docker_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#docker_sd_configs for details (default 30s)
+  -promscrape.dockerswarmSDCheckInterval duration
+     Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#dockerswarm_sd_configs for details (default 30s)
+  -promscrape.dropOriginalLabels
+     Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs
+  -promscrape.ec2SDCheckInterval duration
+     Interval for checking for changes in ec2. This works only if ec2_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#ec2_sd_configs for details (default 1m0s)
+  -promscrape.eurekaSDCheckInterval duration
+     Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#eureka_sd_configs for details (default 30s)
+  -promscrape.fileSDCheckInterval duration
+     Interval for checking for changes in 'file_sd_config'. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#file_sd_configs for details (default 1m0s)
+  -promscrape.gceSDCheckInterval duration
+     Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#gce_sd_configs for details (default 1m0s)
+  -promscrape.hetznerSDCheckInterval duration
+     Interval for checking for changes in Hetzner API. This works only if hetzner_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs for details (default 1m0s)
+  -promscrape.httpSDCheckInterval duration
+     Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs for details (default 1m0s)
+  -promscrape.kubernetes.apiServerTimeout duration
+     How frequently to reload the full state from Kubernetes API server (default 30m0s)
+  -promscrape.kubernetes.attachNamespaceMetadataAll
+     Whether to set attach_metadata.namespace=true for all the kubernetes_sd_configs at -promscrape.config . It is possible to set attach_metadata.namespace=false individually per each kubernetes_sd_configs . See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs
+  -promscrape.kubernetes.attachNodeMetadataAll
+     Whether to set attach_metadata.node=true for all the kubernetes_sd_configs at -promscrape.config . It is possible to set attach_metadata.node=false individually per each kubernetes_sd_configs . See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs
+  -promscrape.kubernetes.useHTTP2Client
+     Whether to use HTTP/2 client for connection to Kubernetes API server. This may reduce amount of concurrent connections to API server when watching for a big number of Kubernetes objects.
+  -promscrape.kubernetesSDCheckInterval duration
+     Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs for details (default 30s)
+  -promscrape.kumaSDCheckInterval duration
+     Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kuma_sd_configs for details (default 30s)
+  -promscrape.marathonSDCheckInterval duration
+     Interval for checking for changes in Marathon REST API. This works only if marathon_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#marathon_sd_configs for details (default 30s)
+  -promscrape.maxDroppedTargets int
+     The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 10000)
+  -promscrape.maxResponseHeadersSize size
+     The maximum size of http response headers from Prometheus scrape targets
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 4096)
+  -promscrape.maxScrapeSize size
+     The maximum size of uncompressed scrape response in bytes to process from Prometheus targets. Bigger uncompressed responses are rejected. See also max_scrape_size option at https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16777216)
+  -promscrape.minResponseSizeForStreamParse size
+     The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/victoriametrics/vmagent/#stream-parsing-mode
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000)
+  -promscrape.noStaleMarkers
+     Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
+  -promscrape.nomad.waitTime duration
+     Wait time used by Nomad service discovery. Default value is used if not set
+  -promscrape.nomadSDCheckInterval duration
+     Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#nomad_sd_configs for details (default 30s)
+  -promscrape.openstackSDCheckInterval duration
+     Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#openstack_sd_configs for details (default 30s)
+  -promscrape.ovhcloudSDCheckInterval duration
+     Interval for checking for changes in OVH Cloud API. This works only if ovhcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#ovhcloud_sd_configs for details (default 30s)
+  -promscrape.puppetdbSDCheckInterval duration
+     Interval for checking for changes in PuppetDB API. This works only if puppetdb_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#puppetdb_sd_configs for details (default 30s)
+  -promscrape.seriesLimitPerTarget int
+     Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter for more info
+  -promscrape.streamParse
+     Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is possible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine-grained control
+  -promscrape.suppressDuplicateScrapeTargetErrors
+     Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting for details
+  -promscrape.suppressScrapeErrors
+     Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay
+  -promscrape.suppressScrapeErrorsDelay duration
+     The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
+  -promscrape.vultrSDCheckInterval duration
+     Interval for checking for changes in Vultr. This works only if vultr_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#vultr_sd_configs for details (default 30s)
+  -promscrape.yandexcloudSDCheckInterval duration
+     Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs for details (default 30s)
   -pushmetrics.disableCompression
      Whether to disable request body compression when pushing metrics to every -pushmetrics.url
   -pushmetrics.extraLabel array
@@ -277,7 +396,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      The minimum memory bytes consumption for queries to track in query stats at /api/v1/status/top_queries. Queries with lower memory bytes consumption are ignored in query stats
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1024)
   -search.resetCacheAuthKey value
-     Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It could be passed via authKey query arg. It overrides -httpAuth.*
+     Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It could be passed via authKey query arg. It overrides -httpAuth.*. It'll be used when reset request is propagate to other -selectNode.
      Flag value can be read from the given file when using -search.resetCacheAuthKey=file:///abs/path/to/file or -search.resetCacheAuthKey=file://./relative/path/to/file.
      Flag value can be read from the given http/https url when using -search.resetCacheAuthKey=http://host/path or -search.resetCacheAuthKey=https://host/path
   -search.resetRollupResultCacheOnStartup
@@ -295,7 +414,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -selectNode array
-     A list of vmselect node addresses to propagate the '/internal/resetRollupResultCache' call. If this flag isn't set, then cache need to be purged from each vmselect individually. Comma-separated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN
+     A list of vmselect node addresses to propagate the '/internal/resetRollupResultCache?propagate=1' call. If either this flag or the 'propagate=1' query parameter is missing, the cache must be purged on each vmselect instance individually.Comma-separated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -storageNode array

docs/victoriametrics/vmselect_enterprise_flags.md

@@ -45,7 +45,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -69,7 +69,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmstorage_common_flags.md

@@ -77,7 +77,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
@@ -130,6 +130,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxBackfillAge value
+     The maximum allowed age for the ingested samples with historical timestamps. Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention
+     The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 0)
   -maxConcurrentInserts int
      The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 2*cgroup.AvailableCPUs())
   -memory.allowedBytes size

docs/victoriametrics/vmstorage_enterprise_flags.md

@@ -35,7 +35,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -53,7 +53,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

go.mod

@@ -1,6 +1,6 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
-go 1.26.4
+go 1.26.5
 
 require (
 	cloud.google.com/go/storage v1.62.3
@@ -11,7 +11,7 @@ require (
 	github.com/VictoriaMetrics/easyproto v1.2.0
 	github.com/VictoriaMetrics/fastcache v1.13.3
 	github.com/VictoriaMetrics/metrics v1.44.0
-	github.com/VictoriaMetrics/metricsql v0.87.2
+	github.com/VictoriaMetrics/metricsql v0.87.3
 	github.com/aws/aws-sdk-go-v2 v1.42.0
 	github.com/aws/aws-sdk-go-v2/config v1.32.25
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.27
@@ -34,9 +34,9 @@ require (
 	github.com/valyala/gozstd v1.24.0
 	github.com/valyala/histogram v1.2.0
 	github.com/valyala/quicktemplate v1.8.0
-	golang.org/x/net v0.56.0
+	golang.org/x/net v0.57.0
 	golang.org/x/oauth2 v0.36.0
-	golang.org/x/sys v0.46.0
+	golang.org/x/sys v0.47.0
 	google.golang.org/api v0.284.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -130,6 +130,7 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
+	github.com/yuin/goldmark v1.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/collector/component v1.60.0 // indirect
 	go.opentelemetry.io/collector/confmap v1.60.0 // indirect
@@ -155,12 +156,19 @@ require (
 	go.uber.org/zap v1.28.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.53.0 // indirect
-	golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
-	golang.org/x/sync v0.21.0 // indirect
-	golang.org/x/term v0.44.0 // indirect
-	golang.org/x/text v0.38.0 // indirect
+	golang.org/x/crypto v0.54.0 // indirect
+	golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect
+	golang.org/x/mod v0.38.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
+	golang.org/x/tools v0.48.0 // indirect
+	golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
+	golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
+	golang.org/x/tools/godoc v0.1.0-deprecated // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto v0.0.0-20260615183401-62b3387ff324 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 // indirect

go.sum

@@ -62,6 +62,8 @@ github.com/VictoriaMetrics/metrics v1.44.0 h1:Fr8yqQSV+ZfYaDD/anqk1E8e9YPgfleSle
 github.com/VictoriaMetrics/metrics v1.44.0/go.mod h1:xDM82ULLYCYdFRgQ2JBxi8Uf1+8En1So9YUwlGTOqTc=
 github.com/VictoriaMetrics/metricsql v0.87.2 h1:7OsrcDBWREWKqqpnFyIUEOM4FNv2qHvCoww2GYz3Tc0=
 github.com/VictoriaMetrics/metricsql v0.87.2/go.mod h1:d4EisFO6ONP/HIGDYTAtwrejJBBeKGQYiRl095bS4QQ=
+github.com/VictoriaMetrics/metricsql v0.87.3 h1:JU4JnVKSC5Vp3b4AvogXyOAjkz1iFF9n1KBMphS4WO8=
+github.com/VictoriaMetrics/metricsql v0.87.3/go.mod h1:d4EisFO6ONP/HIGDYTAtwrejJBBeKGQYiRl095bS4QQ=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
@@ -436,6 +438,8 @@ github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAz
 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.8.4 h1:oat/nd3U6NeQqFEL3xpEJq7d7c86NI+DbSNGAs4xnjA=
+github.com/yuin/goldmark v1.8.4/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/collector/component v1.60.0 h1:LpIjHMn7OOjUsFR84ROc2kqPbP1xnKyDCGi7ZVqEaKU=
@@ -519,18 +523,26 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
 golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
 golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
+golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A=
+golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
 golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
+golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
 golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -538,18 +550,28 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
 golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75 h1:I9ygRooEYoVHV0SRNOSr/KVjTf5EeJ52BuNkVjsP2GU=
+golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg=
 golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
 golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
 golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -558,10 +580,21 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk=
 golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
+golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
+golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
+golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
+golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
+golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8=
+golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
+golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/api v0.284.0 h1:i+cKTgeQRcRySkP7QTl5PDO7/pAm8EcMFIUMlNbk4Vc=

lib/backup/s3remote/s3.go

@@ -164,6 +164,14 @@ func (fs *FS) Init(ctx context.Context) error {
 						// when using EKS Pod Identity or similar.
 						// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9280
 						"ExpiredToken": {},
+						// S3-compatible stores may return TooManyRequests instead of TooManyRequestsException.
+						// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218
+						"TooManyRequests": {},
+					},
+				}, retry.RetryableHTTPStatusCode{
+					Codes: map[int]struct{}{
+						// S3-compatible stores may return HTTP 429 for rate limiting instead of HTTP 503.
+						429: {},
 					},
 				}, retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
 					if errors.Is(err, io.ErrUnexpectedEOF) {

lib/httputil/transport_lb.go

@@ -0,0 +1,334 @@
+package httputil
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+)
+
+const (
+	brokenBackendTimeout     = 5 * time.Second
+	backendDiscoveryInterval = 10 * time.Second
+	backendDiscoveryTimeout  = 10 * time.Second
+)
+
+// NewLoadBalancerTransport returns new RoundTripper that performs least-loaded HTTP requests loadbalancing
+// based on discovered backends for the given url host
+// and update url with load-balancing prefix
+//
+// It returns origin transport and url if load-balancing is not needed for given url
+func NewLoadBalancerTransport(origin http.RoundTripper, originURL *url.URL) (http.RoundTripper, *url.URL) {
+
+	modifiedURL := *originURL
+	var discoverFunc func(context.Context, string, string) ([]*backend, error)
+	switch {
+	case strings.HasPrefix(originURL.Host, "dns+"):
+		modifiedURL.Host = modifiedURL.Host[4:]
+		discoverFunc = discoverDNSBackends
+	case strings.HasPrefix(originURL.Host, "srv+"):
+		modifiedURL.Host = modifiedURL.Host[4:]
+		discoverFunc = discoverSRVBackends
+	default:
+		return origin, originURL
+	}
+	host, port, err := net.SplitHostPort(modifiedURL.Host)
+	if err != nil {
+		host = modifiedURL.Host
+		port = "80"
+		if modifiedURL.Scheme == "https" {
+			port = "443"
+		}
+	}
+	t := &loadbalancerTransport{
+		tr:           origin,
+		host:         host,
+		port:         port,
+		discoverFunc: discoverFunc,
+	}
+	t.discoverBackends()
+	return t, &modifiedURL
+}
+
+type loadbalancerTransport struct {
+	tr   http.RoundTripper
+	host string
+	port string
+
+	discoverFunc func(context.Context, string, string) ([]*backend, error)
+
+	nextDiscoveryDeadline atomic.Uint64
+	discovering           atomic.Bool
+	dbs                   atomic.Pointer[discoveredBackends]
+}
+
+type discoveredBackends struct {
+	backends []*backend
+	// n is an atomic counter, which is used for balancing load among available backends.
+	n atomic.Uint32
+}
+
+// getLeastLoadedBackend returns least loaded backend
+// caller must release backend with backend.put() method
+func (dbs *discoveredBackends) getLeastLoadedBackend() *backend {
+	firstB := dbs.backends[0]
+	if len(dbs.backends) == 1 {
+		firstB.get()
+		return firstB
+	}
+
+	// Slow path - select other backends.
+	n := dbs.n.Add(1) - 1
+	for i := range uint32(len(dbs.backends)) {
+		idx := (n + i) % uint32(len(dbs.backends))
+		bu := dbs.backends[idx]
+		if bu.isBroken() {
+			continue
+		}
+
+		// The Load() in front of CompareAndSwap() avoids CAS overhead for items with values bigger than 0.
+		if bu.concurrentRequests.Load() == 0 && bu.concurrentRequests.CompareAndSwap(0, 1) {
+			dbs.n.CompareAndSwap(n+1, idx+1)
+			// There is no need in the call b.get(), because we already incremented b.concurrentRequests above.
+			return bu
+		}
+	}
+
+	// Slow path - return the backend with the minimum number of concurrently executed requests.
+	bMinIdx := n % uint32(len(dbs.backends))
+	minRequests := dbs.backends[bMinIdx].concurrentRequests.Load()
+	for i := uint32(1); i < uint32(len(dbs.backends)); i++ {
+		idx := (n + i) % uint32(len(dbs.backends))
+		bu := dbs.backends[idx]
+		if bu.isBroken() {
+			continue
+		}
+
+		reqs := bu.concurrentRequests.Load()
+		if reqs < minRequests || dbs.backends[bMinIdx].isBroken() {
+			bMinIdx = idx
+			minRequests = reqs
+		}
+	}
+	bMin := dbs.backends[bMinIdx]
+	if bMin.isBroken() {
+		// If all backends are broken, then returns the first backend.
+		firstB.get()
+		return firstB
+	}
+	bMin.get()
+	dbs.n.CompareAndSwap(n+1, bMinIdx+1)
+	return bMin
+}
+
+type backend struct {
+	addr               string
+	concurrentRequests atomic.Int32
+	brokenDeadline     atomic.Uint64
+}
+
+func (b *backend) get() {
+	b.concurrentRequests.Add(1)
+}
+
+func (b *backend) put() {
+	b.concurrentRequests.Add(-1)
+}
+
+func (b *backend) isBroken() bool {
+	bd := b.brokenDeadline.Load()
+	if bd == 0 {
+		return false
+	}
+	ct := fasttime.UnixTimestamp()
+	return ct < bd
+}
+
+// RoundTrip implements http.RoundTripper interface
+func (lb *loadbalancerTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	dbs := lb.getBackends()
+	if dbs == nil || len(dbs.backends) == 0 {
+		return nil, fmt.Errorf("no backends found for hostname=%q", lb.host)
+	}
+
+	maxRetries := len(dbs.backends)
+	var lastErr error
+	for range maxRetries {
+		b := dbs.getLeastLoadedBackend()
+		resp, err := lb.doRequest(r, b)
+		if err != nil {
+			ct := fasttime.UnixTimestamp()
+			brokenDeadline := ct + uint64(brokenBackendTimeout.Seconds())
+			b.brokenDeadline.Store(brokenDeadline)
+			if !netutil.IsTrivialNetworkError(err) {
+				return nil, err
+			}
+			// perform the same check for retry as http.Request.isReplayable does
+			canRetry := r.Body == nil || r.Body == http.NoBody || r.GetBody != nil
+			if !canRetry {
+				return nil, err
+			}
+			lastErr = err
+			continue
+		}
+		return resp, nil
+	}
+	return nil, fmt.Errorf("all backends are unavailable: %w", lastErr)
+}
+
+func (lb *loadbalancerTransport) doRequest(r *http.Request, b *backend) (*http.Response, error) {
+	r2 := r.Clone(r.Context())
+	if r.GetBody != nil {
+		body, err := r.GetBody()
+		if err != nil {
+			b.put()
+			return nil, err
+		}
+		r2.Body = body
+	}
+	r2.URL.Host = b.addr
+	if r2.Host == "" {
+		r2.Host = r.URL.Host
+	}
+	resp, err := lb.tr.RoundTrip(r2)
+	if err != nil {
+		b.put()
+		return nil, err
+	}
+	// wrap response body with readCloser that releases backend after Close call
+	// it's needed to properly account loaded backends at getLeastLoadedBackends
+	resp.Body = newReleaseReadCloser(resp.Body, b)
+	return resp, nil
+}
+
+func (lb *loadbalancerTransport) getBackends() *discoveredBackends {
+	ct := fasttime.UnixTimestamp()
+	deadline := lb.nextDiscoveryDeadline.Load()
+	if ct < deadline || !lb.discovering.CompareAndSwap(false, true) {
+		return lb.dbs.Load()
+	}
+	lb.discoverBackends()
+	return lb.dbs.Load()
+}
+
+func (lb *loadbalancerTransport) discoverBackends() {
+	ctx, cancel := context.WithTimeout(context.Background(), backendDiscoveryTimeout)
+	defer func() {
+		cancel()
+		ct := fasttime.UnixTimestamp()
+		nextDeadline := ct + uint64(backendDiscoveryInterval.Seconds())
+		lb.nextDiscoveryDeadline.Store(nextDeadline)
+		lb.discovering.Store(false)
+	}()
+	backends, err := lb.discoverFunc(ctx, lb.host, lb.port)
+	if err != nil {
+		logger.Errorf("cannot discover backends: %s, retry in %s", err, backendDiscoveryInterval)
+		return
+	}
+	dbs := &discoveredBackends{
+		backends: backends,
+	}
+	sort.Slice(dbs.backends, func(i, j int) bool {
+		return dbs.backends[i].addr < dbs.backends[j].addr
+	})
+
+	prevBackends := lb.dbs.Load()
+	if areBackendsEqual(prevBackends, dbs) {
+		return
+	}
+
+	lb.dbs.Store(dbs)
+}
+
+func discoverDNSBackends(ctx context.Context, host, port string) ([]*backend, error) {
+	addrs, err := netutil.Resolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return nil, fmt.Errorf("failed to lookupIPAddr for host: %q: %w", host, err)
+	}
+	backends := make([]*backend, 0, len(addrs))
+	for _, addr := range addrs {
+		if !netutil.TCP6Enabled() {
+			ip, ok := netip.AddrFromSlice(addr.IP)
+			if !ok {
+				logger.Panicf("BUG: cannot build netip Addr from slice addr: %q", addr.IP.String())
+			}
+			if !ip.Unmap().Is4() {
+				continue
+			}
+		}
+		ip := addr.IP.String()
+		if len(port) > 0 {
+			ip = net.JoinHostPort(ip, port)
+		}
+		backends = append(backends, &backend{addr: ip})
+	}
+	return backends, nil
+}
+
+func discoverSRVBackends(ctx context.Context, host, port string) ([]*backend, error) {
+	_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", host)
+	if err != nil {
+		return nil, fmt.Errorf("failed to LookupSRV records for host: %q: %w", host, err)
+	}
+	backends := make([]*backend, 0, len(addrs))
+	for _, addr := range addrs {
+		hostPort := port
+		if addr.Port > 0 {
+			hostPort = strconv.FormatUint(uint64(addr.Port), 10)
+		}
+		hostAddr := net.JoinHostPort(addr.Target, hostPort)
+		backends = append(backends, &backend{addr: hostAddr})
+	}
+	return backends, nil
+}
+
+func newReleaseReadCloser(responseBody io.ReadCloser, b *backend) io.ReadCloser {
+	return &releaseReadCloser{
+		ReadCloser: responseBody,
+		b:          b,
+	}
+}
+
+type releaseReadCloser struct {
+	io.ReadCloser
+	b        *backend
+	released atomic.Bool
+}
+
+func (rrc *releaseReadCloser) Close() error {
+	if rrc.released.CompareAndSwap(false, true) {
+		// Close method could be called multiple times
+		// and it must produce idempotent result
+		rrc.b.put()
+	}
+	return rrc.ReadCloser.Close()
+}
+
+func areBackendsEqual(left, right *discoveredBackends) bool {
+	if left == nil || right == nil {
+		return left == right
+	}
+	if len(left.backends) != len(right.backends) {
+		return false
+	}
+	for idx, leftB := range left.backends {
+		rightB := right.backends[idx]
+		if leftB.addr != rightB.addr {
+			return false
+		}
+	}
+
+	return true
+}

lib/httputil/transport_lb_test.go

@@ -0,0 +1,127 @@
+package httputil
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"sync"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+)
+
+type testRemoteServer struct {
+	mu              sync.Mutex
+	requestsPerHost map[string]int
+
+	totalRequests int
+	firstError    error
+}
+
+func (trs *testRemoteServer) RoundTrip(r *http.Request) (*http.Response, error) {
+	trs.mu.Lock()
+	if trs.firstError != nil && trs.totalRequests == 0 {
+		err := trs.firstError
+		trs.firstError = nil
+		trs.totalRequests++
+		trs.mu.Unlock()
+		return nil, err
+	}
+	trs.totalRequests++
+
+	if trs.requestsPerHost == nil {
+		trs.requestsPerHost = make(map[string]int)
+	}
+	trs.requestsPerHost[r.URL.Host]++
+	trs.mu.Unlock()
+
+	return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil
+}
+
+type testDNSResolver struct {
+	ips []net.IPAddr
+}
+
+func (tdr *testDNSResolver) LookupSRV(_ context.Context, _, _, name string) (cname string, addrs []*net.SRV, err error) {
+	return "", nil, fmt.Errorf("unexpected LookupMX call for name=%q", name)
+}
+func (tdr *testDNSResolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) {
+	return tdr.ips, nil
+}
+
+func (tdr *testDNSResolver) LookupMX(_ context.Context, name string) ([]*net.MX, error) {
+	return nil, fmt.Errorf("unexpected LookupMX call for name=%q", name)
+}
+
+func TestLoadbalancerTransport(t *testing.T) {
+	f := func(discoveredIPs []string, trs *testRemoteServer) {
+		t.Helper()
+
+		parsedIPs := make([]net.IPAddr, 0, len(discoveredIPs))
+		for _, dIP := range discoveredIPs {
+			pIP, err := netip.ParseAddr(dIP)
+			if err != nil {
+				t.Fatalf("cannot parse IP=%q: %s", dIP, err)
+			}
+			parsedIPs = append(parsedIPs, net.IPAddr{IP: pIP.AsSlice()})
+		}
+		tdr := &testDNSResolver{ips: parsedIPs}
+		originResolver := netutil.Resolver
+		defer func() { netutil.Resolver = originResolver }()
+
+		netutil.Resolver = tdr
+		requestURL, err := url.Parse("http://dns+vmsingle.example.com:8429/api/v1/write")
+		if err != nil {
+			t.Fatalf("cannot parse url: %s", err)
+		}
+		lbt, requestURL := NewLoadBalancerTransport(trs, requestURL)
+		if len(discoveredIPs) == 0 {
+			r, err := http.NewRequest(http.MethodGet, requestURL.String(), nil)
+			if err != nil {
+				t.Fatalf("cannot create http request: %s", err)
+			}
+			_, err = lbt.RoundTrip(r)
+			if err == nil {
+				t.Fatalf("expected no backends found error")
+			}
+			return
+		}
+		expectedRequestsPerHost := 2
+		for range len(discoveredIPs) * expectedRequestsPerHost {
+			r, err := http.NewRequest(http.MethodGet, requestURL.String(), nil)
+			if err != nil {
+				t.Fatalf("cannot create http request: %s", err)
+			}
+			resp, err := lbt.RoundTrip(r)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			resp.Body.Close()
+		}
+		requestsPerHost := trs.requestsPerHost
+
+		for _, dIP := range discoveredIPs {
+			expectedHostPort := net.JoinHostPort(dIP, "8429")
+			gotRequestsPerHost, ok := requestsPerHost[expectedHostPort]
+			if !ok {
+				t.Fatalf("not found expected backend request for: %q", expectedHostPort)
+			}
+			if gotRequestsPerHost != expectedRequestsPerHost {
+				t.Fatalf("unexpected requests per host:%q %d:%d (-;+)", expectedHostPort, expectedRequestsPerHost, gotRequestsPerHost)
+			}
+		}
+	}
+	trs := testRemoteServer{}
+	f([]string{"1.1.1.1"}, &trs)
+
+	trs = testRemoteServer{}
+	f([]string{"1.1.1.1", "2.2.2.2", "5.5.5.5"}, &trs)
+
+	// empty backends, expecting error
+	trs = testRemoteServer{}
+	f([]string{}, &trs)
+
+}

lib/mergeset/part.go

@@ -78,7 +78,8 @@ type part struct {
 
 	size uint64
 
-	mrs []metaindexRow
+	mrs                []metaindexRow
+	metaindexSizeBytes uint64
 
 	indexFile fs.MustReadAtCloser
 	itemsFile fs.MustReadAtCloser
@@ -131,6 +132,7 @@ func newPart(ph *partHeader, path string, size uint64, metaindexReader filestrea
 	p.path = path
 	p.size = size
 	p.mrs = mrs
+	p.metaindexSizeBytes = metaindexSizeBytes(mrs)
 
 	p.indexFile = indexFile
 	p.itemsFile = itemsFile
@@ -155,6 +157,14 @@ func (p *part) MustClose() {
 	ibSparseCache.RemoveBlocksForPart(p)
 }
 
+func metaindexSizeBytes(mrs []metaindexRow) uint64 {
+	n := uint64(cap(mrs)) * uint64(unsafe.Sizeof(metaindexRow{}))
+	for i := range mrs {
+		n += uint64(cap(mrs[i].firstItem))
+	}
+	return n
+}
+
 type indexBlock struct {
 	bhs []blockHeader
 

lib/mergeset/table.go

@@ -584,6 +584,8 @@ type TableMetrics struct {
 	PartsRefCount uint64
 
 	TooLongItemsDroppedTotal uint64
+
+	MetaindexSizeBytes uint64
 }
 
 // TotalItemsCount returns the total number of items in the table.
@@ -617,6 +619,7 @@ func (tb *Table) UpdateMetrics(m *TableMetrics) {
 		m.InmemoryBlocksCount += p.ph.blocksCount
 		m.InmemoryItemsCount += p.ph.itemsCount
 		m.InmemorySizeBytes += p.size
+		m.MetaindexSizeBytes += p.metaindexSizeBytes
 		m.PartsRefCount += uint64(pw.refCount.Load())
 	}
 
@@ -626,6 +629,7 @@ func (tb *Table) UpdateMetrics(m *TableMetrics) {
 		m.FileBlocksCount += p.ph.blocksCount
 		m.FileItemsCount += p.ph.itemsCount
 		m.FileSizeBytes += p.size
+		m.MetaindexSizeBytes += p.metaindexSizeBytes
 		m.PartsRefCount += uint64(pw.refCount.Load())
 	}
 	tb.partsLock.Unlock()

lib/persistentqueue/fastqueue.go

@@ -95,14 +95,14 @@ func mustOpenFastQueue(path, name string, opts OpenFastQueueOpts) *FastQueue {
 	}
 	fq.cond.L = &fq.mu
 	fq.lastInmemoryBlockReadTime = fasttime.UnixTimestamp()
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_bytes_pending{path=%q}`, path), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_bytes_pending{path=%q, name=%q}`, path, name), func() float64 {
 		fq.mu.Lock()
 		n := fq.pq.GetPendingBytes()
 		fq.mu.Unlock()
 		return float64(n)
 	})
 
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_free_disk_space_bytes{path=%q}`, path), func() float64 {
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_free_disk_space_bytes{path=%q, name=%q}`, path, name), func() float64 {
 		freeSpaceBytes := fs.MustGetFreeSpace(path)
 		// Limited by disk space if remoteWrite.maxDiskUsagePerURL wasn't set
 		if maxPendingBytes == 0 {

lib/persistentqueue/persistentqueue.go

@@ -52,6 +52,7 @@ type queue struct {
 	writerFlushedOffset uint64
 
 	lastMetainfoFlushTime uint64
+	hasDataToFlush        bool
 
 	blocksDropped *metrics.Counter
 	bytesDropped  *metrics.Counter
@@ -84,6 +85,7 @@ func (q *queue) mustResetFiles() {
 	}
 	q.reader.MustClose()
 	q.writer.MustClose()
+	q.hasDataToFlush = false
 	fs.MustRemovePath(q.readerPath)
 
 	q.writerOffset = 0
@@ -154,12 +156,12 @@ func tryOpeningQueue(path, name string, chunkFileSize, maxBlockSize, maxPendingB
 	q.dir = path
 	q.name = name
 
-	q.blocksDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_dropped_total{path=%q}`, path))
-	q.bytesDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_dropped_total{path=%q}`, path))
-	q.blocksWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_written_total{path=%q}`, path))
-	q.bytesWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_written_total{path=%q}`, path))
-	q.blocksRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_read_total{path=%q}`, path))
-	q.bytesRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_read_total{path=%q}`, path))
+	q.blocksDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_dropped_total{path=%q, name=%q}`, path, name))
+	q.bytesDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_dropped_total{path=%q, name=%q}`, path, name))
+	q.blocksWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_written_total{path=%q, name=%q}`, path, name))
+	q.bytesWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_written_total{path=%q, name=%q}`, path, name))
+	q.blocksRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_read_total{path=%q, name=%q}`, path, name))
+	q.bytesRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_read_total{path=%q, name=%q}`, path, name))
 
 	cleanOnError := func() {
 		if q.reader != nil {
@@ -318,6 +320,7 @@ func tryOpeningQueue(path, name string, chunkFileSize, maxBlockSize, maxPendingB
 func (q *queue) MustClose() {
 	// Close writer.
 	q.writer.MustClose()
+	q.hasDataToFlush = false
 	q.writer = nil
 
 	// Close reader.
@@ -414,7 +417,7 @@ func (q *queue) writeBlock(block []byte) error {
 	}
 	q.blocksWritten.Inc()
 	q.bytesWritten.Add(len(block))
-	return q.flushWriterMetainfoIfNeeded()
+	return q.flushBufAndMetainfoIfNeeded()
 }
 
 var writeDurationSeconds = metrics.NewFloatCounter(`vm_persistentqueue_write_duration_seconds_total`)
@@ -422,6 +425,7 @@ var writeDurationSeconds = metrics.NewFloatCounter(`vm_persistentqueue_write_dur
 func (q *queue) nextChunkFileForWrite() error {
 	// Finalize the current chunk and start new one.
 	q.writer.MustClose()
+	q.hasDataToFlush = false
 	// There is no need to do fs.MustSyncPath(q.writerPath) here,
 	// since MustClose already does this.
 	if n := q.writerOffset % q.chunkFileSize; n > 0 {
@@ -513,7 +517,7 @@ again:
 	}
 	q.blocksRead.Inc()
 	q.bytesRead.Add(int(blockLen))
-	if err := q.flushReaderMetainfoIfNeeded(); err != nil {
+	if err := q.flushBufAndMetainfoIfNeeded(); err != nil {
 		return dst, err
 	}
 	return dst, nil
@@ -566,6 +570,7 @@ func (q *queue) write(buf []byte) error {
 	}
 	q.writerLocalOffset += bufLen
 	q.writerOffset += bufLen
+	q.hasDataToFlush = true
 	return nil
 }
 
@@ -595,24 +600,16 @@ func (q *queue) checkReaderWriterOffsets() error {
 	return nil
 }
 
-func (q *queue) flushReaderMetainfoIfNeeded() error {
+func (q *queue) flushBufAndMetainfoIfNeeded() error {
 	t := fasttime.UnixTimestamp()
 	if t == q.lastMetainfoFlushTime {
 		return nil
 	}
-	if err := q.flushMetainfo(); err != nil {
-		return fmt.Errorf("cannot flush metainfo: %w", err)
+	if q.hasDataToFlush {
+		q.writer.MustFlush(true)
+		q.writerFlushedOffset = q.writerOffset
+		q.hasDataToFlush = false
 	}
-	q.lastMetainfoFlushTime = t
-	return nil
-}
-
-func (q *queue) flushWriterMetainfoIfNeeded() error {
-	t := fasttime.UnixTimestamp()
-	if t == q.lastMetainfoFlushTime {
-		return nil
-	}
-	q.writer.MustFlush(true)
 	if err := q.flushMetainfo(); err != nil {
 		return fmt.Errorf("cannot flush metainfo: %w", err)
 	}
@@ -651,7 +648,7 @@ func (mi *metainfo) WriteToFile(path string) error {
 	if err != nil {
 		return fmt.Errorf("cannot marshal persistent queue metainfo %#v: %w", mi, err)
 	}
-	fs.MustWriteSync(path, data)
+	fs.MustWriteAtomic(path, data, true)
 	return nil
 }