introduce rawRowsFlusher interface to decouple rawRowsShards and partition types

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/codeql-analysis-go.yml

@@ -54,14 +54,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           category: 'language:go'

.golangci.yml

@@ -23,3 +23,4 @@ linters:
         text: 'SA(4003|1019|5011):'
     paths:
       - ^app/vmui/
+      - app/vmui/packages/vmui/node_modules/

Makefile

@@ -447,7 +447,7 @@ vet:
 	go vet ./app/...
 	go vet ./apptest/...
 
-check-all: fmt vet golangci-lint govulncheck
+check-all: fmt vet golangci-lint
 
 clean-checkers: remove-golangci-lint remove-govulncheck
 
@@ -471,8 +471,9 @@ test-full-386:
 
 apptest:
 	$(MAKE) victoria-metrics-race vmagent-race vmalert-race vmauth-race vmctl-race vmbackup-race vmrestore-race
-	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+	go test ./apptest/... -skip="^Test(Cluster|Mixed|Legacy).*"
 
+# App tests for legacy indexDB
 apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
@@ -489,6 +490,22 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
+# App tests for mixed setups where vmsingle and vmcluster coexist.
+apptest-mixed: victoria-metrics-race
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.147.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VMSELECT_PATH=$${DIR}/vmselect-prod \
+	go test ./apptest/tests -run="^TestMixed.*"
+
 benchmark:
 	go test -run=NO_TESTS -bench=. ./lib/...
 	go test -run=NO_TESTS -bench=. ./app/...

app/victoria-metrics/main.go

@@ -89,7 +89,7 @@ func main() {
 	}
 	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
-	vmstorage.Init(*vmselectMaxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	vmstorage.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
 	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()

app/vmagent/influx/request_handler.go

@@ -51,7 +51,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vmagent/promremotewrite/request_handler.go

@@ -35,6 +35,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 }
 
 func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
+	if len(extraLabels) == 0 && !prommetadata.IsEnabled() && at == nil {
+		return insertRowsFast(at, timeseries)
+	}
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -102,3 +105,17 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.Met
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }
+
+func insertRowsFast(at *auth.Token, timeseries []prompb.TimeSeries) error {
+	rowsTotal := 0
+	for i := range timeseries {
+		rowsTotal += len(timeseries[i].Samples)
+	}
+	wr := &prompb.WriteRequest{Timeseries: timeseries}
+	if !remotewrite.TryPush(at, wr) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/remotewrite.go

@@ -12,19 +12,18 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/cespare/xxhash/v2"
-
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mdx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -106,6 +105,9 @@ var (
 		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
 	disableMetadataPerURL = flagutil.NewArrayBool("remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
+
+	enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
 )
 
 var (
@@ -162,8 +164,8 @@ func InitSecretFlags() {
 }
 
 var (
-	shardByURLLabelsMap       map[string]struct{}
-	shardByURLIgnoreLabelsMap map[string]struct{}
+	shardByURLLabelsFilter       []string
+	shardByURLIgnoreLabelsFilter []string
 )
 
 // Init initializes remotewrite.
@@ -210,8 +212,8 @@ func Init() {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
 	}
-	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
-	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
+	shardByURLLabelsFilter = slices.Clone(*shardByURLLabels)
+	shardByURLIgnoreLabelsFilter = slices.Clone(*shardByURLIgnoreLabels)
 
 	initLabelsGlobal()
 
@@ -307,6 +309,10 @@ func initRemoteWriteCtxs(urls []string) {
 	}
 	fs.RegisterPathFsMetrics(*tmpDataPath)
 
+	if slices.Contains(*enableMdx, true) && *shardByURL {
+		logger.Fatalf("-remoteWrite.mdx.enable and -remoteWrite.shardByURL cannot be set to true simultaneously.")
+	}
+
 	if *shardByURL {
 		consistentHashNodes := make([]string, 0, len(urls))
 		for i, url := range urls {
@@ -565,6 +571,14 @@ func tryPushMetadataToRemoteStorages(at *auth.Token, rwctxs []*remoteWriteCtx, m
 			mm.ProjectID = at.ProjectID
 		}
 	}
+	tmp := mms[:0]
+	for _, mm := range mms {
+		if timeserieslimits.IsMetricMetadataExceeding(&mm) {
+			continue
+		}
+		tmp = append(tmp, mm)
+	}
+	mms = tmp
 	// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
 	// Since metadata is usually small and there is no guarantee that metadata can be sent to
 	// the same remote storage with the corresponding metrics.
@@ -698,18 +712,18 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
 
 	for _, ts := range tssBlock {
 		hashLabels := ts.Labels
-		if len(shardByURLLabelsMap) > 0 {
+		if len(shardByURLLabelsFilter) > 0 {
 			hashLabels = tmpLabels.Labels[:0]
 			for _, label := range ts.Labels {
-				if _, ok := shardByURLLabelsMap[label.Name]; ok {
+				if slices.Contains(shardByURLLabelsFilter, label.Name) {
 					hashLabels = append(hashLabels, label)
 				}
 			}
 			tmpLabels.Labels = hashLabels
-		} else if len(shardByURLIgnoreLabelsMap) > 0 {
+		} else if len(shardByURLIgnoreLabelsFilter) > 0 {
 			hashLabels = tmpLabels.Labels[:0]
 			for _, label := range ts.Labels {
-				if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
+				if !slices.Contains(shardByURLIgnoreLabelsFilter, label.Name) {
 					hashLabels = append(hashLabels, label)
 				}
 			}
@@ -810,34 +824,26 @@ var (
 // it omits the '=' separator between label name and value for backward compatibility.
 // Changing it would re-shard all series across remoteWrite targets.
 func getLabelsHashForShard(labels []prompb.Label) uint64 {
-	bb := labelsHashBufPool.Get()
-	b := bb.B[:0]
+	var d xxhash.Digest
+	d.Reset()
 	for _, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, label.Value...)
+		_, _ = d.WriteString(label.Name)
+		_, _ = d.WriteString(label.Value)
 	}
-	h := xxhash.Sum64(b)
-	bb.B = b
-	labelsHashBufPool.Put(bb)
-	return h
+	return d.Sum64()
 }
 
 func getLabelsHash(labels []prompb.Label) uint64 {
-	bb := labelsHashBufPool.Get()
-	b := bb.B[:0]
+	var d xxhash.Digest
+	d.Reset()
 	for _, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, '=')
-		b = append(b, label.Value...)
+		_, _ = d.WriteString(label.Name)
+		_, _ = d.WriteString("=")
+		_, _ = d.WriteString(label.Value)
 	}
-	h := xxhash.Sum64(b)
-	bb.B = b
-	labelsHashBufPool.Put(bb)
-	return h
+	return d.Sum64()
 }
 
-var labelsHashBufPool bytesutil.ByteBufferPool
-
 func logSkippedSeries(labels []prompb.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
@@ -862,6 +868,7 @@ type remoteWriteCtx struct {
 
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
+	mdxFilter    *mdx.Filter
 
 	streamAggrKeepInput bool
 	streamAggrDropInput bool
@@ -876,6 +883,7 @@ type remoteWriteCtx struct {
 
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
+	mdxRowsPreserved       *metrics.Counter
 
 	pushFailures                 *metrics.Counter
 	metadataDroppedOnPushFailure *metrics.Counter
@@ -972,7 +980,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	for i := range pss {
 		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
 	}
-
 	rwctx := &remoteWriteCtx{
 		idx:            argIdx,
 		fq:             fq,
@@ -989,6 +996,16 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	}
 	rwctx.initStreamAggrConfig()
 
+	if enableMdx.GetOptionalArg(argIdx) {
+		mdxFilter := mdx.NewFilter()
+		rwctx.mdxFilter = mdxFilter
+		rwctx.mdxRowsPreserved = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_mdx_rows_preserved_total{path=%q,url=%q}`, queuePath, sanitizedURL))
+		_ = metrics.NewGauge(fmt.Sprintf(`vmagent_remotewrite_mdx_tracked_instances{path=%q,url=%q}`, queuePath, sanitizedURL), func() float64 {
+			return float64(mdxFilter.VMInstancesCount())
+		})
+
+	}
+
 	return rwctx
 }
 
@@ -1002,6 +1019,11 @@ func (rwctx *remoteWriteCtx) MustStop() {
 		rwctx.deduplicator.MustStop()
 		rwctx.deduplicator = nil
 	}
+	if rwctx.mdxFilter != nil {
+		rwctx.mdxFilter.MustStop()
+		rwctx.mdxFilter = nil
+		rwctx.mdxRowsPreserved = nil
+	}
 
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
@@ -1017,6 +1039,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 	rwctx.rowsPushedAfterRelabel = nil
 	rwctx.rowsDroppedByRelabel = nil
+
 }
 
 // TryPushTimeSeries sends tss series to the configured remote write endpoint
@@ -1024,16 +1047,41 @@ func (rwctx *remoteWriteCtx) MustStop() {
 // TryPushTimeSeries doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
 func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	var rctx *relabelCtx
+	var mctx *mdx.Ctx
 	var v *[]prompb.TimeSeries
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if mctx != nil {
+			mdx.PutContext(mctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v := tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
+	if rwctx.mdxFilter != nil {
+		mctx = mdx.GetContext()
+		// Make a copy of tss before applying relabeling in order to prevent
+		// from affecting time series for other remoteWrite.mdx configs.
+		copyTimeSeriesIfNeeded()
+		tss = rwctx.mdxFilter.Filter(mctx, tss)
+		if len(tss) == 0 {
+			return true
+		}
+		rowsCount := getRowsCount(tss)
+		rwctx.mdxRowsPreserved.Add(rowsCount)
+	}
+
 	// Apply relabeling
 	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
@@ -1043,8 +1091,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 		// from affecting time series for other remoteWrite.url configs.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, pcs)
 		rowsCountAfterRelabel := getRowsCount(tss)
@@ -1062,8 +1109,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 			if rctx == nil {
 				rctx = getRelabelCtx()
 				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
-				tss = append(*v, tss...)
+				copyTimeSeriesIfNeeded()
 			}
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		} else if rwctx.streamAggrDropInput {
@@ -1071,8 +1117,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 			if rctx == nil {
 				rctx = getRelabelCtx()
 				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
-				tss = append(*v, tss...)
+				copyTimeSeriesIfNeeded()
 			}
 			tss = dropUnaggregatedSeries(tss, matchIdxs.B)
 		}
@@ -1191,15 +1236,6 @@ func getRowsCount(tss []prompb.TimeSeries) int {
 	}
 	return rowsCount
 }
-
-func newMapFromStrings(a []string) map[string]struct{} {
-	m := make(map[string]struct{}, len(a))
-	for _, s := range a {
-		m[s] = struct{}{}
-	}
-	return m
-}
-
 func getMaxHourlySeries() int {
 	limit := *maxHourlySeries
 	if limit == -1 || limit > math.MaxInt32 {

app/vmalert-tool/unittest/recording.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"reflect"
 	"sort"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -94,10 +93,3 @@ Outer:
 	}
 	return
 }
-
-func durationToTime(pd *promutil.Duration) time.Time {
-	if pd == nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(pd.Duration().Milliseconds())
-}

app/vmalert-tool/unittest/unittest.go

@@ -44,12 +44,19 @@ import (
 var (
 	storagePath    string
 	httpListenAddr string
-	// insert series from 1970-01-01T00:00:00
-	testStartTime          = time.Unix(0, 0).UTC()
+	// Insert series from 2000-01-01T00:00:00.
+	testStartTime          = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
 
+func durationToTime(pd *promutil.Duration) time.Time {
+	if pd == nil {
+		return testStartTime
+	}
+	return testStartTime.Add(pd.Duration())
+}
+
 const (
 	testStoragePath = "vmalert-unittest"
 )
@@ -282,7 +289,8 @@ func processFlags() {
 
 func setUp() {
 	const maxConcurrentRequests = 4
-	vmstorage.Init(maxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
+	maxQueueDuration := 5 * time.Second
+	vmstorage.Init(maxConcurrentRequests, maxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	readyCheckFunc := func() bool {

app/vmalert/config/config_test.go

@@ -145,10 +145,10 @@ func TestRuleValidate(t *testing.T) {
 }
 
 func TestGroupValidate_Failure(t *testing.T) {
-	f := func(group *Group, validateExpressions bool, errStrExpected string) {
+	f := func(data []byte, validateExpressions bool, errStrExpected string) {
 		t.Helper()
 
-		err := group.Validate(nil, validateExpressions)
+		_, err := parse(map[string][]byte{"test.yaml": data}, nil, validateExpressions)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -158,275 +158,238 @@ func TestGroupValidate_Failure(t *testing.T) {
 		}
 	}
 
-	f(&Group{}, false, "group name must be set")
+	f([]byte(`
+groups:
+- name: ""
+`), false, "group name must be set")
 
-	f(&Group{
-		Name: "both record and alert are not set",
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutil.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
+	f([]byte(`
+groups:
+- name: both record and alert are not set
+  rules:
+  - expr: "sum(up == 0 ) by (host)"
+    for: 10ms
+  - expr: "sumSeries(time('foo.bar',10))"
+`), false, "invalid rule")
 
-	f(&Group{
-		Name:     "negative interval",
-		Interval: promutil.NewDuration(-1),
-	}, false, "interval shouldn't be lower than 0")
+	f([]byte(`
+groups:
+- name: negative interval
+  interval: -1ms
+`), false, "interval shouldn't be lower than 0")
 
-	f(&Group{
-		Name:       "too big eval_offset",
-		Interval:   promutil.NewDuration(time.Minute),
-		EvalOffset: promutil.NewDuration(2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
+	f([]byte(`
+groups:
+- name: too big eval_offset
+  interval: 1m
+  eval_offset: 2m
+`), false, "eval_offset should be smaller than interval")
 
-	f(&Group{
-		Name:       "too big negative eval_offset",
-		Interval:   promutil.NewDuration(time.Minute),
-		EvalOffset: promutil.NewDuration(-2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
+	f([]byte(`
+groups:
+- name: too big negative eval_offset
+  interval: 1m
+  eval_offset: -2m
+`), false, "eval_offset should be smaller than interval")
 
-	limit := -1
-	f(&Group{
-		Name:  "wrong limit",
-		Limit: &limit,
-	}, false, "invalid limit")
+	f([]byte(`
+groups:
+- name: wrong limit
+  limit: -1
+`), false, "invalid limit")
 
-	f(&Group{
-		Name:        "wrong concurrency",
-		Concurrency: -1,
-	}, false, "invalid concurrency")
+	f([]byte(`
+groups:
+- name: wrong concurrency
+  concurrency: -1
+`), false, "invalid concurrency")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-			},
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-			},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+  - alert: alert
+    expr: up == 1
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - record: record
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+  - record: record
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, "duplicate")
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test thanos",
-		Type: NewRawType("thanos"),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, true, "unknown datasource type")
+	f([]byte(`
+groups:
+- name: test thanos
+  type: thanos
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: "{{ value|query }}"
+`), true, "unknown datasource type")
 
 	// validate expressions
-	f(&Group{
-		Name: "test prometheus expr",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "bad MetricsQL expr")
+	f([]byte(`
+groups:
+- name: test prometheus expr
+  type: prometheus
+  rules:
+  - record: record
+    expr: "up | 0"
+`), true, "bad MetricsQL expr")
 
-	f(&Group{
-		Name: "test graphite expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "some-description",
-			}},
-		},
-	}, true, "bad GraphiteQL expr")
+	f([]byte(`
+groups:
+- name: test graphite expr
+  type: graphite
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: some-description
+`), true, "bad GraphiteQL expr")
 
-	f(&Group{
-		Name: "test vlogs expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "stats count(*) as requests"},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs expr
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: "stats count(*) as requests"
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test vlogs expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs expr multipart
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test graphite with prometheus expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				ID:     1,
-				Expr:   "sumSeries(time('foo.bar',10))",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Record: "r2",
-				ID:     2,
-				Expr:   "sum(up == 0 ) by (host)",
-			},
-		},
-	}, true, "bad GraphiteQL expr")
+	f([]byte(`
+groups:
+- name: test graphite with prometheus expr
+  type: graphite
+  rules:
+  - record: r1
+    expr: "sumSeries(time('foo.bar',10))"
+    for: 10ms
+  - record: r2
+    expr: "sum(up == 0 ) by (host)"
+`), true, "bad GraphiteQL expr")
 
-	f(&Group{
-		Name: "test vlogs with prometheus exp",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				Expr:   "sum(up == 0 ) by (host)",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-		},
-	}, true, "bad LogsQL expr")
+	f([]byte(`
+groups:
+- name: test vlogs with prometheus expr
+  type: vlogs
+  rules:
+  - record: r1
+    expr: "sum(up == 0 ) by (host)"
+    for: 10ms
+`), true, "bad LogsQL expr")
 
-	f(&Group{
-		Name: "test prometheus with vlogs exp",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{
-				Record: "r1",
-				Expr:   "* | stats by (path) count()",
-				For:    promutil.NewDuration(10 * time.Millisecond),
-			},
-		},
-	}, true, "bad MetricsQL expr")
+	f([]byte(`
+groups:
+- name: test prometheus with vlogs expr
+  type: prometheus
+  rules:
+  - record: r1
+    expr: "* | stats by (path) count()"
+    for: 10ms
+`), true, "bad MetricsQL expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {
-	f := func(group *Group, validateAnnotations, validateExpressions bool) {
+	f := func(data []byte, validateAnnotations, validateExpressions bool) {
 		t.Helper()
 
 		var validateTplFn ValidateTplFn
 		if validateAnnotations {
 			validateTplFn = notifier.ValidateTemplates
 		}
-		err := group.Validate(validateTplFn, validateExpressions)
+		_, err := parse(map[string][]byte{"test.yaml": data}, validateTplFn, validateExpressions)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	}
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, false, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - record: record
+    expr: "up | 0"
+`), false, false)
 
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": "{{ value|query }}",
-				},
-			},
-		},
-	}, false, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "{{ value|query }}"
+`), false, false)
 
 	// validate annotations
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": `
-{{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
-  {{ . | first | value | humanize1024 }}B
-{{ end }}`,
-				},
-			},
-		},
-	}, true, false)
+	f([]byte(`
+groups:
+- name: test
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      summary: "\n{{ with printf \"node_memory_MemTotal{job='node',instance='%s'}\" \"localhost\" | query }}\n  {{ . | first | value | humanize1024 }}B\n{{ end }}"
+`), true, false)
 
 	// validate expressions
-	f(&Group{
-		Name: "test prometheus",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, true)
-	f(&Group{
-		Name: "test victorialogs",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: " _time: 1m | stats count(*) as requests", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, true)
+	f([]byte(`
+groups:
+- name: test prometheus
+  type: prometheus
+  rules:
+  - alert: alert
+    expr: up == 1
+    labels:
+      description: "{{ value|query }}"
+`), false, true)
+
+	f([]byte(`
+groups:
+- name: test victorialogs
+  type: vlogs
+  rules:
+  - alert: alert
+    expr: " _time: 1m | stats count(*) as requests"
+    labels:
+      description: "{{ value|query }}"
+`), false, true)
 }
 
 func TestHashRule_NotEqual(t *testing.T) {

app/vmalert/rule/group.go

@@ -97,6 +97,7 @@ type groupMetrics struct {
 	iterationMissed   *metrics.Counter
 	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
+	iterationLimit    *metrics.Gauge
 }
 
 // merges group rule labels into result map
@@ -336,6 +337,12 @@ func (g *Group) Init() {
 		i := g.Interval.Seconds()
 		return i
 	})
+	g.metrics.iterationLimit = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_rule_group_results_limit{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		limit := g.Limit
+		g.mu.RUnlock()
+		return float64(limit)
+	})
 	for i := range g.Rules {
 		g.Rules[i].registerMetrics(g.metrics.set)
 	}

app/vmauth/auth_config.go

@@ -118,9 +118,10 @@ type AccessLogFilters struct {
 }
 
 func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
-	if ui.AccessLog == nil {
+	if ui == nil || ui.AccessLog == nil {
 		return
 	}
+
 	filters := ui.AccessLog.Filters
 	if filters != nil && len(filters.SkipStatusCodes) > 0 {
 		if slices.Contains(filters.SkipStatusCodes, statusCode) {
@@ -134,6 +135,17 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int,
 		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
 }
 
+// hasAnyURLs reports whether ui has at least one backend URL route configured.
+// It is used only for unauthorized_user config section, since other users
+// must always have either URLPrefix or URLMaps set.
+func (ui *UserInfo) hasAnyURLs() bool {
+	if ui == nil {
+		return false
+	}
+
+	return ui.URLPrefix != nil || len(ui.URLMaps) > 0 || ui.DefaultURL != nil
+}
+
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
 	RequestHeaders     []*Header `yaml:"headers,omitempty"`
@@ -983,8 +995,11 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
 			return nil, err
 		}
-		if err := ui.initURLs(); err != nil {
-			return nil, err
+
+		if ui.hasAnyURLs() {
+			if err := ui.initURLs(); err != nil {
+				return nil, err
+			}
 		}
 
 		metricLabels, err := ui.getMetricLabels()

app/vmauth/example_config.yml

@@ -140,6 +140,18 @@ users:
   - "ProjectID: {{.MetricsProjectID}}"
   url_prefix: "http://vminsert:8480/insert/prometheus"
 
+# JWT-based routing that relies solely on custom claims.
+# The `vm_access` claim is missing, default value will be used.
+# e.g. {"role": "admin"}.
+- name: jwt-custom-claims
+  jwt:
+    skip_verify: true
+    vm_default_access_claim:
+        metrics_account_id: 1
+    match_claims:
+      role: admin
+  url_prefix: "http://vmselect-admin:8481/select/0/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -65,6 +65,8 @@ type JWTConfig struct {
 	MatchClaims       map[string]string `yaml:"match_claims,omitempty"`
 	parsedMatchClaims []*jwt.Claim
 
+	DefaultVMAccessClaim *jwt.VMAccessClaim `yaml:"default_vm_access_claim,omitempty"`
+
 	// verifierPool is used to verify JWT tokens.
 	// It is initialized from PublicKeys and/or PublicKeyFiles.
 	// In this case, it is initialized once at config reload and never updated until next reload
@@ -432,7 +434,6 @@ func validateJWTPlaceholdersForURL(up *URLPrefix, isAllowed bool) error {
 			}
 			if strings.Contains(p, placeholderPrefix) {
 				return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
-
 			}
 		}
 		for param, values := range bu.Query() {
@@ -487,7 +488,6 @@ func hasAnyPlaceholders(u *url.URL) bool {
 				return true
 			}
 		}
-
 	}
 	return false
 }

app/vmauth/main.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/textproto"
@@ -31,6 +32,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 )
 
 var (
@@ -173,11 +175,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
-		if ui != nil {
+		if ui.hasAnyURLs() {
 			processUserRequest(w, r, ui, nil)
 			return true
 		}
 
+		ui.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 		handleMissingAuthorizationError(w)
 		return true
 	}
@@ -191,17 +194,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
 		}
 		defer putToken(tkn)
-		processUserRequest(w, r, ui, tkn)
-		return true
+		// Call processUserRequest only if the token contains the vm_access claim
+		// or a default claim is configured; otherwise fall through to unauthorized_user.
+		if tkn.HasVMAccessClaim() || ui.JWT.DefaultVMAccessClaim != nil {
+			processUserRequest(w, r, ui, tkn)
+			return true
+		}
 	}
 
 	uu := authConfig.Load().UnauthorizedUser
-	if uu != nil {
+	if uu.hasAnyURLs() {
 		processUserRequest(w, r, uu, nil)
 		return true
 	}
 
 	invalidAuthTokenRequests.Inc()
+	slowdownUnauthorizedResponse(r)
+	uu.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 	if *logInvalidAuthTokens {
 		err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 		err = &httpserver.ErrorWithStatusCode{
@@ -424,8 +433,12 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		}
 		targetURL := bu.url
 		if tkn != nil {
+			vmac := tkn.VMAccess()
+			if !tkn.HasVMAccessClaim() {
+				vmac = ui.JWT.DefaultVMAccessClaim
+			}
 			// for security reasons allow templating only for configured url values and headers
-			targetURL, hc = replaceJWTPlaceholders(bu, hc, tkn.VMAccess())
+			targetURL, hc = replaceJWTPlaceholders(bu, hc, vmac)
 		}
 		if isDefault {
 			// Don't change path and add request_path query param for default route.
@@ -881,3 +894,20 @@ func debugInfo(u *url.URL, r *http.Request) string {
 	fmt.Fprint(s, ")")
 	return s.String()
 }
+
+// slowdownUnauthorizedResponse adds a random delay in the [2..3] seconds range before returning an unauthorized response.
+// This reduces the effectiveness of brute-force.
+//
+// Recommended by OWASP Top10:
+// https://owasp.org/Top10/2025/A07_2025-Authentication_Failures
+func slowdownUnauthorizedResponse(r *http.Request) {
+
+	d := 2*time.Second + time.Duration(rand.IntN(1000))*time.Millisecond
+	t := timerpool.Get(d)
+
+	select {
+	case <-t.C:
+	case <-r.Context().Done():
+	}
+	timerpool.Put(t)
+}

app/vmauth/main_test.go

@@ -739,6 +739,12 @@ users:
 		"vm_access": map[string]any{},
 	}, false)
 
+	// token without vm_access claim, but with a custom claim usable for routing
+	roleToken := genToken(t, map[string]any{
+		"exp":  time.Now().Add(10 * time.Minute).Unix(),
+		"role": "admin",
+	}, true)
+
 	fullToken := genToken(t, map[string]any{
 		"exp": time.Now().Add(10 * time.Minute).Unix(),
 		"vm_access": map[string]any{
@@ -779,6 +785,45 @@ statusCode=401
 Unauthorized`
 	f(simpleCfgStr, request, responseExpected)
 
+	// token without vm_access claim should fall through to unauthorized_user
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
+	responseExpected = `
+statusCode=200
+path: /bar/abc
+query:
+headers:`
+	f(fmt.Sprintf(`
+unauthorized_user:
+  url_prefix: {BACKEND}/bar
+users:
+- jwt:
+    public_keys:
+    - %q
+    match_claims:
+      role: admin
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
+
+	// token without vm_access claim is accepted when default_vm_access_claim configured
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+roleToken)
+	responseExpected = `
+statusCode=200
+path: /foo/abc
+query:
+headers:`
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+    default_vm_access_claim:
+      metrics_account_id: 10
+      metrics_project_id: 10
+    match_claims:
+      role: admin
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
+
 	// expired token
 	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
 	request.Header.Set(`Authorization`, `Bearer `+expiredToken)

app/vmctl/vm/vm.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -111,7 +112,7 @@ func AddExtraLabelsToImportPath(path string, extraLabels []string) (string, erro
 		if strings.Contains(dst, "?") {
 			separator = "&"
 		}
-		dst += fmt.Sprintf("%sextra_label=%s", separator, extraLabel)
+		dst += fmt.Sprintf("%sextra_label=%s", separator, url.QueryEscape(extraLabel))
 	}
 	return dst, nil
 }

app/vmctl/vm/vm_test.go

@@ -33,11 +33,14 @@ func TestAddExtraLabelsToImportPath_Success(t *testing.T) {
 	f("/api/v1/import", nil, "/api/v1/import")
 
 	// ok one extra label
-	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance=host-1")
+	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance%3Dhost-1")
 
 	// ok two extra labels
-	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
 
 	// ok two extra with exist param
-	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance=host-2&extra_label=job=vmagent")
+	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
+
+	// ok special chars in label value
+	f("/api/v1/import", []string{"team=a&b"}, "/api/v1/import?extra_label=team%3Da%26b")
 }

app/vminsert/common/insert_ctx.go

@@ -175,13 +175,19 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmpbs))
-	for idx, mmpb := range mmpbs {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmpbs {
+		if timeserieslimits.IsMetricMetadataExceeding(&mmpb) {
+			continue
+		}
+		mm := &mms[cnt]
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
 		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
 
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
@@ -201,14 +207,19 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
 	}
 	mms := ctx.mms
 	mms = slicesutil.SetLength(mms, len(mmps))
-	for idx, mmpb := range mmps {
-		mm := &mms[idx]
+	var cnt int
+	for _, mmpb := range mmps {
+		mm := &mms[cnt]
+		if timeserieslimits.IsPrometheusMetadataExceeding(&mmpb) {
+			continue
+		}
 		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
+		cnt++
 	}
+	mms = mms[:cnt]
 	ctx.mms = mms
-
 	err := vmstorage.VMInsertAPI.WriteMetadata(mms)
 	if err != nil {
 		return &httpserver.ErrorWithStatusCode{

app/vminsert/influx/request_handler.go

@@ -49,7 +49,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	}
 	q := req.URL.Query()
 	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
 	db := q.Get("db")
 	encoding := req.Header.Get("Content-Encoding")
 	isStreamMode := req.Header.Get("Stream-Mode") == "1"

app/vminsert/main.go

@@ -68,9 +68,11 @@ var (
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented.")
+	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
+	maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented. "+
+		"Value must be in range 1..65535.")
 )
 
 var (
@@ -106,7 +108,7 @@ func Init() {
 	promscrape.Init(func(_ *auth.Token, wr *prompb.WriteRequest) {
 		prompush.Push(wr)
 	})
-	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
+	timeserieslimits.MustInit(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 }
 
 // Stop stops vminsert.

app/vmselect/prometheus/prometheus.go

@@ -956,6 +956,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
 	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	mayCache := !httputil.GetBool(r, "nocache")
+	optimizeRepeatedBinaryOpSubexprs := httputil.GetBool(r, "optimize_repeated_binary_op_subexprs")
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
 		return err
@@ -977,18 +978,19 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	}
 
 	ec := &promql.EvalConfig{
-		Start:               start,
-		End:                 end,
-		Step:                step,
-		MaxPointsPerSeries:  *maxPointsPerTimeseries,
-		MaxSeries:           0, // let vmstorage use maxUniqueTimeseries by default
-		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
-		Deadline:            deadline,
-		MayCache:            mayCache,
-		LookbackDelta:       lookbackDelta,
-		RoundDigits:         getRoundDigits(r),
-		EnforcedTagFilterss: etfs,
-		CacheTagFilters:     etfs,
+		Start:                            start,
+		End:                              end,
+		Step:                             step,
+		MaxPointsPerSeries:               *maxPointsPerTimeseries,
+		MaxSeries:                        0, // let vmstorage use maxUniqueTimeseries by default
+		QuotedRemoteAddr:                 httpserver.GetQuotedRemoteAddr(r),
+		Deadline:                         deadline,
+		MayCache:                         mayCache,
+		OptimizeRepeatedBinaryOpSubexprs: optimizeRepeatedBinaryOpSubexprs,
+		LookbackDelta:                    lookbackDelta,
+		RoundDigits:                      getRoundDigits(r),
+		EnforcedTagFilterss:              etfs,
+		CacheTagFilters:                  etfs,
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},

app/vmselect/promql/eval.go

@@ -132,6 +132,9 @@ type EvalConfig struct {
 	// Whether the response can be cached.
 	MayCache bool
 
+	// Whether repeated cacheable binary op subexpressions can be optimized.
+	OptimizeRepeatedBinaryOpSubexprs bool
+
 	// LookbackDelta is analog to `-query.lookback-delta` from Prometheus.
 	LookbackDelta int64
 
@@ -171,6 +174,7 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.MaxPointsPerSeries = src.MaxPointsPerSeries
 	ec.Deadline = src.Deadline
 	ec.MayCache = src.MayCache
+	ec.OptimizeRepeatedBinaryOpSubexprs = src.OptimizeRepeatedBinaryOpSubexprs
 	ec.LookbackDelta = src.LookbackDelta
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
@@ -467,83 +471,178 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
 }
 
 func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
-	if !canPushdownCommonFilters(be) {
-		// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
-		// from exprFirst to exprSecond.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
-		qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
-		defer qt.Done()
-		var wg sync.WaitGroup
-
-		var tssFirst []*timeseries
-		var errFirst error
-		qtFirst := qt.NewChild("expr1")
-		wg.Go(func() {
-			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
-			qtFirst.Done()
-		})
-
-		var tssSecond []*timeseries
-		var errSecond error
-		qtSecond := qt.NewChild("expr2")
-		wg.Go(func() {
-			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
-			qtSecond.Done()
-		})
-
-		wg.Wait()
-		if errFirst != nil {
-			return nil, nil, errFirst
+	if canPushdownCommonFilters(be) {
+		// Execute binary operation in the following way:
+		//
+		// 1) execute the exprFirst
+		// 2) get common label filters for series returned at step 1
+		// 3) push down the found common label filters to exprSecond. This filters out unneeded series
+		//    during exprSecond execution instead of spending compute resources on extracting and processing these series
+		//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
+		// 4) execute the exprSecond with possible additional filters found at step 3
+		//
+		// Typical use cases:
+		// - Kubernetes-related: show pod creation time with the node name:
+		//
+		//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
+		//
+		//   Without the optimization `kube_pod_info` would select and spend compute resources
+		//   for more time series than needed. The selected time series would be dropped later
+		//   when matching time series on the right and left sides of binary operand.
+		//
+		// - Generic alerting queries, which rely on `info` metrics.
+		//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
+		//
+		// - Queries, which get additional labels from `info` metrics.
+		//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
+		tssFirst, err := evalExpr(qt, ec, exprFirst)
+		if err != nil {
+			return nil, nil, err
 		}
-		if errSecond != nil {
-			return nil, nil, errSecond
+		if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
+			// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
+			// since the "exprFirst op exprSecond" would return an empty result in any case.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
+			return nil, nil, nil
+		}
+		lfs := getCommonLabelFilters(tssFirst)
+		lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
+		exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
+		tssSecond, err := evalExpr(qt, ec, exprSecond)
+		if err != nil {
+			return nil, nil, err
 		}
 		return tssFirst, tssSecond, nil
 	}
 
-	// Execute binary operation in the following way:
-	//
-	// 1) execute the exprFirst
-	// 2) get common label filters for series returned at step 1
-	// 3) push down the found common label filters to exprSecond. This filters out unneeded series
-	//    during exprSecond execution instead of spending compute resources on extracting and processing these series
-	//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
-	// 4) execute the exprSecond with possible additional filters found at step 3
-	//
-	// Typical use cases:
-	// - Kubernetes-related: show pod creation time with the node name:
-	//
-	//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
-	//
-	//   Without the optimization `kube_pod_info` would select and spend compute resources
-	//   for more time series than needed. The selected time series would be dropped later
-	//   when matching time series on the right and left sides of binary operand.
-	//
-	// - Generic alerting queries, which rely on `info` metrics.
-	//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
-	//
-	// - Queries, which get additional labels from `info` metrics.
-	//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
-	tssFirst, err := evalExpr(qt, ec, exprFirst)
-	if err != nil {
-		return nil, nil, err
+	// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
+	// in exprFirst and exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575
+	if shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
+		qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
+		defer qt.Done()
+
+		qtFirst := qt.NewChild("expr1")
+		tssFirst, err := evalExpr(qtFirst, ec, exprFirst)
+		qtFirst.Done()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		qtSecond := qt.NewChild("expr2")
+		tssSecond, err := evalExpr(qtSecond, ec, exprSecond)
+		qtSecond.Done()
+		if err != nil {
+			return nil, nil, err
+		}
+		return tssFirst, tssSecond, nil
 	}
-	if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
-		// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
-		// since the "exprFirst op exprSecond" would return an empty result in any case.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
-		return nil, nil, nil
+
+	// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
+	// from exprFirst to exprSecond.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
+	qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
+	defer qt.Done()
+	var wg sync.WaitGroup
+
+	var tssFirst []*timeseries
+	var errFirst error
+	qtFirst := qt.NewChild("expr1")
+	wg.Go(func() {
+		tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
+		qtFirst.Done()
+	})
+
+	var tssSecond []*timeseries
+	var errSecond error
+	qtSecond := qt.NewChild("expr2")
+	wg.Go(func() {
+		tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
+		qtSecond.Done()
+	})
+
+	wg.Wait()
+	if errFirst != nil {
+		return nil, nil, errFirst
 	}
-	lfs := getCommonLabelFilters(tssFirst)
-	lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
-	exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
-	tssSecond, err := evalExpr(qt, ec, exprSecond)
-	if err != nil {
-		return nil, nil, err
+	if errSecond != nil {
+		return nil, nil, errSecond
 	}
 	return tssFirst, tssSecond, nil
 }
 
+func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {
+	if !ec.OptimizeRepeatedBinaryOpSubexprs {
+		return false
+	}
+	if ec.Start == ec.End {
+		return false
+	}
+	if !ec.mayCache() {
+		return false
+	}
+
+	candidatesFirst := make(map[string]struct{}, 1)
+	var b []byte
+	visitOptimizedAggrs(exprFirst, func(ae *metricsql.AggrFuncExpr) {
+		if hasUnseededVolatileFunc(ae) {
+			return
+		}
+		b = ae.AppendString(b[:0])
+		candidatesFirst[string(b)] = struct{}{}
+	})
+	if len(candidatesFirst) == 0 {
+		return false
+	}
+
+	repeated := false
+	visitOptimizedAggrs(exprSecond, func(ae *metricsql.AggrFuncExpr) {
+		if repeated {
+			return
+		}
+		b = ae.AppendString(b[:0])
+		_, repeated = candidatesFirst[string(b)]
+	})
+	return repeated
+}
+
+func visitOptimizedAggrs(e metricsql.Expr, f func(ae *metricsql.AggrFuncExpr)) {
+	metricsql.VisitAll(e, func(expr metricsql.Expr) {
+		ae, ok := expr.(*metricsql.AggrFuncExpr)
+		if !ok {
+			return
+		}
+		if getIncrementalAggrFuncCallbacks(ae.Name) == nil {
+			return
+		}
+		fe, _ := tryGetArgRollupFuncWithMetricExpr(ae)
+		if fe == nil {
+			return
+		}
+		f(ae)
+	})
+}
+
+func hasUnseededVolatileFunc(e metricsql.Expr) bool {
+	found := false
+	metricsql.VisitAll(e, func(expr metricsql.Expr) {
+		if found {
+			return
+		}
+		fe, ok := expr.(*metricsql.FuncExpr)
+		if !ok {
+			return
+		}
+		switch strings.ToLower(fe.Name) {
+		case "now":
+			found = true
+		case "rand", "rand_normal", "rand_exponential":
+			found = len(fe.Args) == 0
+		}
+	})
+	return found
+}
+
 func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {
 	if len(tss) == 0 {
 		return nil

app/vmselect/promql/eval_test.go

@@ -170,3 +170,87 @@ func TestGetSumInstantValues(t *testing.T) {
 		[]*timeseries{ts("foo", 100, 1)},
 	)
 }
+
+func TestShouldOptimizeRepeatedBinaryOpSubexprsGate(t *testing.T) {
+	e, err := metricsql.Parse(`count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`)
+	if err != nil {
+		t.Fatalf("unexpected error in metricsql.Parse(): %s", err)
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok {
+		t.Fatalf("unexpected expr type; got %T; want *metricsql.BinaryOpExpr", e)
+	}
+
+	f := func(name string, ec *EvalConfig, resultExpected bool) {
+		t.Helper()
+		result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %q; got %v; want %v", name, result, resultExpected)
+		}
+	}
+
+	f("disabled optimization", &EvalConfig{
+		Start: 1000,
+		End:   2000,
+		Step:  1000,
+	}, false)
+	f("disabled cache", &EvalConfig{
+		Start:                            1000,
+		End:                              2000,
+		Step:                             1000,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+	f("instant query", &EvalConfig{
+		Start:                            1000,
+		End:                              1000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+	f("repeated cacheable aggregate subexpression", &EvalConfig{
+		Start:                            1000,
+		End:                              2000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, true)
+	f("unaligned range query", &EvalConfig{
+		Start:                            1001,
+		End:                              2000,
+		Step:                             1000,
+		MayCache:                         true,
+		OptimizeRepeatedBinaryOpSubexprs: true,
+	}, false)
+}
+
+func TestShouldOptimizeRepeatedBinaryOpSubexprsExpressions(t *testing.T) {
+	f := func(name, q string, resultExpected bool) {
+		t.Helper()
+		e, err := metricsql.Parse(q)
+		if err != nil {
+			t.Fatalf("unexpected error in metricsql.Parse(%q) for %q: %s", q, name, err)
+		}
+		be, ok := e.(*metricsql.BinaryOpExpr)
+		if !ok {
+			t.Fatalf("unexpected expr type for %q; got %T; want *metricsql.BinaryOpExpr", name, e)
+		}
+		ec := &EvalConfig{Start: 1000, End: 2000, Step: 1000, MayCache: true, OptimizeRepeatedBinaryOpSubexprs: true}
+		result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %q; got %v; want %v; query: %q", name, result, resultExpected, q)
+		}
+	}
+
+	f("original issue query", `count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`, true)
+	f("right side contains repeated count aggregate", `count(foo) by (job) / (count(foo) by (job) + 1)`, true)
+	f("same sum aggregate", `sum(rate(foo[5m])) by (job) / sum(rate(foo[5m])) by (job)`, true)
+	f("same inner rollup but different aggregates", `sum(rate(foo[5m])) by (job) / count(rate(foo[5m])) by (job)`, false)
+	f("different count aggregates", `count(foo) by (job) / count(bar) by (job)`, false)
+	f("bare metric selector", `foo / foo`, false)
+	f("bare rollup function", `rate(a[5m]) / rate(a[5m])`, false)
+	f("now at modifier", `sum(rate(foo[5m] @ now())) by (job) / sum(rate(foo[5m] @ now())) by (job)`, false)
+	f("unseeded rand at modifier", `sum(rate(foo[5m] @ rand())) by (job) / sum(rate(foo[5m] @ rand())) by (job)`, false)
+	f("unseeded rand_normal at modifier", `sum(rate(foo[5m] @ rand_normal())) by (job) / sum(rate(foo[5m] @ rand_normal())) by (job)`, false)
+	f("unseeded rand_exponential at modifier", `sum(rate(foo[5m] @ rand_exponential())) by (job) / sum(rate(foo[5m] @ rand_exponential())) by (job)`, false)
+	f("seeded rand at modifier", `sum(rate(foo[5m] @ rand(1))) by (job) / sum(rate(foo[5m] @ rand(1))) by (job)`, true)
+}

app/vmselect/promql/exec_test.go

@@ -4833,13 +4833,137 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`buckets_limit(zero)`, func(t *testing.T) {
+	// buckets that are consecutively empty at left and right ends will not be preserved.
+	t.Run(`buckets_limit(trim_zero_preserve_empty_when_limit_not_reached)`, func(t *testing.T) {
 		t.Parallel()
-		q := `buckets_limit(0, (
-			alias(label_set(100, "le", "inf", "x", "y"), "metric"),
-			alias(label_set(50, "le", "120", "x", "y"), "metric"),
-		))`
-		resultExpected := []netstorage.Result{}
+		q := `sort(buckets_limit(3, (
+			alias(label_set(36, "le", "+Inf"), "metric"),
+			alias(label_set(36, "le", "25"), "metric"),
+			alias(label_set(36, "le", "21"), "metric"),
+			alias(label_set(36, "le", "19"), "metric"),
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+
+	// the number of non-empty bucket doesn't reach the given "limit", so some empty buckets will be preserved, and left buckets are preferred to be kept.
+	t.Run(`buckets_limit(trim_zero)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(buckets_limit(5, (
+			alias(label_set(36, "le", "18"), "metric"),
+			alias(label_set(36, "le", "17"), "metric"),
+			alias(label_set(36, "le", "16"), "metric"),
+			alias(label_set(27, "le", "12"), "metric"),
+			alias(label_set(14, "le", "9"), "metric"),
+			alias(label_set(0, "le", "6"), "metric"),
+			alias(label_set(0, "le", "1"), "metric"),
+			)))`
+
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("metric")
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("metric")
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("6"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.MetricGroup = []byte("metric")
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("9"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{27, 27, 27, 27, 27, 27},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.MetricGroup = []byte("metric")
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("12"),
+			},
+		}
+		r5 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{36, 36, 36, 36, 36, 36},
+			Timestamps: timestampsExpected,
+		}
+		r5.MetricName.MetricGroup = []byte("metric")
+		r5.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("16"),
+			},
+		}
+
+		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
 		f(q, resultExpected)
 	})
 	t.Run(`buckets_limit(unused)`, func(t *testing.T) {
@@ -6228,50 +6352,6 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6, r7}
 		f(q, resultExpected)
 	})
-	t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
-		t.Parallel()
-		q := `sort_by_label(
-			buckets_limit(
-				3,
-				sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
-			), "le"
-		)`
-		r1 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r1.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("+Inf"),
-			},
-		}
-		r2 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 0, 0, 0},
-			Timestamps: timestampsExpected,
-		}
-		r2.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("1.000e+00"),
-			},
-		}
-		r3 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{40, 40, 40, 40, 40, 40},
-			Timestamps: timestampsExpected,
-		}
-		r3.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("2.448e+00"),
-			},
-		}
-		resultExpected := []netstorage.Result{r1, r2, r3}
-		f(q, resultExpected)
-	})
 	t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`

app/vmselect/promql/rollup.go

@@ -1055,7 +1055,7 @@ func newRollupHoltWinters(args []any) (rollupFunc, error) {
 			return nan
 		}
 
-		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing .
+		// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29 .
 		// TODO: determine whether this shit really works.
 		s0 := rfa.prevValue
 		if math.IsNaN(s0) {

app/vmselect/promql/timeseries.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"fmt"
+	"math"
 	"sort"
 	"strconv"
 	"sync"
@@ -288,6 +289,9 @@ func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
 }
 
 func marshalBytesFast(dst []byte, s []byte) []byte {
+	if len(s) > math.MaxUint16 {
+		logger.Panicf("BUG: s len %d cannot exceed %d", len(s), math.MaxUint16)
+	}
 	dst = encoding.MarshalUint16(dst, uint16(len(s)))
 	dst = append(dst, s...)
 	return dst

app/vmselect/promql/transform.go

@@ -393,7 +393,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 	if limit <= 0 {
-		return nil, nil
+		return nil, fmt.Errorf("limit must be greater than 0; got %d", limit)
 	}
 	if limit < 3 {
 		// Preserve the first and the last bucket for better accuracy for min and max values.
@@ -461,6 +461,23 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 				prevValue = value
 			}
 		}
+
+		// Remove buckets that are consecutively empty at left and right ends to obtain more accurate max and min values.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417.
+		epsilon := 1e-9
+		isEmptyBucket := func(hits float64) bool {
+			return !math.IsNaN(hits) && math.Abs(hits) < epsilon
+		}
+		l := 0
+		r := len(leGroup) - 1
+		for r-l+1 > limit && isEmptyBucket(leGroup[r].hits) {
+			r--
+		}
+		for r-l+1 > limit && isEmptyBucket(leGroup[l].hits) {
+			l++
+		}
+		leGroup = leGroup[l : r+1]
+
 		for len(leGroup) > limit {
 			// Preserve the first and the last bucket for better accuracy for min and max values
 			xxMinIdx := 1
@@ -1121,29 +1138,29 @@ func groupLeTimeseries(tss []*timeseries) map[string][]leTimeseries {
 
 func fixBrokenBuckets(i int, xss []leTimeseries) {
 	// Buckets are already sorted by le, so their values must be in ascending order,
-	// since the next bucket includes all the previous buckets.
-	// If the next bucket has lower value than the current bucket,
-	// then the next bucket must be substituted with the current bucket value.
+	// since the upper bucket includes all the lower buckets.
+	// If the upper bucket has lower value than the current bucket,
+	// then the upper bucket must be substituted with the current bucket value.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580#issuecomment-2186659102
 	if len(xss) < 2 {
 		return
 	}
 
-	vNext := xss[0].ts.Values[i]
+	vPrev := xss[0].ts.Values[i]
 	// Set the lowest bucket to 0 if its value is NaN, so it can be properly
 	// compared with upper buckets in the loop below.
-	if math.IsNaN(vNext) {
-		vNext = 0
-		xss[0].ts.Values[i] = vNext
+	if math.IsNaN(vPrev) {
+		vPrev = 0
+		xss[0].ts.Values[i] = vPrev
 	}
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
-	// or are bigger than the lower bucket values.
+	// or are smaller than the lower bucket values.
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
-		if math.IsNaN(v) || vNext > v {
-			xss[j].ts.Values[i] = vNext
+		if math.IsNaN(v) || vPrev > v {
+			xss[j].ts.Values[i] = vPrev
 		} else {
-			vNext = v
+			vPrev = v
 		}
 	}
 }

app/vmselect/vmui/assets/MetricsQL-CQAtdgI6.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmselect/vmui/index.html

@@ -37,7 +37,7 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-CusQvJzs.js"></script>
+  <script type="module" crossorigin src="./assets/index-xYKUiOTH.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Cyuzqnbw.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-B83wxFqK.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">

app/vmstorage/main.go

@@ -30,6 +30,9 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
 		"The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	vmselectAddr                  = flag.String("vmselectAddr", "", "TCP address to accept connections from vmselect services")
+	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
+		"This reduces CPU usage at the cost of higher network bandwidth usage")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -108,7 +111,7 @@ func DataPath() string {
 }
 
 // Init initializes vmstorage.
-func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
+func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Duration, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
@@ -169,6 +172,21 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
 	metrics.RegisterSet(storageMetrics)
 
+	if *vmselectAddr != "" {
+		var err error
+		limits := vmselectapi.Limits{
+			MaxConcurrentRequests:         vmselectMaxConcurrentRequests,
+			MaxConcurrentRequestsFlagName: "search.maxConcurrentRequests",
+			MaxQueueDuration:              vmselectMaxQueueDuration,
+			MaxQueueDurationFlagName:      "search.maxQueueDuration",
+		}
+		api := newVMStorageWithTenantID(vmStorage)
+		vmselectSrv, err = vmselectapi.NewServer(*vmselectAddr, api, limits, *vmselectDisableRPCCompression)
+		if err != nil {
+			logger.Fatalf("cannot create a server with -vmselectAddr=%s: %s", *vmselectAddr, err)
+		}
+	}
+
 	VMInsertAPI = vmStorage
 	VMSelectAPI = vmStorage
 	GetSearch = vmStorage.GetSearch
@@ -191,6 +209,8 @@ var (
 
 	// TODO(@rtm0): Remove this dependency from vmalert-tool unit tests.
 	DebugFlush func()
+
+	vmselectSrv *vmselectapi.Server
 )
 
 // Stop stops the vmstorage
@@ -201,6 +221,10 @@ func Stop() {
 
 	logger.Infof("gracefully closing the storage at %s", *storageDataPath)
 	startTime := time.Now()
+
+	if vmselectSrv != nil {
+		vmselectSrv.MustStop()
+	}
 	vmStorage.Stop()
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 

app/vmstorage/vmstorage.go

@@ -164,6 +164,10 @@ func (vms *VMStorage) IsReadOnly() bool {
 }
 
 func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	return vms.initSearch(qt, sq, marshalDefault, deadline)
+}
+
+func (vms *VMStorage) initSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, marshal marshalFunc, deadline uint64) (vmselectapi.BlockIterator, error) {
 	vms.wg.Add(1)
 
 	tr := sq.GetTimeRange()
@@ -178,6 +182,7 @@ func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery
 		return nil, fmt.Errorf("missing tag filters")
 	}
 	bi := getBlockIterator()
+	bi.marshal = marshal
 	bi.wgDone = vms.wg.Done
 	bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
 	if err := bi.sr.Error(); err != nil {
@@ -198,11 +203,19 @@ func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
 	return searchQueryLimit
 }
 
+type marshalFunc func(dst []byte, src *storage.MetricBlock) []byte
+
+// marshalDefault is the default implementation of the MetricBlock marshaling.
+func marshalDefault(dst []byte, src *storage.MetricBlock) []byte {
+	return src.Marshal(dst)
+}
+
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
-	sr     storage.Search
-	mb     storage.MetricBlock
-	wgDone func()
+	sr      storage.Search
+	mb      storage.MetricBlock
+	marshal marshalFunc
+	wgDone  func()
 }
 
 var blockIteratorsPool sync.Pool
@@ -228,10 +241,10 @@ func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
 	if !bi.sr.NextMetricBlock() {
 		return dst, false
 	}
-	mb := bi.mb
+	mb := &bi.mb
 	mb.MetricName = bi.sr.MetricBlockRef.MetricName
 	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
-	dst = mb.Marshal(dst[:0])
+	dst = bi.marshal(dst[:0], mb)
 	return dst, true
 }
 

app/vmstorage/vmstorage_with_tenant_id.go

@@ -0,0 +1,282 @@
+package vmstorage
+
+import (
+	"flag"
+	"fmt"
+	"math"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+)
+
+var (
+	accountID = flag.Uint64("accountID", 0, "The accountID of the stored data")
+	projectID = flag.Uint64("projectID", 0, "The projectID of the stored data")
+)
+
+func newVMStorageWithTenantID(vms *VMStorage) *VMStorageWithTenantID {
+	if *accountID > math.MaxUint32 {
+		logger.Fatalf("-accountID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *accountID)
+	}
+	if *projectID > math.MaxUint32 {
+		logger.Fatalf("-projectID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *projectID)
+	}
+	return &VMStorageWithTenantID{
+		vms:       vms,
+		accountID: uint32(*accountID),
+		projectID: uint32(*projectID),
+	}
+}
+
+// VMStorageWithTenantID is a thin wrapper around VMStorage type that overrides
+// its methods to properly serve requests coming from a vmselect (require
+// tenantID).
+//
+// A new instance of this type should be created using
+// newVMStorageWithTenantID(). The created instance does not require closing.
+// The instance also does not take ownership of vms and it is the responsibility
+// of the caller to close vms.
+type VMStorageWithTenantID struct {
+	vms *VMStorage
+
+	accountID uint32
+	projectID uint32
+}
+
+// InitSearch initializes a storage search for a request initiated by a
+// vmselect.
+//
+// The search is initialized only if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+// Otherwise, the method returns an interator that will return no data.
+//
+// The method also overrides the data format of the data returned by the
+// iterator by prepending accountID and projectID bytes to the metric name and
+// the data block (a format used in vmcluster).
+func (vmst *VMStorageWithTenantID) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return emptyBI, nil
+	}
+	return vmst.vms.initSearch(qt, sq, vmst.marshalMetricBlock, deadline)
+}
+
+var emptyBI = &emptyBlockIterator{}
+
+// emptyBlockIterator is an implementation of vmselectapi.BlockIterator that
+// always returns no data.
+type emptyBlockIterator struct{}
+
+func (*emptyBlockIterator) MustClose() {}
+
+func (*emptyBlockIterator) NextBlock(dst []byte) ([]byte, bool) {
+	return dst, false
+}
+
+func (*emptyBlockIterator) Error() error {
+	return nil
+}
+
+// marshalMetricBlock serializes a metric block in the format expected by
+// vmselect.
+//
+// vmselect expects metric names and data blocks to have the tenantID but
+// vmsingle does not have it. Therefore the tenantID needs to be included to
+// every metric name and block.
+func (vmst *VMStorageWithTenantID) marshalMetricBlock(dst []byte, src *storage.MetricBlock) []byte {
+	// Marshal metric name:
+	// 1. Marshal metric name length + accountID length + projectID length (in
+	//    bytes).
+	// 2. append accountID and projectID bytes
+	// 3. Finally append metric name bytes
+	dst = encoding.MarshalVarUint64(dst, uint64(len(src.MetricName))+8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = append(dst, src.MetricName...)
+
+	// Marshal data block.
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	dst = storage.MarshalBlock(dst, &src.Block)
+
+	return dst
+}
+
+// SearchMetricNames searches the storage for metric names that match the query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+//
+// Found metric names are prepended with accountID and projectID bytes (a format
+// used in vmcluster).
+func (vmst *VMStorageWithTenantID) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+
+	metricNames, err := vmst.vms.SearchMetricNames(qt, sq, deadline)
+	if err != nil {
+		return nil, err
+	}
+
+	// vmselect expects metric names to have the tenantID but vmsingle does not
+	// have it. Therefore the tenantID needs to be prepended to every metric
+	// name.
+	dst := make([]byte, 0, 8)
+	dst = encoding.MarshalUint32(dst, vmst.accountID)
+	dst = encoding.MarshalUint32(dst, vmst.projectID)
+	tenantID := string(dst)
+
+	for i, metricName := range metricNames {
+		metricNames[i] = tenantID + metricName
+	}
+	return metricNames, nil
+}
+
+// LabelValues searches the storage for label values that match the query and
+// correspond to a label whose name is `labelName`. The returned result
+// will contain not more than `maxLabelValues`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelValues(qt, sq, labelName, maxLabelValues, deadline)
+}
+
+// TagValueSuffixes searches the storage for Graphite tag value suffixes. The
+// returned result will contain not more than `maxSuffixes`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return nil, nil
+	}
+	return vmst.vms.TagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
+}
+
+// LabelNames searches the storage for label names that match the query.
+// The returned result will contain not more than `maxLabelNames`.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return an
+// empty result.
+func (vmst *VMStorageWithTenantID) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return nil, nil
+	}
+	return vmst.vms.LabelNames(qt, sq, maxLabelNames, deadline)
+}
+
+// SeriesCount returns the total number of metrics stored in the database.
+//
+// The method may return inflated numbers. How inflated the count depends
+// on the churn rate and the retention period. For example, if a metric lasts
+// for 2 months, it will be counted twice.
+//
+// The method also counts the deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return 0.
+func (vmst *VMStorageWithTenantID) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
+	if !vmst.isValidTenantID(accountID, projectID) {
+		return 0, nil
+	}
+	return vmst.vms.SeriesCount(qt, accountID, projectID, deadline)
+}
+
+// Tenants returns just one tenant consisting of the -accountID and -projectID
+// flag values.
+func (vmst *VMStorageWithTenantID) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
+	tenantID := fmt.Sprintf("%d:%d", vmst.accountID, vmst.projectID)
+	return []string{tenantID}, nil
+}
+
+// TSDBStatus retrieves the status for metrics that match to the search query.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, the method will return empty
+// status.
+func (vmst *VMStorageWithTenantID) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return &storage.TSDBStatus{}, nil
+	}
+	return vmst.vms.TSDBStatus(qt, sq, focusLabel, topN, deadline)
+}
+
+// DeleteSeries marks as deleted metrics that match the search query.
+// The method returns the number of deleted metrics.
+//
+// If the query is not multitenant or the query accountID and projectID do not
+// match the -accoutID and -projectID flag values, no metrics will be deleted
+// and the method will return 0.
+func (vmst *VMStorageWithTenantID) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	if !vmst.hasValidTenantID(sq) {
+		return 0, nil
+	}
+	return vmst.vms.DeleteSeries(qt, sq, deadline)
+}
+
+// RegisterMetricNames registers metric names in the index, the sample values
+// and timestamps are ignored.
+func (vmst *VMStorageWithTenantID) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
+	return vmst.vms.RegisterMetricNames(qt, mrs, deadline)
+}
+
+// GetMetricNamesUsageStats retrieves the usage stats for metrics whose name
+// matches the pattern.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return metricnamestats.StatsResult{}, nil
+	}
+	return vmst.vms.GetMetricNamesUsageStats(qt, tt, limit, le, matchPattern, deadline)
+}
+
+// ResetMetricNamesUsageStats resets the metric name usage stats.
+func (vmst *VMStorageWithTenantID) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
+	return vmst.vms.ResetMetricNamesUsageStats(qt, deadline)
+}
+
+// GetMetadataRecords retrieves the metadata for the metricName.
+//
+// If the request is not multitenant or the request accountID and projectID do
+// not match the -accoutID and -projectID flag values, no metrics will be
+// deleted and the method will return 0.
+func (vmst *VMStorageWithTenantID) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
+	if !vmst.isValidTenantToken(tt) {
+		return nil, nil
+	}
+	return vmst.vms.GetMetadataRecords(qt, tt, limit, metricName, deadline)
+}
+
+// hasValidTenantID returns true if the search query is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) hasValidTenantID(sq *storage.SearchQuery) bool {
+	return sq.IsMultiTenant || vmst.isValidTenantID(sq.AccountID, sq.ProjectID)
+}
+
+// isValidTenantToken returns true if the TenantToken is either multitenant or
+// its accountID and projectID match -accountID and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantToken(tt *storage.TenantToken) bool {
+	return tt == nil || vmst.isValidTenantID(tt.AccountID, tt.ProjectID)
+}
+
+// isValidTenantID returns true if the accountID and projectID match -accountID
+// and -projectID flag values.
+func (vmst *VMStorageWithTenantID) isValidTenantID(accountID, projectID uint32) bool {
+	return accountID == vmst.accountID && projectID == vmst.projectID
+}

app/vmui/Dockerfile-web

@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.23.4
+FROM alpine:3.24.1
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -183,7 +183,7 @@ const StepConfigurator: FC = () => {
           <div className="vm-step-control-popper-info">
             <p>
               <code>step</code> - the <Hyperlink
-                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations"
+                href="https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations"
                 text="interval"
               /> between datapoints, which must be returned from the range query.
               The <code>query</code> is executed

app/vmui/packages/vmui/src/utils/time.ts

@@ -16,7 +16,7 @@ const supportedValuesOf = Intl.supportedValuesOf;
 export const supportedTimezones = supportedValuesOf ? supportedValuesOf("timeZone") as string[] : timezones;
 
 // The list of supported units could be the following -
-// https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations
+// https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations
 export const supportedDurations = [
   { long: "years", short: "y", possible: "year" },
   { long: "weeks", short: "w", possible: "week" },

apptest/testdata.go

@@ -0,0 +1,358 @@
+package apptest
+
+import (
+	"fmt"
+	"slices"
+)
+
+type TestData struct {
+	Samples              []string
+	Step                 int64
+	WantSeries           []map[string]string
+	WantLabels           []string
+	WantLabelValues      []string
+	WantQueryResults     []*QueryResult
+	WantMetadata         map[string][]MetadataEntry
+	WantMetricNamesStats []MetricNamesStatsRecord
+}
+
+func GenerateTestData(prefix string, numMetrics, start, end int64) TestData {
+	d := TestData{
+		Samples:              []string{},
+		Step:                 (end - start) / numMetrics,
+		WantSeries:           make([]map[string]string, numMetrics),
+		WantLabels:           make([]string, numMetrics),
+		WantLabelValues:      make([]string, numMetrics),
+		WantQueryResults:     make([]*QueryResult, numMetrics),
+		WantMetadata:         make(map[string][]MetadataEntry),
+		WantMetricNamesStats: make([]MetricNamesStatsRecord, numMetrics),
+	}
+	for i := range numMetrics {
+		metricName := fmt.Sprintf("%s_%04d", prefix, i)
+		metricHelp := fmt.Sprintf("# HELP %s some help message", metricName)
+		metricType := fmt.Sprintf("# TYPE %s gauge", metricName)
+		labelName := fmt.Sprintf("label_%04d", i)
+		labelValue := fmt.Sprintf("value_%04d", i)
+		value := i
+		timestamp := start + i*d.Step
+		sample := fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
+
+		d.Samples = append(d.Samples, metricHelp, metricType, sample)
+		d.WantSeries[i] = map[string]string{
+			"__name__": metricName,
+			labelName:  "value",
+			"label":    labelValue,
+		}
+		d.WantLabels[i] = labelName
+		d.WantLabelValues[i] = labelValue
+		d.WantQueryResults[i] = &QueryResult{
+			Metric: map[string]string{
+				"__name__": metricName,
+				labelName:  "value",
+				"label":    labelValue,
+			},
+			Samples: []*Sample{{Timestamp: timestamp, Value: float64(value)}},
+		}
+		d.WantMetadata[metricName] = []MetadataEntry{{Help: "some help message", Type: "gauge"}}
+		d.WantMetricNamesStats[i].MetricName = metricName
+	}
+	d.WantLabels = append(d.WantLabels, "__name__", "label")
+	slices.Sort(d.WantLabels)
+	return d
+}
+
+// AssertSeries retrieves metric names from the storage and compares the result
+// with the expected one.
+func AssertSeries(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []map[string]string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/series response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1Series(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			}).Sort()
+		},
+		Want: &PrometheusAPIV1SeriesResponse{
+			Status: "success",
+			Data:   want,
+		},
+		Retries: 1000,
+		FailNow: true,
+	})
+}
+
+// AssertSeriesCount retrieves series count and compares it with expected one.
+func AssertSeriesCount(tc *TestCase, app PrometheusQuerier, tenantID string, start, end int64, want uint64) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/series/count response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1SeriesCount(tc.T(), QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+		},
+		Want: &PrometheusAPIV1SeriesCountResponse{
+			Status: "success",
+			Data:   []uint64{want},
+		},
+		FailNow: true,
+	})
+}
+
+// AssertLabels retrieves label names from the storage and compares the result
+// with the expected one.
+func AssertLabels(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/labels response",
+		Got: func() any {
+			tc.T().Helper()
+			res := app.PrometheusAPIV1Labels(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+			slices.Sort(res.Data)
+			return res
+		},
+		Want: &PrometheusAPIV1LabelsResponse{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+// AssertLabelValues retrieves values for the label whose name is labelName for
+// the series whose name mathes metricNameRE, compares the result with the
+// expected one.
+func AssertLabelValues(tc *TestCase, app PrometheusQuerier, metricNameRE, labelName, tenantID string, start, end int64, want []string) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/labels/.../values response",
+		Got: func() any {
+			tc.T().Helper()
+			res := app.PrometheusAPIV1LabelValues(tc.T(), labelName, query, QueryOpts{
+				Tenant: tenantID,
+				Start:  fmt.Sprintf("%d", start),
+				End:    fmt.Sprintf("%d", end),
+			})
+			slices.Sort(res.Data)
+			return res
+		},
+		Want: &PrometheusAPIV1LabelValuesResponse{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+// AssertQueryResults sends a data query to storage and compares the query
+// result with the expected one.
+func AssertQueryResults(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end, step int64, want []*QueryResult) {
+	tc.T().Helper()
+
+	query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/query_range response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1QueryRange(tc.T(), query, QueryOpts{
+				Tenant:      tenantID,
+				Start:       fmt.Sprintf("%d", start),
+				End:         fmt.Sprintf("%d", end),
+				Step:        fmt.Sprintf("%dms", step),
+				MaxLookback: fmt.Sprintf("%dms", step-1),
+				NoCache:     "1",
+			})
+		},
+		Want: &PrometheusAPIV1QueryResponse{
+			Status: "success",
+			Data: &QueryData{
+				ResultType: "matrix",
+				Result:     want,
+			},
+		},
+		FailNow: true,
+	})
+}
+
+func AssertMetadata(tc *TestCase, app PrometheusQuerier, metricName, tenantID string, want map[string][]MetadataEntry) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/metadata response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1Metadata(tc.T(), metricName, 0, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want: &PrometheusAPIV1Metadata{
+			Status: "success",
+			Data:   want,
+		},
+		FailNow: true,
+	})
+}
+
+func AssertMetricNamesStats(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, want []MetricNamesStatsRecord) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /prometheus/api/v1/status/metric_names_stats response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.PrometheusAPIV1StatusMetricNamesStats(tc.T(), "", "", metricNameRE, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want: MetricNamesStatsResponse{
+			Records: want,
+		},
+		FailNow: true,
+	})
+
+}
+
+// GraphiteTestData holds the data samples in Graphite Pickle format, distance
+// between samples in milliseconds and expected responses for various Graphite
+// API endpoints.
+type GraphiteTestData struct {
+	Samples             []string
+	Step                int64
+	WantMetricsIndex    []string
+	WantMetricsFind     []GraphiteMetric
+	WantMetricsExpand   []string
+	WantRenderedTargets []GraphiteRenderedTarget
+}
+
+// GenerateGraphiteTestData generates Graphite test data.
+func GenerateGraphiteTestData(prefix string, numMetrics, start, end int64) GraphiteTestData {
+	d := GraphiteTestData{
+		Samples:             make([]string, numMetrics),
+		Step:                (end - start) / numMetrics,
+		WantMetricsIndex:    make([]string, numMetrics),
+		WantMetricsFind:     make([]GraphiteMetric, numMetrics),
+		WantMetricsExpand:   make([]string, numMetrics),
+		WantRenderedTargets: make([]GraphiteRenderedTarget, numMetrics),
+	}
+
+	datapoints := make([][2]float64, numMetrics)
+	for i := range numMetrics {
+		timestamp := (start + i*d.Step) / 1000
+		datapoints[i][1] = float64(timestamp)
+	}
+
+	for i := range numMetrics {
+		suffix := fmt.Sprintf("%04d", i)
+		metricName := fmt.Sprintf("%s.%s", prefix, suffix)
+		value := i
+		timestamp := (start + i*d.Step) / 1000
+		sample := fmt.Sprintf(`%s %d %d`, metricName, value, timestamp)
+
+		d.Samples[i] = sample
+		d.WantMetricsIndex[i] = metricName
+		d.WantMetricsFind[i].Id = metricName
+		d.WantMetricsFind[i].Text = suffix
+		d.WantMetricsFind[i].Leaf = 1
+		d.WantMetricsExpand[i] = metricName
+		d.WantRenderedTargets[i].Target = metricName
+		d.WantRenderedTargets[i].Datapoints = slices.Clone(datapoints)
+		d.WantRenderedTargets[i].Datapoints[i][0] = float64(value)
+	}
+	return d
+}
+
+// AssertGraphiteMetricsIndex retrieves all metrics by sending a request to
+// /graphite/metrics/index.json and compares the result with the expected one.
+func AssertGraphiteMetricsIndex(tc *TestCase, app PrometheusQuerier, tenantID string, want []string) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/index.json response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsIndex(tc.T(), QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		Retries: 30,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteMetricsFind finds metric names by sending a request to
+// /graphite/metrics/find and compares the result with the expected one.
+func AssertGraphiteMetricsFind(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []GraphiteMetric) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/find response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsFind(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteMetricsFind expands metric names by sending a request to
+// /graphite/metrics/expand and compares the result with the expected one.
+func AssertGraphiteMetricsExpand(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []string) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/metrics/expand response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteMetricsExpand(tc.T(), query, QueryOpts{
+				Tenant: tenantID,
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+
+}
+
+// AssertGraphiteRender retieves metric raw data by sending a request to
+// /graphite/render and compares the result with the expected one.
+func AssertGraphiteRender(tc *TestCase, app PrometheusQuerier, target, tenantID string, from, until, step int64, want []GraphiteRenderedTarget) {
+	tc.T().Helper()
+
+	tc.Assert(&AssertOptions{
+		Msg: "unexpected /graphite/render response",
+		Got: func() any {
+			tc.T().Helper()
+			return app.GraphiteRender(tc.T(), target, QueryOpts{
+				Tenant:      tenantID,
+				From:        fmt.Sprintf("%d", from/1000),
+				Until:       fmt.Sprintf("%d", until/1000),
+				StorageStep: fmt.Sprintf("%dms", step),
+			})
+		},
+		Want:    want,
+		FailNow: true,
+	})
+}

apptest/tests/metricsmetadata_test.go

@@ -2,6 +2,7 @@ package tests
 
 import (
 	"fmt"
+	"math"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -25,7 +26,11 @@ func TestSingleMetricsMetadata(t *testing.T) {
 	if len(resp.Data) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
 	}
-
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 	const ingestTimestamp = 1707123456700
 	prometheusTextDataSet := []string{
 		`# HELP metric_name_1 some help message`,
@@ -40,6 +45,12 @@ func TestSingleMetricsMetadata(t *testing.T) {
 		`# TYPE metric_name_3 gauge`,
 		`metric_name_3{label="baz"} 30`,
 	}
+	prometheusTextDataSet = append(prometheusTextDataSet,
+		`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
+		`# TYPE metric_name_4 gauge`,
+		`metric_name_4{label="baz"} 30`,
+	)
+
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Timeseries: []prompb.TimeSeries{
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
@@ -52,6 +63,9 @@ func TestSingleMetricsMetadata(t *testing.T) {
 			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
 			{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
+			{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 
@@ -137,6 +151,11 @@ func TestClusterMetricsMetadata(t *testing.T) {
 	if len(resp.Data) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
 	}
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 
 	const ingestTimestamp = 1707123456700
 	prometheusTextDataSet := []string{
@@ -152,6 +171,11 @@ func TestClusterMetricsMetadata(t *testing.T) {
 		`# TYPE metric_name_3 gauge`,
 		`metric_name_3{label="baz"} 30`,
 	}
+	prometheusTextDataSet = append(prometheusTextDataSet,
+		`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
+		`# TYPE metric_name_4 gauge`,
+		`metric_name_4{label="baz"} 30`,
+	)
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Timeseries: []prompb.TimeSeries{
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
@@ -164,6 +188,9 @@ func TestClusterMetricsMetadata(t *testing.T) {
 			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
 			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
 			{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
+			{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
+			{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 

apptest/tests/vmagent_remotewrite_test.go

@@ -517,10 +517,15 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) {
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
 		fmt.Sprintf(`-remoteWrite.url=http://%s/insert/multitenant/prometheus/api/v1/write`, sut.Vminsert.HTTPAddr()),
 	})
-
+	generateValueExceedLimit := func(prefix string) string {
+		buf := make([]byte, math.MaxUint16+len(prefix))
+		copy(buf, prefix)
+		return string(buf)
+	}
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Metadata: []prompb.MetricMetadata{
 			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary, AccountID: 100},
+			{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset, AccountID: 100},
 		},
 	}
 	vmagent.PrometheusAPIV1Write(t, prometheusRemoteWriteDataSet, apptest.QueryOpts{Tenant: "multitenant"})

apptest/tests/vmsingle_vmselect_rpc_test.go

@@ -0,0 +1,216 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestMixedPrometheusQueries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	const (
+		accountID1 = 12
+		projectID1 = 34
+		accountID2 = 56
+		projectID2 = 78
+		numMetrics = 10
+	)
+	tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
+	tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
+	start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data := apptest.GenerateTestData("metric", numMetrics, start, end)
+	emptySeries := []map[string]string{}
+	emptyLabels := []string{}
+	emptyLabelValues := []string{}
+	emptyQueryResults := []*apptest.QueryResult{}
+	emptyMetadata := map[string][]apptest.MetadataEntry{}
+	emptyMetricNamesStats := []apptest.MetricNamesStatsRecord{}
+
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+		"-retentionPeriod=100y",
+		fmt.Sprintf("-accountID=%d", accountID1),
+		fmt.Sprintf("-projectID=%d", projectID1),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmsingle.VmselectAddr(),
+	})
+
+	vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data.Samples, apptest.QueryOpts{})
+	vmsingle.ForceFlush(t)
+
+	// Ensure vmsingle returns data.
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data.WantSeries)
+	apptest.AssertSeriesCount(tc, vmsingle, "", start, end, numMetrics)
+	apptest.AssertLabels(tc, vmsingle, "metric.*", "", start, end, data.WantLabels)
+	apptest.AssertLabelValues(tc, vmsingle, "metric.*", "label", "", start, end, data.WantLabelValues)
+	apptest.AssertQueryResults(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantQueryResults)
+	apptest.AssertMetadata(tc, vmsingle, "", "", data.WantMetadata)
+	for i := range data.WantMetricNamesStats {
+		data.WantMetricNamesStats[i].QueryRequestsCount = 1
+	}
+	apptest.AssertMetricNamesStats(tc, vmsingle, "", "", data.WantMetricNamesStats)
+
+	// Check that current vmsingle tenant (configured via flags) is tenant1.
+	gotAdminTenantsResponse := vmselect.APIV1AdminTenants(t, apptest.QueryOpts{})
+	wantAdminTenantsResponse := &apptest.AdminTenantsResponse{
+		Status: "success",
+		Data:   []string{tenantID1},
+	}
+	if diff := cmp.Diff(wantAdminTenantsResponse, gotAdminTenantsResponse); diff != "" {
+		t.Fatalf("unexpected tenants (-want, +got):\n%s", diff)
+	}
+
+	// Ensure vmselect returns data for tenant1.
+	apptest.AssertSeries(tc, vmselect, "metric.*", tenantID1, start, end, data.WantSeries)
+	apptest.AssertSeriesCount(tc, vmselect, tenantID1, start, end, numMetrics)
+	apptest.AssertLabels(tc, vmselect, "metric.*", tenantID1, start, end, data.WantLabels)
+	apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", tenantID1, start, end, data.WantLabelValues)
+	apptest.AssertQueryResults(tc, vmselect, "metric.*", tenantID1, start, end, data.Step, data.WantQueryResults)
+	apptest.AssertMetadata(tc, vmselect, "", tenantID1, data.WantMetadata)
+	for i := range data.WantMetricNamesStats {
+		data.WantMetricNamesStats[i].QueryRequestsCount = 2
+	}
+	apptest.AssertMetricNamesStats(tc, vmselect, "", tenantID1, data.WantMetricNamesStats)
+
+	// Ensure vmselect does not return any data for tenant2.
+	apptest.AssertSeries(tc, vmselect, "metric.*", tenantID2, start, end, emptySeries)
+	apptest.AssertSeriesCount(tc, vmselect, tenantID2, start, end, 0)
+	apptest.AssertLabels(tc, vmselect, "metric.*", tenantID2, start, end, emptyLabels)
+	apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", tenantID2, start, end, emptyLabelValues)
+	apptest.AssertQueryResults(tc, vmselect, "metric.*", tenantID2, start, end, data.Step, emptyQueryResults)
+	apptest.AssertMetadata(tc, vmselect, "", tenantID2, emptyMetadata)
+	apptest.AssertMetricNamesStats(tc, vmselect, "", tenantID2, emptyMetricNamesStats)
+
+	// Ensure vmselect returns data for multitenant.
+	for _, v := range data.WantSeries {
+		v["vm_account_id"] = strconv.Itoa(accountID1)
+		v["vm_project_id"] = strconv.Itoa(projectID1)
+	}
+	apptest.AssertSeries(tc, vmselect, "metric.*", "multitenant", start, end, data.WantSeries)
+	data.WantLabels = append(data.WantLabels, "vm_account_id", "vm_project_id")
+	apptest.AssertLabels(tc, vmselect, "metric.*", "multitenant", start, end, data.WantLabels)
+	apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", "multitenant", start, end, data.WantLabelValues)
+	for _, v := range data.WantQueryResults {
+		v.Metric["vm_account_id"] = strconv.Itoa(accountID1)
+		v.Metric["vm_project_id"] = strconv.Itoa(projectID1)
+	}
+	apptest.AssertQueryResults(tc, vmselect, "metric.*", "multitenant", start, end, data.Step, data.WantQueryResults)
+	apptest.AssertMetadata(tc, vmselect, "", "multitenant", data.WantMetadata)
+	for i := range data.WantMetricNamesStats {
+		data.WantMetricNamesStats[i].QueryRequestsCount = 3
+	}
+	apptest.AssertMetricNamesStats(tc, vmselect, "", "multitenant", data.WantMetricNamesStats)
+}
+
+func TestMixedDeleteSeries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	const (
+		accountID1 = 12
+		projectID1 = 34
+		accountID2 = 56
+		projectID2 = 78
+		numMetrics = 10
+	)
+	tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
+	tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
+	start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data1 := apptest.GenerateTestData("metric1", numMetrics, start, end)
+	data2 := apptest.GenerateTestData("metric2", numMetrics, start, end)
+	emptySeries := []map[string]string{}
+
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+		"-retentionPeriod=100y",
+		fmt.Sprintf("-accountID=%d", accountID1),
+		fmt.Sprintf("-projectID=%d", projectID1),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmsingle.VmselectAddr(),
+	})
+
+	vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data1.Samples, apptest.QueryOpts{})
+	vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data2.Samples, apptest.QueryOpts{})
+	vmsingle.ForceFlush(t)
+
+	wantSeries12 := slices.Concat(data1.WantSeries, data2.WantSeries)
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, wantSeries12)
+
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric1.*"}`, apptest.QueryOpts{
+		Tenant: tenantID1,
+	})
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data2.WantSeries)
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric2.*"}`, apptest.QueryOpts{
+		Tenant: tenantID2,
+	})
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data2.WantSeries)
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric2.*"}`, apptest.QueryOpts{
+		Tenant: "multitenant",
+	})
+	apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, emptySeries)
+}
+
+func TestMixedGraphiteQueries(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	const (
+		accountID1 = 12
+		projectID1 = 34
+		accountID2 = 56
+		projectID2 = 78
+		numMetrics = 10
+	)
+	tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
+	tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
+	start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data := apptest.GenerateGraphiteTestData("metric", numMetrics, start, end)
+	emptyMetricsIndex := []string{}
+	emptyMetricsFind := []apptest.GraphiteMetric{}
+	emptyMetricsExpand := []string{}
+	emptyRenderedTargets := []apptest.GraphiteRenderedTarget{}
+
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+		"-retentionPeriod=100y",
+		fmt.Sprintf("-accountID=%d", accountID1),
+		fmt.Sprintf("-projectID=%d", projectID1),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmsingle.VmselectAddr(),
+	})
+
+	vmsingle.GraphiteWrite(tc.T(), data.Samples, apptest.QueryOpts{})
+	vmsingle.ForceFlush(t)
+
+	// Ensure vmsingle returns data.
+	apptest.AssertGraphiteMetricsIndex(tc, vmsingle, "", data.WantMetricsIndex)
+	apptest.AssertGraphiteMetricsFind(tc, vmsingle, "metric.*", "", data.WantMetricsFind)
+	apptest.AssertGraphiteMetricsExpand(tc, vmsingle, "metric.*", "", data.WantMetricsExpand)
+	apptest.AssertGraphiteRender(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantRenderedTargets)
+
+	// Ensure vmselect returns data for tenant1.
+	apptest.AssertGraphiteMetricsIndex(tc, vmselect, tenantID1, data.WantMetricsIndex)
+	apptest.AssertGraphiteMetricsFind(tc, vmselect, "metric.*", tenantID1, data.WantMetricsFind)
+	apptest.AssertGraphiteMetricsExpand(tc, vmselect, "metric.*", tenantID1, data.WantMetricsExpand)
+	apptest.AssertGraphiteRender(tc, vmselect, "metric.*", tenantID1, start, end, data.Step, data.WantRenderedTargets)
+
+	// Ensure vmselect does not return any data for tenant2.
+	apptest.AssertGraphiteMetricsIndex(tc, vmselect, tenantID2, emptyMetricsIndex)
+	apptest.AssertGraphiteMetricsFind(tc, vmselect, "metric.*", tenantID2, emptyMetricsFind)
+	apptest.AssertGraphiteMetricsExpand(tc, vmselect, "metric.*", tenantID2, emptyMetricsExpand)
+	apptest.AssertGraphiteRender(tc, vmselect, "metric.*", tenantID2, start, end, data.Step, emptyRenderedTargets)
+}

apptest/vmsingle.go

@@ -25,12 +25,14 @@ func StartVmsingle(instance string, flags []string, cli *Client, output io.Write
 			"-httpListenAddr":     "127.0.0.1:0",
 			"-graphiteListenAddr": "127.0.0.1:0",
 			"-opentsdbListenAddr": "127.0.0.1:0",
+			"-vmselectAddr":       "127.0.0.1:0",
 		},
 		extractREs: []*regexp.Regexp{
 			storageDataPathRE,
 			httpListenAddrRE,
 			graphiteListenAddrRE,
 			openTSDBListenAddrRE,
+			vmselectAddrRE,
 		},
 		output: output,
 	})
@@ -43,6 +45,7 @@ func StartVmsingle(instance string, flags []string, cli *Client, output io.Write
 		httpListenAddr:     stderrExtracts[1],
 		graphiteListenAddr: stderrExtracts[2],
 		openTSDBListenAddr: stderrExtracts[3],
+		vmselectAddr:       stderrExtracts[4],
 	}), nil
 }
 
@@ -51,6 +54,7 @@ type vmsingleRuntimeValues struct {
 	httpListenAddr     string
 	graphiteListenAddr string
 	openTSDBListenAddr string
+	vmselectAddr       string
 }
 
 func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
@@ -85,6 +89,7 @@ func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
 		},
 		storageDataPath: rt.storageDataPath,
 		httpListenAddr:  rt.httpListenAddr,
+		vmselectAddr:    rt.vmselectAddr,
 	}
 }
 
@@ -99,6 +104,7 @@ type Vmsingle struct {
 
 	storageDataPath string
 	httpListenAddr  string
+	vmselectAddr    string
 }
 
 // HTTPAddr returns the address at which the vminsert process is
@@ -107,6 +113,12 @@ func (app *Vmsingle) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
+// VmselectAddr returns the address at which the vmsingle process is listening
+// for vmselect connections.
+func (app *Vmsingle) VmselectAddr() string {
+	return app.vmselectAddr
+}
+
 // String returns the string representation of the vmsingle app state.
 func (app *Vmsingle) String() string {
 	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q}", []any{

dashboards/query-stats.json

@@ -1,4 +1,59 @@
 {
+  "__inputs": [
+    {
+      "name": "DS_VICTORIALOGS",
+      "label": "VictoriaLogs",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "victoriametrics-logs-datasource",
+      "pluginName": "VictoriaLogs"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "12.4.3"
+    },
+    {
+      "type": "panel",
+      "id": "logs",
+      "name": "Logs",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "text",
+      "name": "Text",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "victoriametrics-logs-datasource",
+      "name": "VictoriaLogs",
+      "version": "0.29.0"
+    }
+  ],
   "annotations": {
     "list": [
       {
@@ -18,7 +73,6 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 861,
   "links": [
     {
       "icon": "doc",
@@ -78,7 +132,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -109,7 +164,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -117,7 +172,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| type:=\"instant\"\n| count()",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| type:=\"instant\"\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -140,7 +195,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -171,7 +227,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -179,7 +235,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| type:=\"range\"\n| count()",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| type:=\"range\"\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -202,7 +258,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           }
@@ -233,7 +290,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -241,7 +298,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| series_fetched:=0\n| count()",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| series_fetched:=0\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -265,7 +322,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               }
             ]
           },
@@ -297,7 +355,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -305,7 +363,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| stats min(start_ms)\n",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| stats min(start_ms)\n",
           "queryType": "stats",
           "refId": "A"
         }
@@ -327,10 +385,6 @@
       "type": "row"
     },
     {
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "gridPos": {
         "h": 2,
         "w": 24,
@@ -347,8 +401,7 @@
         "content": "To filter by specific query copy its hash from the table and put it into `query_hash` filter on the top. To disable filtering enter `*`.",
         "mode": "markdown"
       },
-      "pluginVersion": "11.6.0",
-      "title": "",
+      "pluginVersion": "12.4.3",
       "transparent": true,
       "type": "text"
     },
@@ -367,6 +420,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -374,7 +430,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -423,18 +480,6 @@
                 "value": 204
               }
             ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "duration_max"
-            },
-            "properties": [
-              {
-                "id": "custom.width",
-                "value": 122
-              }
-            ]
           }
         ]
       },
@@ -447,18 +492,10 @@
       "id": 4,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -466,7 +503,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(execution_duration_ms) duration_max \n| sort by(duration_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(execution_duration_ms) duration_max \n| sort by(duration_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -478,7 +515,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -533,6 +570,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -547,7 +585,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -580,7 +619,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -588,7 +627,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(execution_duration_ms) execution_duration_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(execution_duration_ms) execution_duration_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -611,6 +650,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -618,7 +660,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -680,18 +723,10 @@
       "interval": "1m",
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -699,7 +734,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(series_fetched) series_fetched_max\n| sort by(series_fetched_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(series_fetched) series_fetched_max\n| sort by(series_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -711,7 +746,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -766,6 +801,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -780,7 +816,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -813,7 +850,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -821,7 +858,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(series_fetched) series_fetched_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(series_fetched) series_fetched_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -844,6 +881,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -851,7 +891,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -912,18 +953,10 @@
       "id": 5,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -931,7 +964,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(samples_fetched) samples_fetched_max\n| sort by(samples_fetched_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(samples_fetched) samples_fetched_max\n| sort by(samples_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -942,8 +975,10 @@
           "id": "extractFields",
           "options": {
             "delimiter": ",",
+            "format": "json",
+            "keepTime": false,
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -998,6 +1033,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1012,7 +1048,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1045,7 +1082,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1053,7 +1090,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(samples_fetched) samples_fetched_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(samples_fetched) samples_fetched_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1076,6 +1113,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -1083,7 +1123,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1144,18 +1185,10 @@
       "id": 11,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1163,7 +1196,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(bytes) bytes_fetched_max \n| sort by(bytes_fetched_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(bytes) bytes_fetched_max \n| sort by(bytes_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1175,7 +1208,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -1230,6 +1263,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1244,7 +1278,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1277,7 +1312,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1285,7 +1320,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(bytes) bytes_fetched",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(bytes) bytes_fetched",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1308,6 +1343,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -1315,7 +1353,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1388,18 +1427,10 @@
       "id": 13,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1407,7 +1438,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(memory_estimated_bytes) memory_estimated_max\n| sort by(memory_estimated_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(memory_estimated_bytes) memory_estimated_max\n| sort by(memory_estimated_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1419,7 +1450,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -1474,6 +1505,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1488,7 +1520,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1521,7 +1554,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1529,7 +1562,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(memory_estimated_bytes) memory_estimated_bytes",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(memory_estimated_bytes) memory_estimated_bytes",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1552,6 +1585,9 @@
             "cellOptions": {
               "type": "auto"
             },
+            "footer": {
+              "reducers": []
+            },
             "inspect": false
           },
           "mappings": [],
@@ -1559,7 +1595,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1620,18 +1657,10 @@
       "id": 18,
       "options": {
         "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1639,7 +1668,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(range_ms) range_max \n| sort by(range_max) desc | limit $top",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(range_ms) range_max \n| sort by(range_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1651,7 +1680,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "Line"
+            "source": "labels"
           }
         },
         {
@@ -1706,6 +1735,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1720,7 +1750,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1753,7 +1784,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.6.0",
+      "pluginVersion": "12.4.3",
       "targets": [
         {
           "datasource": {
@@ -1761,7 +1792,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:in($query_hash)\n| stats max(range_ms) range_max",
+          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:$query_hash\n| stats max(range_ms) range_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1770,7 +1801,7 @@
       "type": "timeseries"
     },
     {
-      "collapsed": false,
+      "collapsed": true,
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1778,52 +1809,59 @@
         "y": 56
       },
       "id": 12,
-      "panels": [],
-      "title": "Query log",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "victoriametrics-logs-datasource",
-        "uid": "${ds}"
-      },
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 14,
-        "w": 24,
-        "x": 0,
-        "y": 57
-      },
-      "id": 6,
-      "options": {
-        "dedupStrategy": "none",
-        "enableInfiniteScrolling": false,
-        "enableLogDetails": true,
-        "prettifyLogMessage": false,
-        "showCommonLabels": false,
-        "showLabels": false,
-        "showTime": false,
-        "sortOrder": "Descending",
-        "wrapLogMessage": false
-      },
-      "pluginVersion": "11.6.0",
-      "targets": [
+      "panels": [
         {
           "datasource": {
             "type": "victoriametrics-logs-datasource",
             "uid": "${ds}"
           },
-          "editorMode": "code",
-          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:in($query_hash)\n| limit 200",
-          "queryType": "instant",
-          "refId": "A"
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 14,
+            "w": 24,
+            "x": 0,
+            "y": 57
+          },
+          "id": 6,
+          "options": {
+            "dedupStrategy": "none",
+            "detailsMode": "sidebar",
+            "enableInfiniteScrolling": false,
+            "enableLogDetails": true,
+            "fontSize": "small",
+            "prettifyLogMessage": false,
+            "showCommonLabels": false,
+            "showControls": true,
+            "showLabels": false,
+            "showTime": false,
+            "sortOrder": "Descending",
+            "syntaxHighlighting": false,
+            "unwrappedColumns": false,
+            "wrapLogMessage": true
+          },
+          "pluginVersion": "12.4.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-logs-datasource",
+                "uid": "${ds}"
+              },
+              "direction": "desc",
+              "editorMode": "code",
+              "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:$query_hash\n| limit 200",
+              "queryType": "instant",
+              "refId": "A"
+            }
+          ],
+          "title": "Raw logs",
+          "type": "logs"
         }
       ],
-      "title": "Raw logs",
-      "type": "logs"
+      "title": "Query log",
+      "type": "row"
     },
     {
       "collapsed": true,
@@ -1831,7 +1869,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 71
+        "y": 57
       },
       "id": 17,
       "panels": [
@@ -1848,7 +1886,7 @@
             "h": 14,
             "w": 24,
             "x": 0,
-            "y": 70
+            "y": 58
           },
           "id": 15,
           "options": {
@@ -1857,12 +1895,14 @@
             "enableLogDetails": true,
             "prettifyLogMessage": false,
             "showCommonLabels": false,
+            "showControls": false,
             "showLabels": false,
             "showTime": false,
             "sortOrder": "Descending",
+            "unwrappedColumns": false,
             "wrapLogMessage": false
           },
-          "pluginVersion": "11.6.0",
+          "pluginVersion": "12.4.3",
           "targets": [
             {
               "datasource": {
@@ -1870,7 +1910,7 @@
                 "uid": "${ds}"
               },
               "editorMode": "code",
-              "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| series_fetched:=0\n| query_hash:in($query_hash)",
+              "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| series_fetched:=0\n| query_hash:$query_hash",
               "queryType": "instant",
               "refId": "A"
             }
@@ -1885,7 +1925,7 @@
   ],
   "preload": false,
   "refresh": "",
-  "schemaVersion": 41,
+  "schemaVersion": 42,
   "tags": [
     "victoriametrics",
     "victorialogs"
@@ -1894,8 +1934,9 @@
     "list": [
       {
         "current": {
-          "text": "VictoriaLogs",
-          "value": "PD775F2863313E6C7"
+          "text": "",
+          "value": "${ds}",
+          "selected": true
         },
         "name": "ds",
         "options": [],
@@ -1933,21 +1974,17 @@
           }
         ],
         "query": "5,10,15,20",
-        "type": "custom"
+        "type": "custom",
+        "valuesFormat": "csv"
       },
       {
         "allValue": "*",
-        "current": {
-          "text": "All",
-          "value": [
-            "$__all"
-          ]
-        },
+        "current": {},
         "datasource": {
           "type": "victoriametrics-logs-datasource",
           "uid": "${ds}"
         },
-        "definition": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
+        "definition": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
         "includeAll": true,
         "multi": true,
         "name": "tenant",
@@ -1955,12 +1992,13 @@
         "query": {
           "field": "tenant",
           "limit": 25,
-          "query": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
+          "query": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
           "refId": "VictoriaLogsVariableQueryEditor-VariableQuery",
           "type": "fieldValue"
         },
-        "refresh": 1,
+        "refresh": 2,
         "regex": "",
+        "regexApplyTo": "value",
         "type": "query"
       },
       {
@@ -2000,5 +2038,6 @@
   "timezone": "browser",
   "title": "Query Stats (cluster)",
   "uid": "feg3od1zt1fy8e",
-  "version": 1
-}
+  "version": 1,
+  "weekStart": ""
+}

dashboards/victoriametrics.json

@@ -896,7 +896,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(min_over_time(vm_app_version{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
+          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
           "format": "time_series",
           "instant": false,
           "legendFormat": "{{job}}",

dashboards/vm/victoriametrics.json

@@ -897,7 +897,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(min_over_time(vm_app_version{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
+          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
           "format": "time_series",
           "instant": false,
           "legendFormat": "{{job}}",

dashboards/vm/vmagent.json

@@ -892,7 +892,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)",
+          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
           "format": "time_series",
           "instant": false,
           "interval": "",

dashboards/vmagent.json

@@ -891,7 +891,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)",
+          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
           "format": "time_series",
           "instant": false,
           "interval": "",

deployment/docker/Makefile

@@ -3,9 +3,9 @@
 DOCKER_REGISTRIES ?= docker.io quay.io
 DOCKER_NAMESPACE ?= victoriametrics
 
-ROOT_IMAGE ?= alpine:3.23.4
+ROOT_IMAGE ?= alpine:3.24.1
 ROOT_IMAGE_SCRATCH ?= scratch
-CERTS_IMAGE := alpine:3.23.4
+CERTS_IMAGE := alpine:3.24.1
 
 GO_BUILDER_IMAGE := golang:1.26.4
 

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.147.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.147.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.147.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.147.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.147.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.147.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.147.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.146.0
+    image: victoriametrics/vmauth:v1.147.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.147.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.147.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.147.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.147.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts-vmalert.yml

@@ -120,3 +120,39 @@ groups:
           summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
           description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
             Check vmalert's logs for detailed error message."
+
+      - alert: AlertingRuleResultsApproachingLimit
+        expr: |
+          (
+            vmalert_alerting_rules_last_evaluation_samples
+              > on(group,file) group_left()
+            (vmalert_group_rule_results_limit * 0.9)
+          )
+            and on(group,file)
+          (vmalert_group_rule_results_limit > 0)
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Alerting rule {{ $labels.alertname }} in group {{ $labels.group }} is approaching the configured results limit"
+          description: "Alerting rule \"{{ $labels.alertname }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\" produced {{ $value }} samples in last evaluation, which approaches the configured results limit.
+            If the produced results exceed the limit, the rule will be marked with an error and all its results will be discarded. 
+            Try increasing the results limit for the group or reducing the number of series produced by the rule. See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups."
+
+      - alert: RecordingRuleResultsApproachingLimit
+        expr: |
+          (
+            vmalert_recording_rules_last_evaluation_samples
+              > on(group,file) group_left()
+            (vmalert_group_rule_results_limit * 0.9)
+          )
+            and on(group,file)
+          (vmalert_group_rule_results_limit > 0)
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Recording rule {{ $labels.recording }} in group {{ $labels.group }} is approaching the configured results limit"
+          description: "Recording rule \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\" produced {{ $value }} samples in last evaluation, which approaches the configured results limit.
+            If the produced results exceed the limit, the rule will be marked with an error and all its results will be discarded. 
+            Try increasing the results limit for the group or reducing the number of series produced by the rule. See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups."

deployment/docker/rules/alerts-vmauth.yml

@@ -56,3 +56,20 @@ groups:
           summary: "Too many errors served for user {{ $labels.username }} (instance {{ $labels.instance }})"
           description: "Requests from user {{ $labels.username }} are receiving errors.
             Please check the vmauth logs to verify that the configuration is correct and clients are sending valid requests."
+      - alert: InvalidAuthTokenRequestErrors
+        expr: sum(increase(vmauth_http_request_errors_total{reason="invalid_auth_token"}[5m])) without (instance, reason) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "{{ $externalURL }}/d/nbuo5Mr4k?viewPanel=16&var-job={{ $labels.job }}"
+          summary: "vmauth {{ $labels.job }} is receiving many requests with invalid auth tokens"
+          description: |
+            vmauth {{ $labels.job }} received {{ $value }} requests with invalid auth tokens in the last 5 minutes.
+            This may indicate:
+              - credentials have been updated on vmauth but not on clients
+              - client misconfiguration or use of an expired token
+              - a brute-force attack.
+
+            Check vmauth metrics for longevity and scale of the issue. 
+            Check access log for detailed information: https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.147.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.147.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.147.0
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/CHANGELOG.md

@@ -17,11 +17,11 @@ Please find the changelog for VictoriaMetrics Anomaly Detection below.
 ## v1.29.7
 Released: 2026-06-25
 
-- UI: updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.1](https://docs.victoriametrics.com/anomaly-detection/ui/#v171) to [v1.7.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v172), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v172) for details.
+- UI: updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.1](https://docs.victoriametrics.com/anomaly-detection/ui/#v171) to [v1.7.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v172), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v172) for details. Notable mentions include `api/v1/server/model` endpoint for accessing production models config and queries from UI, manually or through [AI assistant](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance).
 
 - IMPROVEMENT: Increased high-cardinality inference scaling by optionally scattering periodic infer jobs to reduce contention on shared resources (e.g. datasource, CPU, RAM) when `settings.n_workers > 1` and `scheduler.infer_every` is smaller than the total time to fetch and process all queries. This is controlled by new `scatter_infer_jobs` boolean argument of [Periodic Scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#parameters-1) (default: `false`).
 
-- IMPROVEMENT: Optimized internal batching for reader post-fetch series processing, exposing reader processing queue depth, and clarifying inference skip logs after data fetch timeouts.
+- IMPROVEMENT: Optimized internal batching for reader post-fetch series processing, exposing reader processing queue depth (`vmanomaly_reader_processing_tasks_queued` [metric](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics)), and clarifying inference skip logs after data fetch timeouts. See `series_processing_batch_size` argument of [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [VLogsReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#victorialogs-reader) for details.
 
 - IMPROVEMENT: Refined `VmReader` and `VLogsReader` logging after datasource request failures by suppressing the follow-up generic "No data" or "No unseen data" warning for failed fetches. Failed requests now keep the original datasource error while empty successful responses still emit the no-data warning.
 
@@ -283,7 +283,7 @@ Released: 2025-06-13
 ## v1.23.2
 Released: 2025-06-09
 
-- IMPROVEMENT: Increased convergence speed for [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score), [ZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score), [MADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#mad), and [OnlineMADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) models. Now it works better for tight optimization budgets (n_trials < 10, timeout < 1s)
+- IMPROVEMENT: Increased convergence speed for [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score), [ZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score), [MADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#mad-median-absolute-deviation), and [OnlineMADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) models. Now it works better for tight optimization budgets (n_trials < 10, timeout < 1s)
 
 - BUGFIX: Now mean and variance of [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) with exponential `decay` < 1 [arg](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay) are properly calculated for unbiased predictions.
 
@@ -538,7 +538,7 @@ Released: 2024-10-01
 
 > A bug was discovered in this release that causes the service to crash. Please use the patch [v1.16.1](#v1161) to resolve this issue.
 
-- FEATURE: Introduced data dumps to a host filesystem for [VmReader](https://docs.victoriametrics.com/anomaly-detection/#vm-reader).  Resource-intensive setups (multiple queries returning many metrics, bigger `fit_window` arg) will have RAM consumption reduced during fit calls.
+- FEATURE: Introduced data dumps to a host filesystem for [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader).  Resource-intensive setups (multiple queries returning many metrics, bigger `fit_window` arg) will have RAM consumption reduced during fit calls.
 - IMPROVEMENT: Added a `groupby` argument for logical grouping in [multivariate models](https://docs.victoriametrics.com/anomaly-detection/components/models/#multivariate-models). When specified, a separate multivariate model is trained for each unique combination of label values in the `groupby` columns. For example, to perform multivariate anomaly detection on metrics at the machine level without cross-entity interference, you can use `groupby: [host]` or `groupby: [instance]`, ensuring one model per entity being trained (e.g., per host). Please find more details [here](https://docs.victoriametrics.com/anomaly-detection/components/models/#group-by).
 - IMPROVEMENT: Improved performance of [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) on multicore instances for reading and data processing.
 - IMPROVEMENT: Introduced new CLI argument aliases to enhance compatibility with [Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md) (i.e. using secrets) and better align with [VictoriaMetrics flags](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#list-of-command-line-flags):
@@ -685,7 +685,7 @@ Released: 2024-02-15
 
 ## v1.9.2
 Released: 2024-01-29
-- BUGFIX: now multivariate models (like [`IsolationForestMultivariateModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-foresthttpsenwikipediaorgwikiisolation_forest-multivariate)) are properly handled throughout fit/infer phases.
+- BUGFIX: now multivariate models (like [`IsolationForestMultivariateModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate)) are properly handled throughout fit/infer phases.
 
 
 ## v1.9.1

docs/anomaly-detection/UI.md

@@ -553,7 +553,7 @@ preset: ui
 # other optional server/settings parameters, e.g. port, max_concurrent_tasks, n_workers, logger_levels, etc.
 ```
 
-using one of the [deployment methods](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly) in a [QuickStart guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/#quickstart), e.g. via Docker.
+using one of the [deployment methods](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly) in a [QuickStart guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/), e.g. via Docker.
 
 Retrieve the UI at `http://<vmanomaly-host>:<port>` (e.g. at `http://localhost:8490` if running locally with default port) and start exploring anomaly detection models and their configurations interactively.
 

docs/anomaly-detection/components/reader.md

@@ -893,6 +893,19 @@ If a path to a CA bundle file (like `ca.crt`), it will verify the certificate us
 (Optional) Password for authentication. If set, it will be used to authenticate the request.
             </td>
         </tr>
+        <tr>
+            <td>
+
+<span style="white-space: nowrap;">`series_processing_batch_size`</span>
+            </td>
+            <td>
+
+`8`
+            </td>
+            <td>
+Optional argument {{% available_from "v1.29.7" anomaly %}}, allows specifying the number of time series to process together while preparing data for fit or infer stages. Defaults to `8`. Suggested values are 4-16 for high-cardinality queries.
+            </td>
+        </tr>
     </tbody>
 </table>
 
@@ -911,6 +924,7 @@ reader:
   # tenant_id: '0:0'  # for cluster version only
   sampling_period: '1m'
   max_points_per_query: 10000
+  series_processing_batch_size: 8
   data_range: [0, 'inf']  # reader-level
   offset: '0s'  # reader-level
   timeout: '30s'

docs/anomaly-detection/components/server.md

@@ -32,7 +32,7 @@ Server component of VictoriaMetrics Anomaly Detection (`vmanomaly`) is responsib
 ### Example Configuration
 
 > [!TIP]
-> If [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#hot-reloading) is enabled in vmanomaly service, the server will automatically pick up changes made to the configuration file without requiring a restart.
+> If [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) is enabled in vmanomaly service, the server will automatically pick up changes made to the configuration file without requiring a restart.
 
 ```yaml
 server:
@@ -63,4 +63,4 @@ reader:
 
 After starting the `vmanomaly` server with the above configuration, UI can be accessed at `<vmanomaly-host>:8490/vmanomaly/vmui/` (e.g. `http://localhost:8490/vmanomaly/vmui/`).
 
-Rest API endpoints (e.g. `/metrics`) can be accessed at `<vmanomaly-host>:8490/vmanomaly/metrics` (e.g. `http://localhost:8490/vmanomaly/metrics`).
+Rest API endpoints (e.g. `/metrics`) can be accessed at `<vmanomaly-host>:8490/vmanomaly/metrics` (e.g. `http://localhost:8490/vmanomaly/metrics`).

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -10,9 +10,9 @@ sitemap:
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.146.0)
-  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.146.0)
-  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.146.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.147.0)
+  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.147.0)
+  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.147.0)
 - [Grafana](https://grafana.com/) (v12.2.0)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
@@ -323,7 +323,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.147.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -340,7 +340,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.147.0
     ports:
       - 8428:8428
     volumes:
@@ -373,7 +373,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.147.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -240,23 +240,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.147.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.147.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.147.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.147.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.147.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -308,7 +308,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.146.0
+    image: docker.io/victoriametrics/vmauth:v1.147.0
     ports:
       - 8427:8427
     volumes:

docs/guides/vm-architectures/README.md

@@ -39,7 +39,7 @@ Each subsequent section of this guide presents an architecture designed to handl
 ### The decision tree
 
 <p align="center">
-<img src="decision-tree.webp" alt="Decision Tree" width="80%">
+<img src="/guides/vm-architectures/decision-tree.webp" alt="Decision Tree" width="80%">
 </p>
 
 ## Basic
@@ -62,7 +62,7 @@ Installation guide reference: [VictoriaMetrics Single](https://docs.victoriametr
 **Schema:**
 
 <p align="center">
-<img src="basic-architecture.webp" alt="Basic Architecture" width="40%">
+<img src="/guides/vm-architectures/basic-architecture.webp" alt="Basic Architecture" width="40%">
 </p>
 
 ### Unavailability Scenarios
@@ -93,7 +93,7 @@ High availability implementation: [HA VictoriaMetrics Cluster](https://docs.vict
 **Schema:**
 
 <p align="center">
-<img src="single-az-architecture.webp" alt="Single AZ Architecture" width="60%">
+<img src="/guides/vm-architectures/single-az-architecture.webp" alt="Single AZ Architecture" width="60%">
 </p>
 
 ### Application vs. Storage Replication
@@ -210,7 +210,7 @@ To ensure reliability, vmagent implements the bulkhead pattern: each destination
 **Schema:**
 
 <p align="center">
-<img src="multi-az-architecture.webp" alt="Multi-AZ Architecture" width="65%">
+<img src="/guides/vm-architectures/multi-az-architecture.webp" alt="Multi-AZ Architecture" width="65%">
 </p>
 
 ### Unavailability Scenarios
@@ -257,7 +257,7 @@ For complete disaster recovery, this entire cell-based architecture is duplicate
 A global, stateless layer of routing cells (vmagent, vmauth) sits on top. It routes traffic to several logical groups of storage cells. Each storage group contains multiple AZs, and data is replicated or sharded across them. There are several approaches to implementing it.
 
 <p align="center">
-<img src="hyperscale-architecture.webp" alt="Hyperscale Architecture" width="85%">
+<img src="/guides/vm-architectures/hyperscale-architecture.webp" alt="Hyperscale Architecture" width="85%">
 </p>
 
 ### Choosing Your Read Path Strategy
@@ -375,7 +375,7 @@ This multitenancy approach gives us another trade-off in the isolation implement
 **Schema:**
 
 <p align="center">
-<img src="logical-layers-architecture.webp" alt="Logical Layers Architecture" width="80%">
+<img src="/guides/vm-architectures/logical-layers-architecture.webp" alt="Logical Layers Architecture" width="80%">
 </p>
 
 **Path A: Shared resources.** We have a single, shared pool of all cluster components.

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.147.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.147.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.147.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.146.0-enterprise
+    image: victoriametrics/vmauth:v1.147.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.147.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.147.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.147.0
     depends_on:
       - victoriametrics
       - alertmanager

docs/victoriametrics/Articles.md

@@ -49,7 +49,7 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [Brewblox: InfluxDB to Victoria Metrics](https://www.brewblox.com/dev/decisions/20210718_victoria_metrics.html)
 * [Techetio: Evaluating Backend Options For Prometheus Metrics](https://www.techetio.com/2022/08/21/evaluating-backend-options-for-prometheus-metrics/)
 * [Asserts: Announcing Asserts](https://www.asserts.ai/blog/announcing-asserts/)
-* [Optimizing Linkerd metrics in Prometheus](https://aatarasoff.medium.com/optimizing-linkerd-metrics-in-prometheus-de607ec10f6b)
+* [Optimizing Linkerd metrics in Prometheus](https://itnext.io/optimizing-linkerd-metrics-in-prometheus-de607ec10f6b)
 * [VictoriaMetrics vs. OpenTSDB](https://blg.robot-house.us/posts/tsdbs-grow/)
 * [Monitoring of multiple OpenShift clusters with VictoriaMetrics](https://medium.com/ibm-garage/monitoring-of-multiple-openshift-clusters-with-victoriametrics-d4f0979e2544)
 * [Ultra Monitoring with Victoria Metrics](https://dev.to/aws-builders/ultra-monitoring-with-victoria-metrics-1p2)

docs/victoriametrics/CONTRIBUTING.md

@@ -85,6 +85,21 @@ Pull requests requirements:
 
 See a good example of a [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6487).
 
+## AI policy
+
+You are free to use any AI tools when working on a contribution, on code,
+documentation, issues, or anything else. You do not need to disclose whether or
+how you used them.
+
+With or without the help of AI, you are responsible for the changes you submit.
+Take the effort to understand the code base and every change in your pull request,
+and clean up any AI slop before sending it. Do not use AI to automate your
+responses to maintainers.
+
+We review contributions on their quality, regardless of how they were produced. A
+pull request or issue that looks like unreviewed AI output, with low-quality or
+broken changes, may be closed without a detailed review or triage.
+
 ## Merging Pull Request
 
 The person who merges the Pull Request is responsible for satisfying the requirements below:

docs/victoriametrics/CaseStudies.md

@@ -482,7 +482,7 @@ Across our production VictoriaMetrics clusters, in a 12 months period we go beyo
 
 ## Roblox
 
-[Roblox](https://roblox.com/) builds the tools and platform that empower people to create their own immersive experiences,
+[Roblox](https://www.roblox.com/) builds the tools and platform that empower people to create their own immersive experiences,
 so that any world they can imagine can be brought to life.
 
 With more than 200 million active monthly users, Roblox is one of the most popular gaming platforms on the Internet.
@@ -688,8 +688,7 @@ Thanos, Cortex and VictoriaMetrics were evaluated as a long-term storage for Pro
 - The API is compatible with Prometheus and nearly all standard PromQL queries work well out of the box.
 - Handles storage well, with periodic compaction which makes it easy to take snapshots.
 
-Please see [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit) slides,
-[video](https://youtu.be/ZJQYW-cFOms) and [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/) blog post for more details.
+Please see [Monitoring K8S with VictoriaMetrics](https://youtu.be/ZJQYW-cFOms) video and [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/) blog post for more details.
 
 ## zhihu
 

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -639,7 +639,7 @@ Also in the cluster version the `/prometheus/api/v1` endpoint ingests  `jsonl`,
     - `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` at `vmselect` (see below), JSON line format.
     - `prometheus/api/v1/import/native` - for importing data obtained via `api/v1/export/native` on `vmselect` (see below).
     - `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data) for details.
-    - `prometheus/api/v1/import/prometheus` - for importing data in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format) and in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md). This endpoint also supports [Pushgateway protocol](https://github.com/prometheus/pushgateway#url). See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
+    - `prometheus/api/v1/import/prometheus` - for importing data in [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format) and in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md). This endpoint also supports [Pushgateway protocol](https://github.com/prometheus/pushgateway#url). See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
     - `opentelemetry/v1/metrics` - for ingesting data via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/97c826b70e2f89cfdf655d5150791f3f0c2bae19/specification/metrics/data-model.md). See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/).
     - `datadog/api/v1/series` - for ingesting data with DataDog submit metrics API v1. See [these docs](https://docs.victoriametrics.com/victoriametrics/url-examples/#datadogapiv1series) for details.
     - `datadog/api/v2/series` - for ingesting data with [DataDog submit metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/) for details.

docs/victoriametrics/FAQ.md

@@ -247,7 +247,7 @@ See [How to migrate from InfluxDB to VictoriaMetrics](https://docs.victoriametri
 * TimescaleDB insists on using SQL as a query language. While SQL is more powerful than PromQL, this power is rarely required during typical usages of a TSDB. Real-world queries usually [look clearer and simpler when written in PromQL than in SQL](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085).
 * VictoriaMetrics requires [up to 70x less storage space compared to TimescaleDB](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) for storing the same amount of time series data. The gap in storage space usage can be decreased from 70x to 3x if [compression in TimescaleDB is properly configured](https://docs.timescale.com/use-timescale/latest/compression/) (it isn't an easy task in general :)).
 * VictoriaMetrics requires up to 10x less CPU and RAM resources than TimescaleDB for processing production data. See [this article](https://abiosgaming.com/press/high-cardinality-aggregations/) for details.
-* TimescaleDB is [harder to set up, configure and operate](https://docs.timescale.com/timescaledb/latest/how-to-guides/install-timescaledb/self-hosted/ubuntu/installation-apt-ubuntu/) than VictoriaMetrics (see [how to run VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-start-victoriametrics)).
+* TimescaleDB is [harder to set up, configure and operate](https://www.tigerdata.com/docs/get-started/choose-your-path/install-timescaledb#tab=ubuntu) than VictoriaMetrics (see [how to run VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-start-victoriametrics)).
 * VictoriaMetrics accepts data in multiple popular data ingestion protocols – InfluxDB, OpenTSDB, Graphite, CSV – while TimescaleDB supports only SQL inserts.
 * VictoriaMetrics can be queried via [Graphite's API](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#graphite-api-usage).
 
@@ -269,7 +269,7 @@ We provide commercial support for both versions. [Contact us](https://victoriame
 [VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_faq) – the most cost-efficient hosted monitoring platform, operated by VictoriaMetrics core team.
 
 <!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
-## Why doesn't VictoriaMetrics support the [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_read%3E)? {#why-doesnrsquot-victoriametrics-support-the-prometheus-remote-read-api-}
+## Why doesn't VictoriaMetrics support the [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read)? {#why-doesnrsquot-victoriametrics-support-the-prometheus-remote-read-api-}
 
 The remote read API requires transferring all the raw data for all the requested metrics over the given time range. For instance,
 if a query covers 1000 metrics with 10K values each, then the remote read API has to return `1000*10K`=10M metric values to Prometheus.
@@ -338,13 +338,15 @@ File bugs and feature requests in our [GitHub Issues](https://github.com/Victori
 ## Where can I find information about multi-tenancy?
 
 See [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).
-Multitenancy is supported only by the [cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) of VictoriaMetrics.
+Multitenancy is fully supported only by the [cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) of VictoriaMetrics.
+Single-node provides limited [multitenancy](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy) support to facilitate
+the migration [from single-node to cluster](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#from-single-node-to-cluster).
 
 ## How to set a memory limit for VictoriaMetrics components?
 
 All VictoriaMetrics components provide command-line flags to control the size of internal buffers and caches:
 `-memory.allowedPercent` and `-memory.allowedBytes` (pass `-help` to any VictoriaMetrics component in order to see the description for these flags).
-These limits don't take into account additional memory, which may be needed for processing incoming queries.
+These limits don't account for additional memory that may be needed to process incoming queries.
 Hard limits may be enforced only by the OS via [cgroups](https://en.wikipedia.org/wiki/Cgroups),
 Docker (see [these docs](https://docs.docker.com/config/containers/resource_constraints)) or
 Kubernetes (see [these docs](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers)).
@@ -465,9 +467,13 @@ Cluster version of VictoriaMetrics may be preferred over single-node VictoriaMet
 
 ## How to migrate data from single-node VictoriaMetrics to cluster version?
 
-The single-node version of VictoriaMetrics stores data on disk in slightly different format compared to the cluster version of VictoriaMetrics.
-This makes it impossible to just copy the on-disk data from `-storageDataPath` directory from single-node VictoriaMetrics to a `vmstorage` node in VictoriaMetrics cluster.
-If you need to migrate data from a single-node VictoriaMetrics to the cluster version, then [follow these instructions](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
+The single-node version of VictoriaMetrics stores data on disk in a slightly different format compared to the cluster version of VictoriaMetrics.
+This makes it impossible to just copy the on-disk data from the `-storageDataPath` directory from single-node VictoriaMetrics to a `vmstorage` node in a VictoriaMetrics cluster.
+
+There are two options, however:
+
+1. Deploy a new cluster next to the existing single-node and let them co-exist until the cluster is filled with new data and the old data in vmsingle becomes outside of the retention period. This option requires no data migration or downtime. See instructions [here](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#from-single-node-to-cluster).
+2. If you need to actually migrate data from a single-node VictoriaMetrics to the cluster version (and/or possibly modify it), then follow [these vmctl instructions](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
 ## Why isn't MetricsQL 100% compatible with PromQL?
 

docs/victoriametrics/MetricsQL.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 #### holt_winters
 
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
-(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
 Both `sf` and `tf` must be in the range `[0...1]`.
 
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
 #### acosh
 
 `acosh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
+[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
 #### asinh
 
 `asinh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
+[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
 #### atanh
 
 `atanh(q)` is a [transform function](#transform-functions), which returns
-[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
+[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
 
 Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
@@ -1229,8 +1229,7 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The result will preserve the first and the last bucket to improve accuracy for min and max values. 
-So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
+The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

docs/victoriametrics/Quick-Start.md

@@ -36,7 +36,6 @@ VictoriaMetrics is available as:
 * [Ansible Roles](https://github.com/VictoriaMetrics/ansible-playbooks)
 * [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
   See [How to build from sources](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-build-from-sources)
-* [VictoriaMetrics on Linode](https://www.linode.com/marketplace/apps/victoriametrics/victoriametrics/)
 * [VictoriaMetrics on DigitalOcean](https://marketplace.digitalocean.com/apps/victoriametrics-single)
 
 Just download VictoriaMetrics and follow [these instructions](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-start-victoriametrics).
@@ -61,9 +60,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.146.0
+docker pull victoriametrics/victoria-metrics:v1.147.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.146.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.147.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

docs/victoriametrics/README.md

@@ -245,7 +245,7 @@ The following steps must be performed during the upgrade / downgrade procedure:
 * Wait until the process stops. This can take a few seconds.
 * Start the upgraded VictoriaMetrics.
 
-Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
+Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/whats-new-in-prometheus-2-8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
 
 > If you'd prefer not to manage upgrades yourself, [VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_single_upgrade)
 > performs version upgrades automatically during maintenance windows with no action required on your part.
@@ -401,6 +401,7 @@ Resources:
 
 * [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
 * [Cardinality explorer blog post](https://victoriametrics.com/blog/cardinality-explorer/).
+* [skills/victoriametrics-cardinality-analysis](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/victoriametrics-cardinality-analysis/SKILL.md) for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
 
 ### Cardinality explorer statistic inaccuracy
 
@@ -417,7 +418,7 @@ VictoriaMetrics is configured via command-line flags, so it must be restarted wh
 * Wait until the process stops. This can take a few seconds.
 * Start VictoriaMetrics with the new command-line flags.
 
-Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
+Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/whats-new-in-prometheus-2-8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
 
 ## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
 
@@ -478,6 +479,11 @@ and [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyco
 to the given number of digits after the decimal point.
 For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
 
+VictoriaMetrics accepts `optimize_repeated_binary_op_subexprs=1` query arg for [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query)
+handler. It allows `vmselect` to execute left and right sides of binary operators sequentially when they contain the same
+optimized aggregate rollup result expression, so the second side may reuse the rollup result cache populated by the first side.
+The optimization is disabled by default and applies only when rollup result cache can be used for the request.
+
 VictoriaMetrics accepts `limit` query arg for [/api/v1/labels](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels)
 and [`/api/v1/label/<labelName>/values`](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues) handlers for limiting the number of returned entries.
 For example, the query to `/api/v1/labels?limit=5` returns a sample of up to 5 unique labels, while ignoring the rest of labels.
@@ -994,7 +1000,7 @@ Note that it could be required to flush response cache after importing historica
 ### How to import data in Prometheus exposition format
 
 VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/main/docs/instrumenting/exposition_formats.md),
-in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md)
+in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md)
 and in [Pushgateway format](https://github.com/prometheus/pushgateway#url) via `/api/v1/import/prometheus` path.
 
 For example, the following command imports a single line in Prometheus exposition format into VictoriaMetrics:
@@ -1676,9 +1682,46 @@ See also [retention filters](#retention-filters).
 The downsampling can be evaluated for free by downloading and using enterprise binaries from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
 See how to request a [free trial license](https://victoriametrics.com/products/enterprise/trial/).
 
-## Multi-tenancy
+## Multitenancy {#multi-tenancy}
 
-Single-node VictoriaMetrics doesn't support multi-tenancy. Use the [cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) instead.
+Single-node VictoriaMetrics has limited
+[multitenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
+support {{% available_from "v1.147.0" %}}. Specifically, a single-node can serve
+multitenant queries as if it were a `vmstorage`. The write path is not supported.
+
+The functionality is disabled by default and can be enabled by setting the
+`-vmselectAddr` flag. This will start the `vmselect RPC server` that accepts
+requests and serves responses in cluster format.
+
+Cluster data format assumes the presence of a `tenantID`. Single-node data
+format still does not support multitenancy, but it is possible to configure the
+single-node to specify which `tenantID` the single-node's data corresponds to with the
+`-accountID` and `-projectID` flags. Both are `0` by default, which means that
+`"0:0"` `tenantID` is used by default.
+
+For example, the following command will start a single-node that listens for
+vmselect RPC requests on the `8401` port. The requests must be either `multitenant`
+(i.e., want data for all tenants) or for `"12:34"` tenant. Otherwise, the
+single-node will return an empty result:
+
+```shell
+./victoria-metrics -storageDataPath=/data -vmselectAddr=:8401 -accountID=12 -projectID=34
+```
+
+The `tenantID` configuration is not persisted in any way and is enforced only at
+runtime. Thus, it is safe to change the `-accountID` and `-projectID` flag
+values at any time.
+
+Note that the single-node's HTTP handlers still do not support multitenancy.
+
+The purpose of this limited multitenancy support is enabling the single-node to
+operate in VictoriaMetrics cluster setups. I.e., one or more single-nodes that
+contain data for different tenants can be a part of a cluster, and the entire
+non-homogeneous deployment can be queried with a higher-level `vmselect`.
+
+This, in turn, enables easy [migrations from single-node to cluster](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#from-single-node-to-cluster).
+Previously, the only option was the use of
+[vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
 ## Scalability and cluster version
 
@@ -1712,64 +1755,63 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | ✅                 |
 | other releases                                                                 | ❌                 |
 
-### Software Bill of Materials (SBOM)
-
-Every VictoriaMetrics container{{% available_from "v1.137.0" %}} image published to
-[Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay.io](https://quay.io/organization/victoriametrics) include an [SPDX](https://spdx.dev/) SBOM attestation generated automatically by BuildKit during `docker buildx build`.
-
-To inspect the SBOM for an image:
-
-```sh
-docker buildx imagetools inspect \
-  docker.io/victoriametrics/victoria-metrics:latest \
-  --format "{{ json .SBOM }}"
-```
-
-To scan an image using its SBOM attestation with [Trivy](https://github.com/aquasecurity/trivy):
-
-```sh
-trivy image --sbom-sources oci \
-  docker.io/victoriametrics/victoria-metrics:latest
-```
-
 ### Reporting a Vulnerability
 
 Please report any security issues to <security@victoriametrics.com>
 
+### CVE handling policy
+
+**Source code:** Go dependencies are scanned by [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) in CI.
+All vulnerabilities must be fixed before the next scheduled release and backported to [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
+
+**Docker images:** CVE findings in the [Alpine](https://security.alpinelinux.org/) base image pose minimal risk since VictoriaMetrics binaries are statically compiled with no OS dependencies.
+When detected, only the Alpine base tag is updated.
+Releases proceed as planned even if upstream fixes are not yet available.
+For maximum security, hardened [scratch](https://hub.docker.com/_/scratch)-based images are also provided.
+All images are continuously scanned by Docker Hub and verified before release using [grype](https://github.com/anchore/grype).
+
 ### General security recommendations:
 
-* All the VictoriaMetrics components must run in protected private networks without direct access from untrusted networks such as Internet.
+* All VictoriaMetrics components must run in protected private networks without direct access from untrusted networks such as the Internet.
   The exception is [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) and [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/),
   which are intended for serving public requests and performing authorization with [TLS termination](https://en.wikipedia.org/wiki/TLS_termination_proxy).
-* All the requests from untrusted networks to VictoriaMetrics components must go through auth proxy such as [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/)
+* All the requests from untrusted networks to VictoriaMetrics components must go through an auth proxy, such as [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/)
   or [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/). The proxy must be set up with proper authentication and authorization.
 * Prefer using lists of allowed API endpoints, while disallowing access to other endpoints when configuring [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/)
   in front of VictoriaMetrics components.
-* Set reasonable [`Strict-Transport-Security`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security) header value to all the components to mitigate [MitM attacks](https://en.wikipedia.org/wiki/Man-in-the-middle_attack), for example: `max-age=31536000; includeSubDomains`. See `-http.header.hsts` flag.
+* Set a reasonable [`Strict-Transport-Security`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security) header value on all the components to mitigate [MitM attacks](https://en.wikipedia.org/wiki/Man-in-the-middle_attack), for example: `max-age=31536000; includeSubDomains`. See `-http.header.hsts` flag.
 * Set reasonable [`Content-Security-Policy`](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP) header value to mitigate [XSS attacks](https://en.wikipedia.org/wiki/Cross-site_scripting). See `-http.header.csp` flag.
 * Set reasonable [`X-Frame-Options`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options) header value to mitigate [clickjacking attacks](https://en.wikipedia.org/wiki/Clickjacking), for example `DENY`. See `-http.header.frameOptions` flag.
 
-VictoriaMetrics provides the following security-related command-line flags:
+The following security-related command-line flags are available for all components with HTTP API:
 
-* `-tls`, `-tlsCertFile` and `-tlsKeyFile` for switching from HTTP to HTTPS at `-httpListenAddr` (TCP port 8428 is listened by default).
+* `-tls`, `-tlsCertFile` and `-tlsKeyFile` for switching from HTTP to HTTPS at `-httpListenAddr`.
   [Enterprise version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/enterprise/) supports automatic issuing of TLS certificates.
   See [these docs](#automatic-issuing-of-tls-certificates).
 * `-mtls` and `-mtlsCAFile` for enabling [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) for requests to `-httpListenAddr`. See [these docs](#mtls-protection).
 * `-httpAuth.username` and `-httpAuth.password` for protecting all the HTTP endpoints
   with [HTTP Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
+* `-http.header.hsts`, `-http.header.csp`, and `-http.header.frameOptions` for serving `Strict-Transport-Security`, `Content-Security-Policy`
+  and `X-Frame-Options` HTTP response headers.
+
+### Protecting service endpoints
+
+All VictoriaMetrics components expose internal metrics in Prometheus exposition format at the `/metrics` page for [#Monitoring](https://docs.victoriametrics.com/victoriametrics/#monitoring).
+Consider limiting access to the `/metrics` page to trusted networks only.
+
+The following service endpoints may require protection:
+
 * `-deleteAuthKey` for protecting the `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
 * `-snapshotAuthKey` for protecting the `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
 * `-forceFlushAuthKey` for protecting the `/internal/force_flush` endpoint. See [force flush docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-flush).
 * `-forceMergeAuthKey` for protecting the `/internal/force_merge` endpoint. See [force merge docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-merge).
 * `-search.resetCacheAuthKey` for protecting the `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
-* `-reloadAuthKey` for protecting the `/-/reload` endpoint, which is used for force reloading of [`-promscrape.config`](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
+* `-reloadAuthKey` for protecting the `/-/reload` endpoint, which is used to force reload the [`-promscrape.config`](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
 * `-configAuthKey` for protecting the `/config` endpoint, since it may contain sensitive information such as passwords.
 * `-flagsAuthKey` for protecting the `/flags` endpoint.
 * `-pprofAuthKey` for protecting the `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
 * `-metricNamesStatsResetAuthKey` for protecting the `/api/v1/admin/status/metric_names_stats/reset` endpoint, used for [Metric Names Tracker](#track-ingested-metrics-usage).
 * `-denyQueryTracing` for disallowing [query tracing](#query-tracing).
-* `-http.header.hsts`, `-http.header.csp`, and `-http.header.frameOptions` for serving `Strict-Transport-Security`, `Content-Security-Policy`
-  and `X-Frame-Options` HTTP response headers.
 
 Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
 For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`. This protects from unexpected requests from untrusted network interfaces.
@@ -1777,17 +1819,6 @@ For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<i
 See also [security recommendation for VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#security)
 and [the general security page at VictoriaMetrics website](https://victoriametrics.com/security/).
 
-### CVE handling policy
-
-**Source code:** Go dependencies are scanned by [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) in CI. 
-All vulnerabilities must be fixed before next scheduled release and backported to [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
-
-**Docker images:** CVE findings in [Alpine](https://security.alpinelinux.org/) base image pose minimal risk since VictoriaMetrics binaries are statically compiled with no OS dependencies.
-When detected, only the Alpine base tag is updated.
-Releases proceed as planned even if upstream fixes are not yet available.
-For maximum security, hardened [scratch](https://hub.docker.com/_/scratch)-based images are also provided.
-All images are continuously scanned by Docker Hub and verified before release using [grype](https://github.com/anchore/grype).
-
 ### mTLS protection
 
 By default `VictoriaMetrics` accepts http requests at `8428` port (this port can be changed via `-httpListenAddr` command-line flags).
@@ -1817,19 +1848,39 @@ This functionality can be evaluated for free according to [these docs](https://d
 
 See also [security recommendations](#security).
 
+### Software Bill of Materials (SBOM)
+
+Every VictoriaMetrics container{{% available_from "v1.137.0" %}} image published to
+[Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay.io](https://quay.io/organization/victoriametrics) include an [SPDX](https://spdx.dev/) SBOM attestation generated automatically by BuildKit during `docker buildx build`.
+
+To inspect the SBOM for an image:
+
+```sh
+docker buildx imagetools inspect \
+  docker.io/victoriametrics/victoria-metrics:latest \
+  --format "{{ json .SBOM }}"
+```
+
+To scan an image using its SBOM attestation with [Trivy](https://github.com/aquasecurity/trivy):
+
+```sh
+trivy image --sbom-sources oci \
+  docker.io/victoriametrics/victoria-metrics:latest
+```
+
 ## Tuning
 
-* No need in tuning for VictoriaMetrics - it uses reasonable defaults for command-line flags,
+* No need to tune for VictoriaMetrics - it uses reasonable defaults for command-line flags,
   which are automatically adjusted for the available CPU and RAM resources.
-* No need in tuning for Operating System - VictoriaMetrics is optimized for default OS settings.
+* No need to tune for Operating System - VictoriaMetrics is optimized for default OS settings.
   The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a).
-  The recommendation is not specific for VictoriaMetrics only but also for any service which handles many HTTP connections and stores data on disk.
-* VictoriaMetrics is a write-heavy application and its performance depends on disk performance. So be careful with other
-  applications or utilities (like [fstrim](https://manpages.ubuntu.com/manpages/lunar/en/man8/fstrim.8.html))
+  The recommendation is not specific to VictoriaMetrics only, but also for any service that handles many HTTP connections and stores data on disk.
+* VictoriaMetrics is a write-heavy application, and its performance depends on disk performance. So be careful with other
+  applications or utilities (like [fstrim](https://manpages.ubuntu.com/manpages/noble/en/man8/fstrim.8.html))
   which could [exhaust disk resources](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1521).
 * The recommended filesystem is `ext4`, the recommended persistent storage is [persistent HDD-based disk on GCP](https://cloud.google.com/compute/docs/disks/#pdspecs),
-  since it is protected from hardware failures via internal replication and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).
-  If you plan to store more than 1TB of data on `ext4` partition, then the following options are recommended to pass to `mkfs.ext4`:
+  since it is protected from hardware failures via internal replication, and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).
+  If you plan to store more than 1TB of data on an `ext4` partition, then the following options are recommended to pass to `mkfs.ext4`:
 
 ```sh
 mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
@@ -1967,6 +2018,9 @@ in [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victori
 via [cache removal](https://docs.victoriametrics.com/victoriametrics/#cache-removal) procedure. This reset state endpoint can be protected via `-metricNamesStatsResetAuthKey`
 cmd-line flag. See [Security](https://docs.victoriametrics.com/victoriametrics/#security) for details.
 
+See [skills/victoriametrics-unused-metrics-analysis](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/victoriametrics-unused-metrics-analysis/SKILL.md)
+for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis of unused metrics.
+
 ## Query tracing
 
 VictoriaMetrics supports query tracing, which can be used for determining bottlenecks during query processing.
@@ -2035,6 +2089,9 @@ Query tracing is allowed by default. It can be denied by passing `-denyQueryTrac
 * for query tracing - just click `Trace query` checkbox and re-run the query in order to investigate its' trace.
 * for exploring custom trace - go to the tab `Trace analyzer` and upload or paste JSON with trace information.
 
+See also [skills/vm-trace-analyzer](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/vm-trace-analyzer/SKILL.md)
+for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
+
 ## Cardinality limiter
 
 By default, VictoriaMetrics doesn't limit the number of stored time series. The limit can be enforced by setting the following command-line flags:
@@ -2274,6 +2331,30 @@ Things to consider when copying data:
 For scenarios like single-to-cluster, cluster-to-single, re-sharding or migrating only a fraction of data: 
 [see how to migrate data from VictoriaMetrics via vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
+### From Single-node to Cluster
+
+When, for some reason, the deployment needs to be switched from single-node to
+cluster (such as the single-node can't be scaled vertically anymore, or
+multitenancy becomes a requirement, etc.) the migration can be as simple as:
+
+1. Restart the existing single-node with multitenancy support enabled as
+   described in [Multitenancy](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy) section.
+1. Deploy an empty cluster next to the existing single-node.
+1. Deploy higher-level `vmselect` and configure it to query both the existing
+   single-node and the new cluster.
+1. Start writing data to the cluster.
+1. Stop writing data to the single-node.
+
+This approach requires no data migration nor downtime (apart from restarting the
+single-node at step 1). And once the single-node data becomes outside the
+retention period, the single-node can be removed from the deployment.
+
+Note that if you need to actually migrate data to cluster and/or modify it, you
+will need to use
+[vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/)
+instead.
+
+
 ### From other systems
 
 Use [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/) to migrate data from other systems to VictoriaMetrics.

docs/victoriametrics/Troubleshooting.md

@@ -212,6 +212,7 @@ These are the most common reasons for slow data ingestion in VictoriaMetrics:
    - Reduce the number of active time series. The [official Grafana dashboards for VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
      contain a graph showing the number of active time series. Use the [cardinality explorer](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-explorer)
      to determine and fix the source of [high cardinality](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-cardinality).
+     See also [skills/victoriametrics-cardinality-analysis](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/victoriametrics-cardinality-analysis/SKILL.md) for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
 
    - Insert performance can degrade when the same time series arrives with labels in a different order.
      Ensure your ingestion client always sends labels in a consistent order for each series.
@@ -304,7 +305,8 @@ to logs.
 These are the solutions that exist for improving the performance of slow queries:
 
 - Investigating the bottleneck in query execution using [query tracing](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing).
-  It will show the percentage of time spent on each execution step and help understand the volume of processed data.
+  It will show the percentage of time spent on each execution step and help understand the volume of processed data. See also [skills/vm-trace-analyzer](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/vm-trace-analyzer/SKILL.md)
+  for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
 
 - Adding more CPU and memory to VictoriaMetrics, so it may perform the slow query faster.
   If you use the cluster version of VictoriaMetrics, then migrating `vmselect` nodes to machines

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,23 +26,50 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply limit to metrics metadata response. See [#11139](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11139).
+
+## [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0)
+
+Released at 2026-07-06
+
+* SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
+
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `default_vm_access_claim` field into `jwt` section of auth config. It could be used at [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), if `JWT` token doesn't have `vm_access` claim. See [#11054](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11054).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduces CPU usage by 10% at [sharding among remote storages](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages). See [#11113](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11113). Thanks to @bennf for contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce `64KiB` size limit for `metric metadata` fields - `Unit`, `Help` and `MetricFamilyName`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328), [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926), and the [documentation](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow log requests with missing or invalid auth tokens to [access log](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). This is useful for identifying `remote_addr` IPs performing brute-force attacks. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fall through to `unauthorized_user` when a [JWT token](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) has no `vm_access` claim and no `default_vm_access_claim` is configured. Previously, vmauth returned `401 Unauthorized` immediately in this case, which prevented `unauthorized_user` from handling such requests. See [#5740](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5740).
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): improve the selection algorithm of [buckets_limit](https://docs.victoriametrics.com/victoriametrics/metricsql/#buckets_limit) to remove consecutive empty buckets at the beginning and end to obtain more accurate min and max values. See [#10417](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_group_rule_results_limit` metric to indicate the number of alerts or recording results that a single rule within the group can produce. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
+
+* BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fixes unexpected rare rerouting. See [#11162](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11162).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly URL-encode `-vm-extra-label` values when building import requests, so special characters such as `&` don't get split into broken query parameters. See [#11144](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11144). Thanks to @immanuwell for contribution.
+* BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmagent](https://docs.victoriametrics.com/vmagent/): ignore `enable.auto.offset.store` option in `kafka.consumer.topic.options`, since `vmagent` manages offset storage internally. Previously, setting this option could cause `vmagent` to stop committing Kafka messages. See [#11208](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11208).
 
 ## [v1.146.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.146.0)
 
 Released at 2026-06-22
 
 * FEATURE: all VictoriaMetrics components: add `-http.header.disableServerHostname` command-line flag for disabling the `X-Server-Hostname` HTTP response header. See [#11067](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11067). Thanks to @zasdaym for contribution.
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `-vm-headers` and `-vm-bearer-token` flags for authenticating requests to the VictoriaMetrics import destination. The flags are available in `opentsdb`, `influx`, `remote-read`, `prometheus`, `mimir`, and `thanos` vmctl sub-commands. See [#8897](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8897).
-* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the `last` value to graph legend statistics. See [#10759](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10759).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): expose `vm_streamaggr_dedup_dropped_samples_total` to allow tracking dropped old samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): use the aggregation rule interval as the default [staleness_interval](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#staleness) instead of `2*interval`, to reduce spikes when there are gaps between received samples. See [#11102](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11102).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add new aggregation output [sum_samples_total](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#sum_samples_total) for summing input delta values into a cumulative counter. See issues [#11002](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11002) and [#4843](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4843).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add a new flag `-remoteWrite.inmemoryQueues` to prioritize recently ingested data over historical data stored at file-based [persistent queue](https://docs.victoriametrics.com/victoriametrics/vmagent/#on-disk-persistence-and-data-processing-order). See [#8833](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8833)
-*  FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-promscrape.cluster.shardByLabels` command-line flag for selecting target labels used for sharding scrape targets among `vmagent` instances in cluster mode. See [#11044](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11044).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-promscrape.cluster.shardByLabels` command-line flag for selecting target labels used for sharding scrape targets among `vmagent` instances in cluster mode. See [#11044](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11044).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `-vm-headers` and `-vm-bearer-token` flags for authenticating requests to the VictoriaMetrics import destination. The flags are available in `opentsdb`, `influx`, `remote-read`, `prometheus`, `mimir`, and `thanos` vmctl sub-commands. See [#8897](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8897).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): log calls to [/api/v1/admin/tsdb/delete_series](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series) API handler. This should help to identify events of metrics deletion from the database. See [#11104](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11104).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the `last` value to graph legend statistics. See [#10759](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10759).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
 
 * BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly expose metric `vm_retention_filters_partitions_scheduled_rows`. See [#11138](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11138)
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808).
@@ -57,6 +84,7 @@ Released at 2026-06-22
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
 
+
 ## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
 
 Released at 2026-06-08
@@ -87,7 +115,6 @@ Released at 2026-05-22
 
 * FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
-* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `clusternative.tls` `vminsert` configuration flags for [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). See [#10958](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10958).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
@@ -219,7 +246,7 @@ Released at 2026-03-27
 **Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
 
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show `seriesCountByMetricName` table when a label is in focus in the [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer). See [#10630](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10630). Thanks to @Roshan1299 for the contribution.
-* FEATURE: [dashboards/unused-metrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/unused-metrics.json): add a new dashboard for exploring stored metrics based on [Caridnality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) and [ingested metrics usage API](https://docs.victoriametrics.com/victoriametrics/#track-ingested-metrics-usage). The dashboard requires [Infinity Grafana plugin](https://grafana.com/grafana/plugins/yesoreyeram-infinity-datasource/) to be installed. See [#10617](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10617) for details.
+* FEATURE: [dashboards/metrics-explorer](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/metrics-explorer.json): add a new dashboard for exploring stored metrics based on [Caridnality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) and [ingested metrics usage API](https://docs.victoriametrics.com/victoriametrics/#track-ingested-metrics-usage). The dashboard requires [Infinity Grafana plugin](https://grafana.com/grafana/plugins/yesoreyeram-infinity-datasource/) to be installed. See [#10617](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10617) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `search` parameter and pagination support in `/api/v1/rules` API. See [#10046](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10046).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add default pagination to improve the Alerting Rules page experience when vmalert loads thousands of rules. See [#10046](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10046).
 * FEATURE: all VictoriaMetrics components: log a warning when an IPv6 listen address (e.g. `[::]:6969`) is specified but `-enableTCP6` is not set. Previously, the server silently listened on IPv4 only. See [#6858](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6858). Thanks to @andriibeee for the contribution.
@@ -233,7 +260,7 @@ Released at 2026-03-27
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix autocomplete dropdown not closing on the Raw Query page. See [#10665](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10665)
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly handle JWKS keys per [RFC 7517](https://datatracker.ietf.org/doc/html/rfc7517#section-4.2) during [OIDC discovery](https://docs.victoriametrics.com/victoriametrics/vmauth/#oidc-discovery): skip keys with `use=enc`, reject `use=sig` keys with unsupported `alg`, and warn-skip keys with empty `use` that have unsupported `alg`. See [#10663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10663). Thanks to @andriibeee for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enforce `datasource_type=prometheus` when [proxying](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert) Grafana requests to [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/). Grafana supports only `prometheus` and `loki` alerts. Without this fix, Grafana shows `Error loading alerts` when non-Prometheus alert types are returned. See [victoriametrics-datasource#329](https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/329).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic during directory deletion on `NFS`-based mounts. The bug was introduced in [83da33d8](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/83da33d8cfe8352fd0022d05a8b6346ebb48420d) and included in [v1.123.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11230). See [#9842](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9842).
 
 ## [v1.138.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.138.0)
@@ -295,6 +322,24 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.13](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.13)
+
+Released at 2026-07-03
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
+
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fixes unexpected rare rerouting. See [#11162](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11162).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly URL-encode `-vm-extra-label` values when building import requests, so special characters such as `&` don't get split into broken query parameters. See [#11144](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11144). Thanks to @immanuwell for contribution.
+* BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmagent](https://docs.victoriametrics.com/vmagent/): ignore `enable.auto.offset.store` option in `kafka.consumer.topic.options`, since `vmagent` manages offset storage internally. Previously, setting this option could cause `vmagent` to stop committing Kafka messages. See [#11208](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11208).
+* BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
+
 ## [v1.136.12](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.12)
 
 Released at 2026-06-19
@@ -467,7 +512,7 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): retry RPC by dialing a new connection instead of reusing a pooled one when the previous attempt fails with `io.EOF`, `broken pipe` or `reset by peer`. This reduces query failures caused by stale connections to restarted vmstorage nodes. See [#10314](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10314)
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix autocomplete dropdown not closing on the Raw Query page. See [#10665](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10665)
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enforce `datasource_type=prometheus` when [proxying](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert) Grafana requests to [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/). Grafana supports only `prometheus` and `loki` alerts. Without this fix, Grafana shows `Error loading alerts` when non-Prometheus alert types are returned. See [victoriametrics-datasource#329](https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/329).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic during directory deletion on `NFS`-based mounts. The bug was introduced in [83da33d8](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/83da33d8cfe8352fd0022d05a8b6346ebb48420d) and included in [v1.123.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11230). See [#9842](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9842).
 
 ## [v1.136.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.2)
@@ -675,6 +720,22 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.26](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.26)
+
+Released at 2026-07-03
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
+
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly URL-encode `-vm-extra-label` values when building import requests, so special characters such as `&` don't get split into broken query parameters. See [#11144](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11144). Thanks to @immanuwell for contribution.
+* BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
+
 ## [v1.122.25](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.25)
 
 Released at 2026-06-19
@@ -835,7 +896,7 @@ Released at 2026-02-13
 All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
 The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
 
-* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.23.3-released.html).
+* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.9-3.21.6-3.22.3-3.23.3-released.html).
 * SECURITY: upgrade Go builder from Go1.24.12 to Go1.24.13. See [the list of issues addressed in Go1.24.13](https://github.com/golang/go/issues?q=milestone%3AGo1.24.13+label%3ACherryPickApproved).
 
 * BUGFIX: all VictoriaMetrics components: respect default http client proxy env variables (`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`). See [#10385](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10385). Thanks to @zane-deg for the contribution.
@@ -997,7 +1058,7 @@ Released at 2026-02-13
 All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
 The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11100) release**
 
-* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.23.3-released.html).
+* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.9-3.21.6-3.22.3-3.23.3-released.html).
 * SECURITY: upgrade Go builder from Go1.24.12 to Go1.24.13. See [the list of issues addressed in Go1.24.13](https://github.com/golang/go/issues?q=milestone%3AGo1.24.13+label%3ACherryPickApproved).
 
 * BUGFIX: VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce time‑based manual offset commit for [kafka consumer](https://docs.victoriametrics.com/victoriametrics/integrations/kafka/) to fix performance degradation with enabled manual commit. After this change, the consumer will commit partition offsets in batch per second to avoid high commit QPS on the Kafka broker. It's no longer recommended to set `enable.auto.commit=true` in `-kafka.consumer.topic.options`, as `vmagent` will automatically manage it. See [#10395](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10395).

docs/victoriametrics/changelog/CHANGELOG_2020.md

@@ -37,7 +37,7 @@ Released at 2020-12-19
 
 * BUGFIX: vmalert: properly populate template variables. This has been broken in v1.50.0. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/974>
 * BUGFIX: properly parse negative combined duration in MetricsQL such as `-1h3m4s`. It must be parsed as `-(1h + 3m + 4s)`. Previously it was parsed as `-1h + 3m + 4s`.
-* BUGFIX: properly parse lines in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md) and in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md) with whitespace after the timestamp. For example, `foo 123 456 ## some comment here`. See <https://github.com/VictoriaMetrics/VictoriaMetrics/pull/970>
+* BUGFIX: properly parse lines in [Prometheus exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format) and in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) with whitespace after the timestamp. For example, `foo 123 456 ## some comment here`. See <https://github.com/VictoriaMetrics/VictoriaMetrics/pull/970>
 
 ## [v1.50.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.50.1)
 
@@ -86,7 +86,7 @@ Released at 2020-12-05
 * FEATURE: upgrade Go builder from v1.15.5 to v1.15.6 . This fixes [issues found in Go since v1.15.5](https://github.com/golang/go/issues?q=milestone%3AGo1.15.6+label%3ACherryPickApproved).
 
 * BUGFIX: properly parse timestamps in OpenMetrics format - they are exposed as floating-point number in seconds instead of integer milliseconds
-  unlike in Prometheus exposition format. See [the docs](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md#timestamps).
+  unlike in Prometheus exposition format. See [the docs](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#timestamps).
 * BUGFIX: return `nan` for `a >bool b` query when `a` equals to `nan` like Prometheus does. Previously `0` was returned in this case. This applies to any comparison operation
   with `bool` modifier. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) for details.
 * BUGFIX: properly parse hex numbers in MetricsQL. Previously hex numbers with non-decimal digits such as `0x3b` couldn't be parsed.
@@ -119,8 +119,8 @@ Released at 2020-11-26
 * FEATURE: log metric name plus all its labels when the metric timestamp is out of the configured retention. This should simplify detecting the source of metrics with unexpected timestamps.
 * FEATURE: add `-dryRun` command-line flag to single-node VictoriaMetrics in order to check config file pointed by `-promscrape.config`.
 
-* BUGFIX: properly parse Prometheus metrics with [exemplars](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#exemplars-1) such as `foo 123 ## {bar="baz"} 1`.
-* BUGFIX: properly parse "infinity" values in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#abnf).
+* BUGFIX: properly parse Prometheus metrics with [exemplars](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) such as `foo 123 ## {bar="baz"} 1`.
+* BUGFIX: properly parse "infinity" values in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#abnf).
   See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/924>
 
 ## [v1.47.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.47.0)

docs/victoriametrics/changelog/CHANGELOG_2021.md

@@ -275,7 +275,7 @@ Released at 2021-07-15
 * FEATURE: support durations anywhere in [MetricsQL queries](https://docs.victoriametrics.com/victoriametrics/metricsql/). For example, `sum_over_time(m[1h]) / 1h` is a valid query, which is equivalent to `sum_over_time(m[1h]) / 3600`.
 * FEATURE: support durations without suffixes in [MetricsQL queries](https://docs.victoriametrics.com/victoriametrics/metricsql/). For example, `rate(m[3600])` is a valid query, which is equivalent to `rate(m[1h])`.
 * FEATURE: export `vmselect_request_duration_seconds` and `vminsert_request_duration_seconds` [VictoriaMetrics histograms](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) at `/metrics` page. These histograms can be used for determining latency distribution and SLI/SLO for the served requests. For example, the following query would return the percent of queries that took less than 500ms during the last hour: `histogram_share(500ms, sum(rate(vmselect_request_duration_seconds_bucket[1h])) by (vmrange))`.
-* FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
+* FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/access-controls/service-credentials/mutual-tls-authentication/). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
 * FEATURE: log http request path plus all the query args on errors during request processing. Previously only http request path was logged without query args, so it could be hard debugging such errors.
 * FEATURE: add `is_set` label to `flag` metrics. This allows determining explicitly set command-line flags with the query `flag{is_set="true"}`.
 * FEATURE: add ability to remove caches stored inside `<-storageDataPath>/cache` on startup if `reset_cache_on_startup` file is present there. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447).
@@ -590,7 +590,7 @@ Released at 2021-01-13
 * FEATURE: add ability to pass multiple labels to `sort_by_label()` and `sort_by_label_desc()` functions. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/992> .
 * FEATURE: enforce at least TLS v1.2 when accepting HTTPS requests if `-tls`, `-tlsCertFile` and `-tlsKeyFile` command-line flags are set, because older TLS protocols such as v1.0 and v1.1 have been deprecated due to security vulnerabilities.
 * FEATURE: support `extra_label` query arg for all HTTP-based [data ingestion protocols](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data). This query arg can be used for specifying extra labels which should be added for the ingested data.
-* FEATURE: vmbackup: increase backup chunk size from 128MB to 1GB. This should reduce the number of Object storage API calls during backups by 8x. This may also reduce costs, since object storage API calls usually have non-zero costs. See <https://aws.amazon.com/s3/pricing/> and <https://cloud.google.com/storage/pricing#operations-pricing> .
+* FEATURE: vmbackup: increase backup chunk size from 128MB to 1GB. This should reduce the number of Object storage API calls during backups by 8x. This may also reduce costs, since object storage API calls usually have non-zero costs. See <https://aws.amazon.com/s3/pricing/> and <https://cloud.google.com/storage/pricing#operation-charges> .
 
 * BUGFIX: properly parse escaped unicode chars in MetricsQL metric names, label names and function names. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/990>
 * BUGFIX: override user-provided labels with labels set in `extra_label` query args during data ingestion over HTTP-based protocols.

docs/victoriametrics/changelog/CHANGELOG_2022.md

@@ -978,7 +978,7 @@ Released at 2022-02-14
 
 Released at 2022-01-18
 
-* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add support for `@` modifier, which is enabled by default in Prometheus starting from [Prometheus v2.33.0](https://github.com/prometheus/prometheus/pull/10121). See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1348). VictoriaMetrics extends `@` modifier with the following additional features:
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add support for `@` modifier, which is enabled by default in Prometheus starting from [Prometheus v2.33.0](https://github.com/prometheus/prometheus/pull/10121). See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1348). VictoriaMetrics extends `@` modifier with the following additional features:
   * It can contain arbitrary expression. For example, `foo @ (end() - 1h)` would return `foo` value at `end - 1 hour` timestamp on the selected time range `[start ... end]`. Another example: `foo @ (now() - 10m)` would return `foo` value 10 minutes ago from the current time.
   * It can be put everywhere in the query. For example, `sum(foo) @ start()` would calculate `sum(foo)` at `start` timestamp on the selected time range `[start ... end]`.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add support for optional `keep_metric_names` modifier, which can be applied to all the [rollup functions](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions) and [transform functions](https://docs.victoriametrics.com/victoriametrics/metricsql/#transform-functions). This modifier prevents from deleting metric names from function results. For example, `rate({__name__=~"foo|bar"}[5m]) keep_metric_names` leaves `foo` and `bar` metric names in `rate()` results. This feature provides an additional workaround for [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/949).

docs/victoriametrics/changelog/CHANGELOG_2023.md

@@ -370,7 +370,7 @@ The v1.93.x line will be supported for at least 12 months since [v1.93.0](https:
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dashboard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit.
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panels for absolute Mem and CPU usage by vmalert. See related issue [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627).
 * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected.
-* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) as it could be related to many VictoriaMetrics components. 
+* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-single-node.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) as it could be related to many VictoriaMetrics components.
 
 * BUGFIX: [storage](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly set next retention time for indexDB. Previously it may enter into endless retention loop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return human readable error if opentelemetry has json encoding. Follow-up after [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2570).
@@ -558,7 +558,7 @@ Released at 2023-05-18
 * BUGFIX: change the max allowed value for `-memory.allowedPercent` from 100 to 200. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4171).
 * BUGFIX: properly limit the number of [OpenTSDB HTTP](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http) concurrent requests specified via `-maxConcurrentInserts` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4204). Thanks to @zouxiang1993 for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4208).
 * BUGFIX: do not ignore trailing empty field in CSV lines when [importing data in CSV format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048).
-* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-format-example). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
+* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#line-format). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
 * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent from possible panic when the number of vmstorage nodes increases when [automatic vmstorage discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery) is enabled.
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): fix a panic when the duration in the query contains uppercase `M` suffix. Such a suffix isn't allowed to use in durations, since it clashes with `a million` suffix, e.g. it isn't clear whether `rate(metric[5M])` means rate over 5 minutes, 5 months or 5 million seconds. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3589) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4120) issues.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): properly handle the `vm_promscrape_config_last_reload_successful` metric after config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4260).
@@ -825,7 +825,7 @@ The v1.87.x line will be supported for at least 12 months since [v1.87.0](https:
 * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.
 * BUGFIX: reduce the possibility of increased CPU usage when data with timestamps older than one hour is ingested into VictoriaMetrics. This reduces spikes for the graph `sum(rate(vm_slow_per_day_index_inserts_total))`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4258).
 * BUGFIX: do not ignore trailing empty field in CSV lines when [importing data in CSV format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048).
-* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-format-example). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
+* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#line-format). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): fix a panic when the duration in the query contains uppercase `M` suffix. Such a suffix isn't allowed to use in durations, since it clashes with `a million` suffix, e.g. it isn't clear whether `rate(metric[5M])` means rate over 5 minutes, 5 months or 5 million seconds. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3589) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4120) issues.
 * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent from possible panic when the number of vmstorage nodes increases when [automatic vmstorage discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery) is enabled.
 * BUGFIX: properly limit the number of [OpenTSDB HTTP](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http) concurrent requests specified via `-maxConcurrentInserts` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4204). Thanks to @zouxiang1993 for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4208).

docs/victoriametrics/changelog/CHANGELOG_2024.md

@@ -443,7 +443,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
   * `position` - the position of the aggregation rule in the corresponding streaming aggregation config file
 * FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent having duplicated aggregation function as `outputs` in one [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). It also prevents using `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` instead of  `outputs: ["quantiles(0.5, 0.9)"]`, as the former has higher computation cost for producing the same result.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-graphite.sanitizeMetricName` command-line flag for sanitizing metrics ingested via [Graphite protocol](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6077).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): [`yandexcloud_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs): add support for obtaining IAM token in [GCE format](https://yandex.cloud/en-ru/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm) additionally to the [deprecated Amazon EC2 IMDSv1 format](https://yandex.cloud/en/docs/security/standard/authentication#aws-token). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5513).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): [`yandexcloud_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs): add support for obtaining IAM token in [GCE format](https://yandex.cloud/en/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm) additionally to the [deprecated Amazon EC2 IMDSv1 format](https://yandex.cloud/en/docs/security/standard/authentication#aws-token). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5513).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): make `-replay.timeTo` optional in [replay mode](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling). When omitted, the current timestamp will be used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6492).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): reduce CPU usage when proxying data ingestion requests.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow disabling request body caching with `-maxRequestBodySizeToRetry=0`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6445). Thanks to @shichanglin5 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6533).
@@ -674,7 +674,7 @@ Released at 2024-03-01
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve [`WITH` templates](https://play.victoriametrics.com/select/0/prometheus/graph/#/expand-with-exprs) when clicking the `prettify query` button at the right side of query input field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5383).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): allow filling gaps on graphs with interpolated lines as Grafana does. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5152) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5862).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert): support filtering by group, rule or labels in [vmalert's UI](https://docs.victoriametrics.com/victoriametrics/vmalert/#web) for `/groups` and `/alerts` pages. See [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5791) by @victoramsantos.
-* FEATURE: [docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#docker-compose-environment-for-victoriametrics): create a separate [docker-compose environment](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#victoriaLogs-server) for VictoriaLogs installation, including fluentbit and [VictoriaLogs Grafana datasource](https://github.com/VictoriaMetrics/victorialogs-datasource). See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#victoriaLogs-server) for details.
+* FEATURE: [docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#docker-compose-environment-for-victoriametrics): create a separate [docker-compose environment](https://github.com/VictoriaMetrics/VictoriaLogs/tree/master/deployment/docker#victorialogs-server) for VictoriaLogs installation, including fluentbit and [VictoriaLogs Grafana datasource](https://github.com/VictoriaMetrics/victorialogs-datasource). See [these docs](https://github.com/VictoriaMetrics/VictoriaLogs/tree/master/deployment/docker#victorialogs-server) for details.
 * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): wait for up 30 seconds before making a [snapshot](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots) for backup if `vmstorage` is temporarily unavailable. This should prevent from `vmbackupmanager` termination in this case. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5859).
 
 * BUGFIX: downgrade Go builder from `1.22.0` to `1.21.7`, since `1.22.0` contains [the bug](https://github.com/golang/go/issues/65705), which can lead to deadlocked HTTP connections to remote storage systems, scrape targets and service discovery endpoints at [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/). This may result in incorrect service discovery, target scraping and failed sending samples to remote storage.

docs/victoriametrics/changelog/CHANGELOG_2025.md

@@ -1251,7 +1251,7 @@ Released at 2025-01-14
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victorialogs/vmalert/): do not append tenant info to VictoriaLogs datasource request path in [clusterMode](https://docs.victoriametrics.com/victoriametrics/vmalert/#multitenancy). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/#how-to-use-multitenancy-in-rules) for how to use multitenancy in VictoriaLogs.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly set `host` field at debug information formatted with `dump_request_on_errors: true` setting.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly handle discovery for ipv6 addresses. Thanks to @badie for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7955).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7921). Thanks to @bitbidu for reporting.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7935). Thanks to @bitbidu for reporting.
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): storage nodes defined in `-storageNode` are now sorted, ensuring that varying node orders across different vminsert instances do not result in inconsistent replication.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly ingest `influx` line protocol metrics with empty tags. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7933) for details.
 * BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): allow to override the default unique time series limit in vmstorage with command-line flags like `-search.maxUniqueTimeseries`, `-search.maxLabelsAPISeries`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7852).
@@ -1515,7 +1515,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): make instant query results consistent. VictoriaMetrics detects and adjusts scrape interval and while this is very useful for range queries (i.e. this eliminates gaps on the graph), it may cause instant queries to return a non-empty result when no result is expected. The fix is to disable scrape interval detection and always use the step as the scrape interval in instant queries. This will guarantee that the samples are searched within the (time-step, time] interval. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5796) for details.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix cursor reset in query input field. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7288).
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly handle discovery for ipv6 addresses. Thanks to @badie for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7955).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7921). Thanks to @bitbidu for reporting.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7935). Thanks to @bitbidu for reporting.
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): storage nodes defined in `-storageNode` are now sorted, ensuring that varying node orders across different vminsert instances do not result in inconsistent replication.
 
 ## [v1.102.9](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.9)
@@ -1622,7 +1622,7 @@ The v1.97.x line will be supported for at least 12 months since [v1.97.0](https:
 * FEATURE: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) components: add support of hot-reload for license key supplied by `-licenseFile` command-line flag.
 
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): make instant query results consistent. VictoriaMetrics detects and adjusts scrape interval and while this is very useful for range queries (i.e. this eliminates gaps on the graph), it may cause instant queries to return a non-empty result when no result is expected. The fix is to disable scrape interval detection and always use the step as the scrape interval in instant queries. This will guarantee that the samples are searched within the (time-step, time] interval. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5796) for details.
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7921). Thanks to @bitbidu for reporting.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7935). Thanks to @bitbidu for reporting.
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): storage nodes defined in `-storageNode` are now sorted, ensuring that varying node orders across different vminsert instances do not result in inconsistent replication.
 
 ## [v1.97.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.14)

docs/victoriametrics/enterprise.md

@@ -121,7 +121,7 @@ It is allowed to run Enterprise components in [cases listed here](https://docs.v
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
 [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
 and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -139,8 +139,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.146.0/victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -155,7 +155,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.146.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.147.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
@@ -165,13 +165,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.146.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.147.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.146.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.147.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -181,7 +181,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.147.0
     ports:
       - 8428:8428
     volumes:
@@ -213,7 +213,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.147.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -224,7 +224,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.147.0-enterprise
 
 license:
   secret:
@@ -274,7 +274,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.147.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -291,7 +291,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.146.0-enterprise
+    tag: v1.147.0-enterprise
 ```
 
 Example secret with license key:
@@ -342,7 +342,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.146.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz`
 
 Includes:
 
@@ -351,7 +351,7 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.146.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.147.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## What Happens to Licensed Components When a License Expires
 

docs/victoriametrics/integrations/OpenShift.md

@@ -11,7 +11,7 @@ menu:
 
 OpenShift uses Prometheus as a core monitoring solution. It cannot be replaced without violating the OpenShift support policy. However, OpenShift can be configured to use VictoriaMetrics as a remote write target.
 
-According to [remote write configuration in the OpenShift documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-core-platform-monitoring#configuring-remote-write-storage_configuring-metrics), the following manifest needs to be applied to send platform metrics to VictoriaMetrics:
+According to [remote write configuration in the OpenShift documentation](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_core_platform_monitoring/configuring-metrics#configuring-remote-write-storage_configuring-metrics), the following manifest needs to be applied to send platform metrics to VictoriaMetrics:
 
 ```yaml
 apiVersion: v1
@@ -91,7 +91,7 @@ data:
             key: token
 ```
 
-Along with core platform monitoring, OpenShift also supports collecting user workload metrics. See [this guide](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-user-workload-monitoring) for more information. In order to send user workload metrics to VictoriaMetrics, the following manifest needs to be applied:
+Along with core platform monitoring, OpenShift also supports collecting user workload metrics. See [this guide](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_user_workload_monitoring/index) for more information. In order to send user workload metrics to VictoriaMetrics, the following manifest needs to be applied:
 ```yaml
 apiVersion: v1
 kind: ConfigMap
@@ -166,6 +166,6 @@ data:
 
 ## References
 
-- [OpenShift Documentation: Core Platform Monitoring](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-core-platform-monitoring)
-- [OpenShift Documentation: User Workload Monitoring](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-user-workload-monitoring)
+- [OpenShift Documentation: Core Platform Monitoring](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_core_platform_monitoring/index)
+- [OpenShift Documentation: User Workload Monitoring](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_user_workload_monitoring/index)
 - [OpenShift Documentation: Hosted Control Planes Overview](https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/hosted_control_planes/hosted-control-planes-overview)

docs/victoriametrics/integrations/grafana/_index.md

@@ -46,7 +46,7 @@ If you don't see an option to create a data source - try contacting system admin
 
 ## Prometheus datasource
 
-Create [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure-prometheus-data-source/)
+Create [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure/)
 in Grafana. Follow the same connection instructions as for [VictoriaMetrics datasource](#VictoriaMetrics-datasource).
 
 In the "Type and version" section set the type to "Prometheus" and the version to at least "2.24.x".

docs/victoriametrics/integrations/influxdb.md

@@ -81,7 +81,7 @@ The `/api/v1/export` endpoint should return the following response:
 ## Data transformations
 
 VictoriaMetrics performs the following transformations to the ingested InfluxDB data:
-* [db query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db`
+* [db query arg](https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite) is mapped into `db`
   [label](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels) value unless `db` tag exists in the InfluxDB line.
   The `db` label name can be overridden via `-influxDBLabel` command-line flag. If more strict data isolation is required,
   read more about multi-tenancy [here](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#multi-tenancy).

docs/victoriametrics/integrations/kafka.md

@@ -19,8 +19,8 @@ Use `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` comm
 * `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
   Messages in this format can be sent by vmagent - see [these docs](#writing-metrics).
 * `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/).
-* `prometheus` - [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
-  and [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md).
+* `prometheus` - [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format)
+  and [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md).
 * `graphite` - [Graphite plaintext format](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol).
 * `jsonline` - [JSON line format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-json-line-format).
 * `opentelemetry`{{% available_from "v1.128.0" %}}  - [Opentelemetry format](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/)

docs/victoriametrics/integrations/pubsub.md

@@ -18,8 +18,8 @@ Use `-gcp.pubsub.subscribe.defaultMessageFormat` and `-gcp.pubsub.subscribe.topi
 * `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
   Messages in this format can be sent by vmagent - see [these docs](#writing-metrics).
 * `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/).
-* `prometheus` - [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
-  and [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md).
+* `prometheus` - [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format)
+  and [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md).
 * `graphite` - [Graphite plaintext format](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol).
 * `jsonline` - [JSON line format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-json-line-format).
 

docs/victoriametrics/keyConcepts/keyConcepts.md

@@ -84,7 +84,7 @@ If the value has more than 12 significant decimal digits, then the less signific
 The `timestamp` is a [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) with millisecond precision.
 
 Below is an example of a single raw sample
-in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format):
+in [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format):
 
 ```
 requests_total{path="/", code="200"} 123 4567890
@@ -456,7 +456,7 @@ The basic monitoring setup of VictoriaMetrics and vmagent is described
 in the [example docker-compose manifest](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#readme).
 In this example vmagent [scrapes a list of targets](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-single.yml)
 and [forwards collected data to VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/9751ea10983d42068487624849cac7ad6fd7e1d8/deployment/docker/compose-vm-single.yml#L16).
-VictoriaMetrics is then used as a [datasource for Grafana](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/provisioning/datasources/prometheus-datasource/single.yml)
+VictoriaMetrics is then used as a [datasource for Grafana](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/provisioning/datasources/prometheus/single.yml)
 installation for querying collected data.
 
 VictoriaMetrics components allow building more advanced topologies. For example, vmagents can push metrics from separate datacenters to the central VictoriaMetrics:
@@ -492,7 +492,7 @@ Params:
 * `time` - optional, [timestamp](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats)
   in millisecond precision to evaluate the `query` at. If omitted, `time` is set to `now()` (current timestamp).
   The `time` param can be specified in [multiple allowed formats](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats).
-* `step` - optional [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations)
+* `step` - optional [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations)
   for searching for raw samples in the past when executing the `query` (used when a sample is missing at the specified `time`).
   For example, the request `/api/v1/query?query=up&step=1m` looks for the last written raw sample for the metric `up`
   in the `(now()-1m, now()]` interval (the first millisecond is not included). If omitted, `step` is set to `5m` (5 minutes)
@@ -590,7 +590,7 @@ Params:
 * `end` - the ending [timestamp](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats)
   of the time range for `query` evaluation.
   If the `end` isn't set, then the `end` is automatically set to the current time.
-* `step` - the [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations) 
+* `step` - the [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations)
   between data points, which must be returned from the range query.
   The `query` is executed at `start`, `start+step`, `start+2*step`, ..., `start+N*step` timestamps,
   where `N` is the whole number of steps that fit between `start` and `end`.
@@ -929,7 +929,7 @@ rate(node_network_receive_bytes_total)
 By default, VictoriaMetrics calculates the `rate` over [raw samples](#raw-samples) on the lookbehind window specified in the `step` param
 passed either to [instant query](#instant-query) or to [range query](#range-query).
 The interval on which `rate` needs to be calculated can be specified explicitly
-as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations) in square brackets:
+as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations) in square brackets:
 
 ```metricsql
  rate(node_network_receive_bytes_total[5m])

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.146.0/victoria-metrics-linux-amd64-v1.146.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.146.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/victoria-metrics-linux-amd64-v1.147.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.147.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.146.0/victoria-metrics-linux-amd64-v1.146.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.146.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/victoria-metrics-linux-amd64-v1.147.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.147.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml
@@ -342,7 +342,7 @@ See [these docs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#ht
 The following [`-promscrape.config`](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter)
 instructs discovering and scraping metrics for all the containers with the name `my-super-app`.
 It is expected that these containers expose only a single TCP port, which serves its metrics at `/metrics` page
-according to [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format):
+according to [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format):
 
 ```yaml
 scrape_configs:

docs/victoriametrics/sd_configs.md

@@ -32,7 +32,7 @@ supports the following Prometheus-compatible service discovery options for Prome
 * `http_sd_configs` is for discovering and scraping targets provided by external http-based service discovery. See [these docs](#http_sd_configs).
 * `kubernetes_sd_configs` is for discovering and scraping [Kubernetes](https://kubernetes.io/) targets. See [these docs](#kubernetes_sd_configs).
 * `kuma_sd_configs` is for discovering and scraping [Kuma](https://kuma.io) targets. See [these docs](#kuma_sd_configs).
-* `marathon_sd_configs` is for discovering and scraping [Marathon](https://mesosphere.github.io/marathon/) targets. See [these docs](#marathon_sd_configs).
+* `marathon_sd_configs` is for discovering and scraping [Marathon](https://github.com/d2iq-archive/marathon) targets. See [these docs](#marathon_sd_configs).
 * `nomad_sd_configs` is for discovering and scraping targets registered in [HashiCorp Nomad](https://www.nomadproject.io/). See [these docs](#nomad_sd_configs).
 * `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [these docs](#openstack_sd_configs).
 * `ovhcloud_sd_configs` is for discovering and scraping OVH Cloud VPS and dedicated server targets. See [these docs](#ovhcloud_sd_configs).
@@ -324,7 +324,7 @@ The list of discovered Consul Agent targets is refreshed at the interval, which
 
 ## digitalocean_sd_configs
 
-DigitalOcean SD configuration allows retrieving scrape targets from [DigitalOcean's Droplets API](https://docs.digitalocean.com/reference/api/api-reference/#tag/Droplets).
+DigitalOcean SD configuration allows retrieving scrape targets from [DigitalOcean's Droplets API](https://docs.digitalocean.com/reference/api/reference/droplets/).
 
 Configuration example:
 
@@ -1315,7 +1315,7 @@ The list of discovered Kuma targets is refreshed at the interval, which can be c
 
 ## marathon_sd_configs
 
-Marathon SD configuration {{% available_from "v1.109.0" %}} allows retrieving scrape targets from [Marathon](https://mesosphere.github.io/marathon/) REST API.
+Marathon SD configuration {{% available_from "v1.109.0" %}} allows retrieving scrape targets from [Marathon](https://github.com/d2iq-archive/marathon) REST API.
 
 Configuration example:
 
@@ -1350,7 +1350,7 @@ The list of discovered Marathon targets is refreshed at the interval, which can
 
 ## nomad_sd_configs
 
-Nomad SD configuration allows retrieving scrape targets from [HashiCorp Nomad Services](https://www.hashicorp.com/blog/nomad-service-discovery).
+Nomad SD configuration allows retrieving scrape targets from [HashiCorp Nomad Services](https://www.hashicorp.com/en/blog/nomad-service-discovery).
 
 Configuration example:
 

docs/victoriametrics/stream-aggregation/README.md

@@ -42,6 +42,8 @@ Stream aggregation can be used in the following cases:
 * [Reducing the number of stored samples](#reducing-the-number-of-stored-samples)
 * [Reducing the number of stored series](#reducing-the-number-of-stored-series)
 
+See [skills/stream-aggregation-helper](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/stream-aggregation-helper/SKILL.md) for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) configuration.
+
 ## Statsd alternative
 
 Stream aggregation can be used as [statsd](https://github.com/statsd/statsd) alternative in the following cases:
@@ -624,13 +626,12 @@ command line flags. See how to [shard data across remote write destinations](htt
 The following requirements must be met for sharded aggregation to work correctly:
 - All sharding vmagents should have the same deterministic sharding configuration.
 - The sharding configuration must align with the `by` and `without` lists:
-  - Labels listed in `by` setting should be a subset of shard's routing key `-remoteWrite.shardByURL.labels`. 
-    With `-remoteWrite.shardByURL.labels=env,job` aggregator's `by` should include `by: env`, `by: job` or both: `by: [env, job]`.
-    This makes sure that all the samples for the same `env` and `job` are aggregated together and produce the complete output.
-  - Labels listed in `without` setting should be a superset of shard's routing key `--remoteWrite.shardByURL.ignoreLabels`.
-    With `-remoteWrite.shardByURL.ignoreLabels=env,job` aggegator's `without` should include at least both labels `without: [env,job]`.
-    This makes sure that `requests_total{env=test, job=foo}` and `requests_total{env=prod, job=foo}` are routed to the same aggregator
-    and are aggregated together. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5938#issuecomment-2018470324).
+  - Labels configured in `-remoteWrite.shardByURL.labels` must be a subset of the labels listed in `by`. 
+    For example, if the aggregation config specifies `by: [env, job]`, then `-remoteWrite.shardByURL.labels` may include `env`, `job`, or both. 
+    This ensures that all samples contributing to the same aggregation result are routed to the same aggregator instance and aggregated together to produce a complete output.
+  - Labels configured in `-remoteWrite.shardByURL.ignoreLabels` must be a superset of the labels listed in `without`.
+    For example, if the aggregation config specifies `without: [env, pod]`, then `-remoteWrite.shardByURL.ignoreLabels` must include at least `env` and `pod`. 
+    This ensures that labels removed during aggregation are not used for shard routing.
 - Aggregating vmagents should not produce collisions: the aggregation output should be unique across all the sharded agents.
   For example, `requests_total:5m_without_env_pod_total` produced by both `vmagent-aggr-1` and `vmagent-aggr-2` will collide
   unless they have labels uniquely identifying them. These labels should be either preserved during sharding and aggregation config,

docs/victoriametrics/stream-aggregation/configuration.md

@@ -222,6 +222,7 @@ Below are aggregation functions that can be put in the `outputs` list at [stream
 * [stddev](#stddev)
 * [stdvar](#stdvar)
 * [sum_samples](#sum_samples)
+* [sum_samples_total](#sum_samples_total)
 * [total](#total)
 * [total_prometheus](#total_prometheus)
 * [unique_samples](#unique_samples)
@@ -505,10 +506,11 @@ See also:
 
 - [count_samples](#count_samples)
 - [count_series](#count_series)
+- [sum_samples_total](#sum_samples_total)
 
 ### `sum_samples_total`
 
-`sum_samples_total` sums input delta values into a cumulative [counter](https://docs.victoriametrics.com/victoriametrics/keyconcepts/index.html#counter) and outputs the result at the given `interval`.
+`sum_samples_total` {{% available_from "v1.146.0" %}}. sums input delta values into a cumulative [counter](https://docs.victoriametrics.com/victoriametrics/keyconcepts/index.html#counter) and outputs the result at the given `interval`.
 `sum_samples_total` makes sense only for aggregating delta values from clients such as [StatsD counter](https://github.com/statsd/statsd/blob/master/docs/metric_types.md#counting).
 
 The results of `sum_samples_total` is roughly equal to the following [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) query:
@@ -559,6 +561,7 @@ See also:
 - [total_prometheus](#total_prometheus)
 - [increase](#increase)
 - [increase_prometheus](#increase_prometheus)
+- [sum_samples_total](#sum_samples_total)
 - [rate_sum](#rate_sum)
 - [rate_avg](#rate_avg)
 
@@ -588,6 +591,7 @@ See also:
 - [total](#total)
 - [increase](#increase)
 - [increase_prometheus](#increase_prometheus)
+- [sum_samples_total](#sum_samples_total)
 - [rate_sum](#rate_sum)
 - [rate_avg](#rate_avg)
 

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -13,6 +13,8 @@ victoria-metrics is a time series database and monitoring solution.
 
 See the docs at https://docs.victoriametrics.com/victoriametrics/
 
+  -accountID uint
+     The accountID of the stored data
   -bigMergeConcurrency int
      Deprecated: this flag does nothing
   -blockcache.missesBeforeCaching int
@@ -104,7 +106,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
@@ -194,11 +196,11 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
   -maxLabelNameLen int
-     The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_name"} metric at /metrics page is incremented (default 256)
+     The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_name"} metric at /metrics page is incremented. Value must be in range 1..65535. (default 256)
   -maxLabelValueLen int
-     The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented (default 4096)
+     The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented. Value must be in range 1..65535. (default 4096)
   -maxLabelsPerTimeseries int
-     The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented (default 40)
+     The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented. (default 40)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -261,6 +263,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss (default 64)
   -prevCacheRemovalPercent float
      Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1)
+  -projectID uint
+     The projectID of the stored data
   -promscrape.azureSDCheckInterval duration
      Interval for checking for changes in Azure. This works only if azure_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#azure_sd_configs for details (default 1m0s)
   -promscrape.cluster.memberLabel string
@@ -401,6 +405,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 1M)
   -retentionTimezoneOffset duration
      The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
+  -rpc.disableCompression
+     Whether to disable compression of the data sent from vmstorage to vmselect. This reduces CPU usage at the cost of higher network bandwidth usage
+  -rpc.handshakeTimeout duration
+     Timeout for RPC handshake between vminsert/vmselect and vmstorage. Increase this value if transient handshake failures occur. See https://docs.victoriametrics.com/victoriametrics/troubleshooting/#cluster-instability section for more details. (default 5s)
   -search.cacheTimestampOffset duration
      The maximum duration since the current time for response data, which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses due to time synchronization issues between VictoriaMetrics and data sources. See also -search.disableAutoCacheReset (default 5m0s)
   -search.disableAutoCacheReset
@@ -628,6 +636,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Show VictoriaMetrics version
   -vmalert.proxyURL string
      Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules . See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert
+  -vmselectAddr string
+     TCP address to accept connections from vmselect services
   -vmui.customDashboardsPath string
      Optional path to vmui dashboards. See https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards
   -vmui.defaultTimezone string

docs/victoriametrics/victoria_metrics_enterprise_flags.md

@@ -21,7 +21,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -45,7 +45,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmagent.md

@@ -268,6 +268,39 @@ for the collected samples. Examples:
   ```sh
   ./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -streamAggr.dropInputLabels=replica -streamAggr.dedupInterval=60s
   ```
+  
+### Monitoring Data eXchange
+
+The Monitoring Data eXchange (MDX){{% available_from "v1.147.0" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
+
+To enable MDX, set `-remoteWrite.mdx.enable=true` for the target URL and `-remoteWrite.mdx.enable=false` for other URLs:
+
+```sh
+./vmagent \
+  -remoteWrite.url=http://service-to-keep-all-metrics:8428/api/v1/write \
+  -remoteWrite.mdx.enable=false \
+  -remoteWrite.url=http://service-to-keep-only-vm-metrics:8428/api/v1/write \
+  -remoteWrite.mdx.enable=true 
+```
+When MDX is enabled for a `-remoteWrite.url`, `vmagent` forwards only metrics that:
+- come from the target that exposes the `vm_app_version` metric (emitted by all VictoriaMetrics components)
+- contain the `victoriametrics_app=true` label, which will be added automatically to the metrics if the instance was deployed via [VictoriaMetrics Operator](https://docs.victoriametrics.com/operator/).
+
+`victoriametrics_app=true` label will be added to all metrics that are preserved by MDX if it's absent.
+
+- contain the label specified via `-mdx.label`.
+
+```sh
+./vmagent \
+  -remoteWrite.url=http://service-to-keep-only-vm-metrics:8428/api/v1/write \
+  -remoteWrite.mdx.enable=true \
+  -mdx.label="service=victoriametrics"
+```
+In this configuration, metrics with the label `service=victoriametrics` are preserved even if their scrape targets do not expose `vm_app_version` metric.
+
+The number of VictoriaMetrics metrics preserved by MDX is exposed as `vmagent_remotewrite_mdx_rows_preserved_total`.
+
+The scope of MDX is at the per-url level, so it works after global level mechanisms, such as stream aggregation, relabeling, complexity limiter, and cardinality limiter. See [Life of a sample](https://docs.victoriametrics.com/victoriametrics/vmagent/#life-of-a-sample).
 
 ### Life of a sample
 
@@ -285,18 +318,20 @@ flowchart TB
     F --> G[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#replication-and-high-availability">replicate</a> to each <b>-remoteWrite.url</b><br/>or <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages">shard</a> if <b>-remoteWrite.shardByURL</b> is set]
 
       %% Left branch
-      G --> H1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
-      H1 --> H2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
-      H2 --> H3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
-      H3 --> H4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
-      H4 --> H5[[push to <b>-remoteWrite.url</b>]]
+      G --> H1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange/">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
+      H1 --> H2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
+      H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
+      H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
+      H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
+      H5 --> H6[[push to <b>-remoteWrite.url</b>]]
 
       %% Right branch
-      G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
-      R1 --> R2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
-      R2 --> R3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
-      R3 --> R4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
-      R4 --> R5[[push to <b>-remoteWrite.url</b>]]
+      G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
+      R1 --> R2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
+      R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
+      R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
+      R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
+      R5 --> R6[[push to <b>-remoteWrite.url</b>]]
 ```
 
 Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:
@@ -309,11 +344,11 @@ Scraping has additional settings that can be applied before samples are pushed t
 `vmagent` supports [the same set of push-based data ingestion protocols as VictoriaMetrics does](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data)
 in addition to the pull-based Prometheus-compatible targets' scraping:
 
-* DataDog "submit metrics" API. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/).
+* Datadog "submit metrics" API. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/).
 * InfluxDB line protocol via `http://<vmagent>:8429/write`. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/influxdb/).
 * Graphite plaintext protocol if the `-graphiteListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting).
-* OpenTelemetry HTTP API. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/).
-* NewRelic API. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/newrelic/#sending-data-from-agent).
+* OpenTelemetry HTTP API via `http://<vmagent>:8429/opentelemetry/v1/metrics`. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/).
+* New Relic API. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/newrelic/#sending-data-from-agent).
 * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/).
 * Zabbix Connector streaming protocol. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/zabbixconnector/#send-data-from-zabbix-connector).
 * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
@@ -481,29 +516,38 @@ by specifying `-remoteWrite.forcePromProto` command-line flag for the correspond
 ## Multitenancy
 
 By default, `vmagent` collects the data without [tenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) identifiers
-and routes it to the remote storage specified via `-remoteWrite.url` command-line flag. The `-remoteWrite.url` can point to `/insert/<tenant_id>/prometheus/api/v1/write` path
-at `vminsert` according to [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format). 
+and routes it to the remote storage specified via `-remoteWrite.url` command-line flag. Point `-remoteWrite.url` to vminsert's `/insert/<tenant_id>/prometheus/api/v1/write` path
+according to [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format). 
+
+> Note: the single-node version of VictoriaMetrics doesn't support multitenancy.
+
 ```mermaid
 flowchart LR
-    A["requests_total{instance=foo}"] --> |/api/v1/write| V[vmagent]
-    B["requests_total{instance=bar}"] <--> |scrape| V
-    V --> |"/insert/#60;tenant_id#62;/#60;suffix#62;"| C[vminsert]
+    A["requests_total{instance=foo}"] --> |<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#how-to-push-data-to-vmagent">push</a>| V[vmagent]
+    B["requests_total{instance=bar}"] <--> |<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#how-to-collect-metrics-in-prometheus-format">pull</a>| V
+    V --> |"/insert/#60;tenant_id#62;/prometheus/api/v1/write"| C[vminsert]
 ```
 
-In this case, all the metrics written to `/insert/tenant_id/prometheus/api/v1/write` will belong to the specified `<tenant_id>` tenant.
+In this case, all the metrics written to `/insert/<tenant_id>/prometheus/api/v1/write` will belong to the specified `<tenant_id>` tenant.
 
 ### Multitenancy via labels
 
-vmagent can write data to multiple distinct tenants if `-remoteWrite.url` points to [multitenant URL at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
-and tenant is specified via [multitenancy labels](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels):
+vmagent can write data to **multiple distinct tenants** if `-remoteWrite.url` points to the [multitenant URL in the VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
+and the tenant is specified via [multitenancy labels](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels):
 
 ```mermaid
 flowchart LR
-    A["requests_total{instance=foo, vm_account_id=0}"] --> |/api/v1/write| V[vmagent]
-    B["requests_total{instance=bar, vm_account_id=1}"] <--> |scrape| V
-    V --> |"/insert/multitenant/#60;suffix#62;"| C[vminsert]
+    A["requests_total{instance=foo, vm_account_id=0}"] --> |<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#how-to-push-data-to-vmagent">push</a>| V[vmagent]
+    B["requests_total{instance=bar, vm_account_id=1}"] <--> |<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#how-to-collect-metrics-in-prometheus-format">pull</a>| V
+    V --> |"/insert/multitenant/prometheus/api/v1/write"| C[vminsert]
 ```
-`<tenant_id>` is extracted from the `vm_account_id` and `vm_project_id` labels.
+
+> A single payload pulled from or pushed to vmagent may contain time series belonging to multiple tenants.
+
+When vminsert receives the data on the `/insert/multitenant` path, it extracts `<tenant_id>` from the `vm_account_id` and `vm_project_id` labels for
+each distinct time series.
+
+> If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding accountID and projectID are set to 0.
 
 The `vm_account_id` and `vm_project_id` labels can be specified via [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) before sending the metrics to `-remoteWrite.url`.
 For example, the following relabeling rule instructs sending metrics to `<account_id>:0` tenant defined in the `prometheus.io/account_id` annotation of Kubernetes pod deployment:
@@ -516,11 +560,12 @@ scrape_configs:
     target_label: vm_account_id
 ```
 
-vmagent can get tenant identifier from `__tenant_id__` label at target discovery phase.
-It implicitly converts `__tenant_id__` label into `vm_account_id` and `vm_project_id` labels and attaches
-it to the scraped metrics and metrics metadata.
+vmagent can get the tenant identifier from the `__tenant_id__` label during the target discovery phase.
+It implicitly converts the `__tenant_id__` label into `vm_account_id` and `vm_project_id` labels and attaches
+them to the scraped metrics and metrics metadata.
 
-For example, the following relabeling rule instructs sending metrics to the `10:5` tenant defined in the `prometheus.io/tenant_id: 10:5` annotation of the Kubernetes pod deployment:
+For example, the following relabeling rule instructs sending metrics to the `10:5` tenant defined in the `prometheus.io/tenant_id: 10:5`
+annotation of the Kubernetes pod deployment:
 
 ```yaml
 scrape_configs:
@@ -531,47 +576,54 @@ scrape_configs:
     target_label: __tenant_id__
 ```
 
-vmagent can [enforce adding labels](https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics) to all scraped
-or forwarded metrics. 
+vmagent can also [enforce adding labels](https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics)
+on all scraped or forwarded metrics. 
 
 ### Multitenancy via path
 
-vmagent can write data to multiple distinct tenants if `-remoteWrite.url` points to [multitenant URL at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels),
-tenant is specified in the [write path](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format), and `-enableMultitenantHandlers` command-line flag is set:
+vmagent can write data to multiple distinct tenants if:
+* its `-remoteWrite.url` points to the [VictoriaMetrics cluster multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
+* its `-enableMultitenantHandlers` command-line flag is set
+* clients ingest data into vmagent with the tenant specified in the [write path](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format)
 
 ```mermaid
 flowchart LR
-    A["requests_total{instance=foo}"] --> |/insert/0/#60;suffix#62;| V[vmagent]
-    B["requests_total{instance=bar}"] --> |/insert/1/#60;suffix#62;| V
-    V --> |"/insert/multitenant/#60;suffix#62;"| C[vminsert]
+    A["requests_total{instance=foo}"] --> |<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format">/insert/0/#60;suffix#62;</a>| V[vmagent]
+    B["requests_total{instance=bar}"] --> |<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format">/insert/1/#60;suffix#62;</a>| V
+    V --> |"/insert/multitenant/prometheus/api/v1/write"| C[vminsert]
 ```
 
 In this configuration, vmagent accepts writes via the same multitenant endpoints (`/insert/<accountID>/<suffix>`) [as vminsert does](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format).
-For all received data, vmagent will automatically convert tenant identifiers from the URL to `vm_account_id` and `vm_project_id` labels and set tenant info in metadata.
+For all the received data, vmagent will automatically convert tenant identifiers in the URL path to `vm_account_id` and `vm_project_id` labels, and set tenant information in metadata.
 
-These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via `-remoteWrite.relabelConfig`
-and `-remoteWrite.urlRelabelConfig` command-line flags. 
+These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/)
+specified via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` command-line flags. 
 
 ### Multitenancy via headers
 
-vmagent can write data to multiple distinct tenants if `-remoteWrite.url` points to [multitenant URL at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels),
-tenant is specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}, both `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are set:
+vmagent can write data to multiple distinct tenants if:
+* its `-remoteWrite.url` points to the [VictoriaMetrics cluster multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
+* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are both set
+* clients ingest data into vmagent with the tenants specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}
+
 ```mermaid
 flowchart LR
-    A["requests_total{instance=foo}"] --> |/insert/#60;suffix#62; <br>--header AccountID: 0| V[vmagent]
-    B["requests_total{instance=bar}"] --> |/insert/#60;suffix#62; <br>--header AccountID: 1| V
-    V --> |"/insert/multitenant/#60;suffix#62;"| C[vminsert]
+    A["requests_total{instance=foo}"] --> |<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers">/insert/#60;suffix#62;</a> <br>--header AccountID: 0| V[vmagent]
+    B["requests_total{instance=bar}"] --> |<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers">/insert/#60;suffix#62;</a> <br>--header AccountID: 1| V
+    V --> |"/insert/multitenant/prometheus/api/v1/write"| C[vminsert]
 ```
 
 In this configuration, vmagent accepts writes via the same simplified multitenant endpoints (`/insert/<suffix>`) [as vminsert does](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format).
-The tenant information is extracted from the `AccountID` and `ProjectID` HTTP headers, which are expected to be included in all incoming requests. If headers are missing, then the tenant is set to `0:0` as the default.
+The tenant information is extracted from the `AccountID` and `ProjectID` HTTP headers, which are expected to be included in all incoming requests.
 
-For all received data, vmagent will automatically convert tenant identifiers from the headers to `vm_account_id` and `vm_project_id` labels and set tenant info in metadata.
-These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via `-remoteWrite.relabelConfig`
+> If headers are missing, then the tenant is set to `0:0` by default.
+
+For all the received data, vmagent will automatically convert tenant identifiers in the headers to `vm_account_id` and `vm_project_id` labels, and set tenant info in metadata.
+These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via the `-remoteWrite.relabelConfig`
 and `-remoteWrite.urlRelabelConfig` command-line flags.
 
 vmauth can [enforce adding headers](https://docs.victoriametrics.com/victoriametrics/vmauth/#modifying-http-headers) to all
-forwarded requests via `headers` param in the config file.
+forwarded requests via the `headers` parameter in the config file.
 
 ## Adding labels to metrics
 

docs/victoriametrics/vmagent_common_flags.md

@@ -83,7 +83,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration
@@ -170,6 +170,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason="too_long_label_value"} metric at /metrics page is incremented
   -maxLabelsPerTimeseries int
      The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented
+  -mdx.label -remoteWrite.url
+     Optional label value in the form 'name=value' used to identify VictoriaMetrics metrics for MDX. Metrics containing the specified label are forwarded to -remoteWrite.url endpoints configured with `-remoteWrite.mdx.enable=true`.
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -465,6 +467,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      The maximum number of metadata to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 5000)
   -remoteWrite.maxRowsPerBlock int
      The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize (default 10000)
+  -remoteWrite.mdx.enable array
+     Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange
+     Supports array of values separated by comma or specified via multiple flags.
+     Empty values are set to false.
   -remoteWrite.oauth2.clientID array
      Optional OAuth2 clientID to use for the corresponding -remoteWrite.url
      Supports an array of values separated by comma or specified via multiple flags.

docs/victoriametrics/vmagent_enterprise_flags.md

@@ -89,7 +89,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -103,7 +103,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```

docs/victoriametrics/vmalert.md

@@ -768,7 +768,7 @@ to set the `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust
 
 ###### Prometheus
 
-vmalert uses [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) for querying
+vmalert uses [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/) for querying
 and [Prometheus Remote Write v1 protocol](https://prometheus.io/docs/specs/prw/remote_write_spec/) for persisting
 recording rules results and alerting state. Hence, it can be integrated with any Prometheus-compatible storage
 that supports these protocols.

docs/victoriametrics/vmalert_common_flags.md

@@ -124,7 +124,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -http.idleConnTimeout duration
      Timeout for incoming idle http connections (default 1m0s)
   -http.maxGracefulShutdownDuration duration
-     The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown (default 7s)
+     The maximum duration for a graceful shutdown of the HTTP server. During this period the server stops accepting new connections, but it will continue serving existing connections. The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. A highly loaded server may require increased value for a graceful shutdown (default 7s)
   -http.pathPrefix string
      An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. See https://www.robustperception.io/using-external-urls-and-proxies-with-prometheus
   -http.shutdownDelay duration

docs/victoriametrics/vmalert_enterprise_flags.md

@@ -23,7 +23,7 @@ sitemap:
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -licenseFile.reloadInterval duration
-     Interval for reloading the license file specified via -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
+     Interval for reloading the license file specified via -licenseFile. A non-positive value disables periodic and SIGHUP-triggered reloads of -licenseFile. See https://victoriametrics.com/products/enterprise/ . This flag is available only in Enterprise binaries (default 1h0m0s)
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports array of values separated by comma or specified via multiple flags.
@@ -49,7 +49,7 @@ sitemap:
   -tlsAutocertEmail string
      Contact email for the issued Let's Encrypt TLS certificates. See also -tlsAutocertHosts and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -tlsAutocertHosts array
-     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsAutocertHosts overrides -tlsCertFile and -tlsKeyFile . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Optional hostnames for automatic issuing of Let's Encrypt TLS certificates. These hostnames must be reachable at -httpListenAddr . The -httpListenAddr must listen tcp port 443 . The -tlsCertFile and -tlsKeyFile must be unset when -tlsAutocertHosts is used . See also -tlsAutocertEmail and -tlsAutocertCacheDir . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
 ```