fix ai code review remarks

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/build.yml

@@ -65,7 +65,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/changelog-linter.yml

@@ -13,7 +13,7 @@ jobs:
       contents: read
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           fetch-depth: 0  # we need full history for commit verification
           persist-credentials: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/codeql-analysis-go.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -54,14 +54,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/autobuild@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
+        uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           path: __vm
           persist-credentials: false
 
       - name: Checkout private code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: 'vm-runner'
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -79,7 +79,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -106,7 +106,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 

.github/workflows/vmui.yml

@@ -26,13 +26,11 @@ jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
     permissions:
-      checks: write
       contents: read
-      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:
           persist-credentials: false
 
@@ -70,12 +68,6 @@ jobs:
         env:
           VMUI_SKIP_INSTALL: true
 
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
       - name: Check overall status
         run: |
           echo "Lint status: ${{ steps.lint.outcome }}"

Makefile

@@ -494,11 +494,13 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 apptest-mixed: victoria-metrics-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
-	VERSION=v1.145.0; \
+	VERSION=v1.147.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
 	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
 	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
 	DIR=/tmp/$${VERSION}; \
 	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
 		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
 	); \
 	VMSELECT_PATH=$${DIR}/vmselect-prod \

README.md

@@ -2,7 +2,6 @@
 
 [![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
-[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)

app/vmagent/main.go

@@ -462,7 +462,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+		rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
+		promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
 		return true
 	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmagent/opentelemetry/request_handler.go

@@ -63,6 +63,7 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
+	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range tss {
@@ -82,7 +83,19 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 
 	var metadataTotal int
 	if prommetadata.IsEnabled() {
-		ctx.WriteRequest.Metadata = mms
+		for i := range mms {
+			mm := &mms[i]
+			mmsDst = append(mmsDst, prompb.MetricMetadata{
+				MetricFamilyName: mm.MetricFamilyName,
+				Help:             mm.Help,
+				Type:             mm.Type,
+				Unit:             mm.Unit,
+
+				AccountID: mm.AccountID,
+				ProjectID: mm.ProjectID,
+			})
+		}
+		ctx.WriteRequest.Metadata = mmsDst
 		metadataTotal = len(mms)
 	}
 

app/vmagent/remotewrite/client.go

@@ -151,17 +151,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
+	rwURL, err := url.Parse(remoteWriteURL)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	retryMaxIntervalFlag := retryMaxTime
 	if retryMaxInterval.String() != "" {
 		retryMaxIntervalFlag = retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
+		remoteWriteURL:   rwURL.String(),
 		authCfg:          authCfg,
 		awsCfg:           awsCfg,
 		fq:               fq,

app/vmagent/remotewrite/obfuscate.go

@@ -0,0 +1,108 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+type obfuscateLabelsCtx struct {
+	labels []prompb.Label
+
+	// buf holds allocations for cached results below
+	buf []byte
+
+	// cacheResults maps original label values to their SHA-256 hex digests,
+	// avoiding redundant hashing of repeated values within a single batch.
+	cacheResults map[string]string
+}
+
+func (olctx *obfuscateLabelsCtx) reset() {
+	promrelabel.CleanLabels(olctx.labels)
+	olctx.labels = olctx.labels[:0]
+	clear(olctx.cacheResults)
+	olctx.buf = olctx.buf[:0]
+}
+
+var obfuscateLabelsCtxPool = &sync.Pool{
+	New: func() any {
+		return &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+	},
+}
+
+func getObfuscateLabelsCtx() *obfuscateLabelsCtx {
+	return obfuscateLabelsCtxPool.Get().(*obfuscateLabelsCtx)
+}
+
+func putObfuscateLabelsCtx(ctx *obfuscateLabelsCtx) {
+	ctx.reset()
+	obfuscateLabelsCtxPool.Put(ctx)
+}
+
+func (olctx *obfuscateLabelsCtx) obfuscate(tss []prompb.TimeSeries, obfuscateLabels []string) []prompb.TimeSeries {
+	if len(obfuscateLabels) == 0 || len(tss) == 0 {
+		return tss
+	}
+	labels := olctx.labels
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		found := false
+		for _, labelName := range obfuscateLabels {
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], labelName)
+			if tmp == nil {
+				continue
+			}
+			found = true
+			if obfuscatedValue, ok := olctx.cacheResults[tmp.Value]; ok {
+				// fast path: the obfuscated result was calculated before
+				tmp.Value = obfuscatedValue
+				continue
+			}
+
+			digest := sha256.Sum256(bytesutil.ToUnsafeBytes(tmp.Value))
+			buf := olctx.buf
+			bufLen := len(buf)
+			buf = hex.AppendEncode(buf, digest[:])
+			obfuscatedValue := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			olctx.buf = buf
+			olctx.cacheResults[tmp.Value] = obfuscatedValue
+			tmp.Value = obfuscatedValue
+		}
+		if found {
+			ts.Labels = labels[labelsLen:]
+		} else {
+			labels = labels[:labelsLen]
+		}
+	}
+	olctx.labels = labels
+	return tss
+}
+
+func (rwctx *remoteWriteCtx) initObfuscateLabels() {
+	if len(*obfuscateLabels) == 0 {
+		return
+	}
+	idx := rwctx.idx
+	rwObfuscateLabels := obfuscateLabels.GetOptionalArg(idx)
+	rwObfuscateLabelsList := strings.Split(rwObfuscateLabels, "^^")
+
+	for _, label := range rwObfuscateLabelsList {
+		if label == "" {
+			continue
+		}
+		if !slices.Contains(rwctx.obfuscateLabels, label) {
+			rwctx.obfuscateLabels = append(rwctx.obfuscateLabels, label)
+		}
+	}
+}

app/vmagent/remotewrite/obfuscate_test.go

@@ -0,0 +1,186 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func TestRemoteWriteObfuscateLabels(t *testing.T) {
+	f := func(obfuscateLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
+		t.Helper()
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		defer metrics.UnregisterAllMetrics()
+		originValue := *obfuscateLabels
+		defer func() {
+			*obfuscateLabels = originValue
+		}()
+		*obfuscateLabels = []string{obfuscateLabelList}
+		rwctx.initObfuscateLabels()
+
+		outputTss := olctx.obfuscate(inputTss, rwctx.obfuscateLabels)
+
+		if !reflect.DeepEqual(expectedTss, outputTss) {
+			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
+		}
+	}
+
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+
+	// 1. obfuscation is not set.
+	f("",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 1. obfuscation is set for another rwctx.
+	f(",ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 2. obfuscate the value of "ip" label
+	f("ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 3. obfuscate the values of "ip" and "instance"
+	f("ip^^instance",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: sha256Result("1234")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 4. duplicate label names in config must produce single SHA-256, not double
+	f("ip^^ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+}

app/vmagent/remotewrite/obfuscate_timing_test.go

@@ -0,0 +1,94 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func BenchmarkRemoteWriteObfuscateLabels(b *testing.B) {
+	originValue := *obfuscateLabels
+	defer func() {
+		*obfuscateLabels = originValue
+	}()
+	*obfuscateLabels = []string{"ip^^instance"}
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+	expected := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("123")},
+				{Name: "instance", Value: sha256Result("12345")},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("1236")},
+				{Name: "instance", Value: sha256Result("some-long-instante-string")},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	defer metrics.UnregisterAllMetrics()
+
+	inputTss := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "123"},
+				{Name: "instance", Value: "12345"},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "1236"},
+				{Name: "instance", Value: "some-long-instante-string"},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	b.RunParallel(func(pb *testing.PB) {
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		rwctx.initObfuscateLabels()
+		var localTss []prompb.TimeSeries
+		for pb.Next() {
+			// always make a shallow copy because obfuscate changes input
+			localTss = localTss[:0]
+			localTss = append(localTss, inputTss...)
+			olctx.reset()
+			outputTss := olctx.obfuscate(localTss, rwctx.obfuscateLabels)
+			if !reflect.DeepEqual(expected, outputTss) {
+				b.Fatalf("unexpected output: got: \n%v\n want: \n%v\n", outputTss, expected)
+			}
+		}
+	})
+
+}

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,6 +83,16 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
 	p := remoteWriteURLRelabelConfigData.Load()
@@ -108,6 +119,24 @@ func WriteURLRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(d)
 }
 
+// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
+func GetURLRelabelConfigString() []string {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
+	}
+	var ss []string
+	for i := range *remoteWriteURLs {
+		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
+		ss = append(ss, string(cfgDataBytes))
+	}
+	return ss
+}
+
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {

app/vmagent/remotewrite/remotewrite.go

@@ -107,7 +107,12 @@ var (
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
 
 	enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
+		"Can be combined with -remoteWrite.obfuscateLabels to hide sensitive label values in the forwarded metrics. "+
 		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
+	obfuscateLabels = flagutil.NewArrayString("remoteWrite.obfuscateLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
+		"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
+		"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
 )
 
 var (
@@ -881,6 +886,8 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
 
+	obfuscateLabels []string
+
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
 	mdxRowsPreserved       *metrics.Counter
@@ -995,6 +1002,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 		rowsDroppedOnPushFailure:     metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
 	rwctx.initStreamAggrConfig()
+	rwctx.initObfuscateLabels()
 
 	if enableMdx.GetOptionalArg(argIdx) {
 		mdxFilter := mdx.NewFilter()
@@ -1064,7 +1072,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 
 	copyTimeSeriesIfNeeded := func() {
 		if v == nil {
-			v := tssPool.Get().(*[]prompb.TimeSeries)
+			v = tssPool.Get().(*[]prompb.TimeSeries)
 			tss = append(*v, tss...)
 		}
 	}
@@ -1198,24 +1206,41 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
 func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompb.TimeSeries
+	var olctx *obfuscateLabelsCtx
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if olctx != nil {
+			putObfuscateLabelsCtx(olctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
 
+	if len(rwctx.obfuscateLabels) != 0 {
+		copyTimeSeriesIfNeeded()
+		olctx = getObfuscateLabelsCtx()
+		tss = olctx.obfuscate(tss, rwctx.obfuscateLabels)
+	}
+
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 

app/vmalert-tool/unittest/recording.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"reflect"
 	"sort"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -94,10 +93,3 @@ Outer:
 	}
 	return
 }
-
-func durationToTime(pd *promutil.Duration) time.Time {
-	if pd == nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(pd.Duration().Milliseconds())
-}

app/vmalert-tool/unittest/unittest.go

@@ -44,12 +44,19 @@ import (
 var (
 	storagePath    string
 	httpListenAddr string
-	// insert series from 1970-01-01T00:00:00
-	testStartTime          = time.Unix(0, 0).UTC()
+	// Insert series from 2000-01-01T00:00:00.
+	testStartTime          = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
 
+func durationToTime(pd *promutil.Duration) time.Time {
+	if pd == nil {
+		return testStartTime
+	}
+	return testStartTime.Add(pd.Duration())
+}
+
 const (
 	testStoragePath = "vmalert-unittest"
 )

app/vmalert/datasource/client.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -267,7 +268,11 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
+		err = &httpserver.ErrorWithStatusCode{
+			StatusCode: resp.StatusCode,
+			Err:        fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body),
+		}
+		return nil, err
 	}
 	return resp, nil
 }

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -94,6 +95,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	hc := &http.Client{Transport: tr}
+	datasourceURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
+	}
+	hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
 
 	if extraParams == nil {
 		extraParams = url.Values{}
@@ -120,9 +127,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	}
 
 	return &Client{
-		c:                &http.Client{Transport: tr},
+		c:                hc,
 		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		datasourceURL:    strings.TrimSuffix(datasourceURL.String(), "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
 		extraParams:      extraParams,

app/vmalert/notifier/alert.go

@@ -31,6 +31,8 @@ type Alert struct {
 	State AlertState
 	// Expr contains expression that was executed to generate the Alert
 	Expr string
+	// Interval contains the evaluation interval of the Alert's group
+	Interval time.Duration
 	// ActiveAt defines the moment of time when Alert has become active
 	ActiveAt time.Time
 	// Start defines the moment of time when Alert has become firing
@@ -84,6 +86,7 @@ type AlertTplData struct {
 	Labels    map[string]string
 	Value     float64
 	Expr      string
+	Interval  time.Duration
 	AlertID   uint64
 	GroupID   uint64
 	ActiveAt  time.Time
@@ -96,6 +99,7 @@ var tplHeaders = []string{
 	"{{ $type := .Type }}",
 	"{{ $labels := .Labels }}",
 	"{{ $expr := .Expr }}",
+	"{{ $interval := .Interval }}",
 	"{{ $externalLabels := .ExternalLabels }}",
 	"{{ $externalURL := .ExternalURL }}",
 	"{{ $alertID := .AlertID }}",
@@ -115,6 +119,7 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
 		Type:     a.Type,
 		Labels:   labels,
 		Expr:     a.Expr,
+		Interval: a.Interval,
 		AlertID:  a.ID,
 		GroupID:  a.GroupID,
 		ActiveAt: a.ActiveAt,

app/vmalert/notifier/alert_test.go

@@ -129,6 +129,17 @@ func TestAlertExecTemplate(t *testing.T) {
 		"exprEscapedHTML":   "vm_rows{"label"="bar"}<0",
 	})
 
+	// interval-template
+	f(&Alert{
+		Interval: 10 * time.Second,
+	}, map[string]string{
+		"interval":         "{{ .Interval }}",
+		"intervalVariable": "{{ $interval }}",
+	}, map[string]string{
+		"interval":         "10s",
+		"intervalVariable": "10s",
+	})
+
 	// query
 	f(&Alert{
 		Expr: `vm_rows{"label"="bar"}>0`,

app/vmalert/remoteread/init.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -76,7 +78,13 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	c := &http.Client{Transport: tr}
+	rrURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
+	}
 
+	c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +97,5 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
-	c := &http.Client{Transport: tr}
-	return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
+	return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Concurrency > 0 {
 		cc = cfg.Concurrency
 	}
+	hc := &http.Client{
+		Timeout:   *sendTimeout,
+		Transport: cfg.Transport,
+	}
+	rwURL, err := url.Parse(cfg.Addr)
+	if err != nil {
+		logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	c := &Client{
-		c: &http.Client{
-			Timeout:   *sendTimeout,
-			Transport: cfg.Transport,
-		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		c:             hc,
+		addr:          strings.TrimSuffix(rwURL.String(), "/"),
 		authCfg:       cfg.AuthCfg,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,

app/vmalert/replay.go

@@ -25,11 +25,12 @@ var (
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
 		"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
-		"Defines how many retries to make before giving up on rule if request for it returns an error.")
+		"Defines how many retries to make before giving up on rule if request for it returns a retriable error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
 		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
+	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
@@ -73,7 +74,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
 
 	for _, cfg := range groupsCfg {
 		ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
-		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency)
+		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency, *continueWithExecutionErr)
 	}
 	logger.Infof("replay evaluation finished, generated %d samples", totalRows)
 	if err := rw.Close(); err != nil {

app/vmalert/replay_test.go

@@ -8,8 +8,10 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
 type fakeReplayQuerier struct {
@@ -32,6 +34,14 @@ func (fc *fakeRWClient) Close() error {
 }
 
 func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
+	_, err = metricsql.Parse(q)
+	if err != nil {
+		return res, &httpserver.ErrorWithStatusCode{
+			StatusCode: 422,
+			Err:        err,
+		}
+	}
+
 	key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
 	dps, ok := fr.registry[q]
 	if !ok {
@@ -275,4 +285,28 @@ func TestReplay(t *testing.T) {
 			},
 		},
 	}, 10)
+
+	// rule with wrong expression won't break the other rule with continueWithExecutionErr
+	continueWithExecutionErrOld := *continueWithExecutionErr
+	defer func() {
+		*continueWithExecutionErr = continueWithExecutionErrOld
+	}()
+	*continueWithExecutionErr = true
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 1, time.Millisecond, []config.Group{
+		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+		{Rules: []config.Rule{{Record: "bar", Expr: "up ++"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string][]datasource.Metric{
+			"sum(up)": {
+				"12:00:00+12:01:00": {
+					{
+						Timestamps: []int64{1, 2},
+						Values:     []float64{1, 2},
+					},
+				},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	}, 2)
 }

app/vmalert/rule/alerting.go

@@ -530,6 +530,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				ar.logDebugf(ts, a, "INACTIVE => PENDING")
 			}
 			a.Value = m.Values[0]
+			a.Interval = ar.EvalInterval
 			a.Annotations = annotations
 			a.KeepFiringSince = time.Time{}
 			continue
@@ -612,6 +613,7 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
 		Type:      ar.Type.String(),
 		Labels:    ls.origin,
 		Expr:      ar.Expr,
+		Interval:  ar.EvalInterval,
 		AlertID:   hash(ls.processed),
 		GroupID:   ar.GroupID,
 		ActiveAt:  activeAt,
@@ -673,6 +675,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
 		Name:        ar.Name,
 		Type:        ar.Type.String(),
 		Expr:        ar.Expr,
+		Interval:    ar.EvalInterval,
 		For:         ar.For,
 		ActiveAt:    start,
 		Value:       m.Values[0],

app/vmalert/rule/alerting_test.go

@@ -662,6 +662,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -682,6 +683,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "for-firing"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -703,6 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -759,6 +762,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -771,6 +775,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series", "foo": "bar"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -1134,7 +1139,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		fq.Add(metrics...)
 		fq.SetPartialResponse(isResponsePartial)
 
-		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
+		ts := time.Unix(3600, 0)
+		if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		for hash, expAlert := range alertsExpected {
@@ -1152,12 +1158,14 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 
 	f(&AlertingRule{
-		Name: "common",
+		Name:         "common",
+		EvalInterval: time.Hour,
 		Labels: map[string]string{
 			"region": "east",
 		},
 		Annotations: map[string]string{
-			"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":   `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"dashboard": `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1166,7 +1174,8 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "foo"`,
+				"summary":   `common: Too high connection number for "foo"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1176,7 +1185,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		},
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "bar"`,
+				"summary":   `common: Too high connection number for "bar"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1388,7 +1398,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
 		"pod":           "vmalert-0",
-		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1398,7 +1408,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)

app/vmalert/rule/group.go

@@ -548,7 +548,7 @@ func (g *Group) infof(format string, args ...any) {
 }
 
 // Replay performs group replay
-func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int) int {
+func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	var total int
 	step := g.Interval * time.Duration(maxDataPoint)
 	ri := rangeIterator{start: start, end: end, step: step}
@@ -576,7 +576,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 			if !disableProgressBar {
 				bar = pb.StartNew(iterations)
 			}
-			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			if bar != nil {
 				bar.Finish()
 			}
@@ -598,7 +598,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 		rule := g.Rules[i]
 		sem <- struct{}{}
 		wg.Go(func() {
-			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			<-sem
 		})
 	}
@@ -618,7 +618,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 	return total
 }
 
-func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
+func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
 	// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
 	// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
@@ -633,7 +633,7 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 		start := ri.s
 		end := ri.e
 		wg.Go(func() {
-			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
+			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts, continueWithExecutionErr)
 			if err != nil {
 				logger.Fatalf("rule %q: %s", r, err)
 			}

app/vmalert/rule/rule.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -118,7 +120,7 @@ func (s *ruleState) add(e StateEntry) {
 	s.entries[s.cur] = e
 }
 
-func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
+func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int, continueWithExecutionErr bool) (int, error) {
 	var err error
 	var tss []prompb.TimeSeries
 	for i := range replayRuleRetryAttempts {
@@ -126,6 +128,21 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
 		if err == nil {
 			break
 		}
+		// retry request if possible to tolerate temporary network or datasource unavailability issues
+		var esc *httpserver.ErrorWithStatusCode
+		if errors.As(err, &esc) {
+			statusCode := esc.StatusCode
+			// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
+			// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
+			if statusCode == http.StatusUnprocessableEntity {
+				if continueWithExecutionErr {
+					logger.Errorf("rule %q: %s", r, err)
+					return 0, nil
+				} else {
+					return 0, err
+				}
+			}
+		}
 		logger.Errorf("attempt %d to execute rule %q failed: %s", i+1, r, err)
 		time.Sleep(time.Second)
 	}

app/vmalert/rule/web.go

@@ -75,10 +75,13 @@ type GroupAlerts struct {
 // ApiRule represents a Rule for web view
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type ApiRule struct {
-	// State must be one of these under following scenarios
-	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing ruleState.
-	//  "firing": at least 1 alert in the rule in firing state.
-	//  "inactive": no alert in the rule in firing or pending state.
+	// Rule state must be one of these under following scenarios:
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state. (only for alerting rules)
+	//  "firing": at least 1 alert in the rule in firing state. (only for alerting rules)
+	//  "inactive": rule's last evaluation was successful but no alert in the rule in firing or pending state. (only for alerting rules)
+	//  "unhealthy": rule's last evaluation was failed with error. (for both alerting and recording rules)
+	// 	"nomatch": rule's last evaluation was successful but no time series matched the rule's expression. (for both alerting and recording rules)
+	//  "ok": the recording rule's last evaluation was successful. (only for recording rules)
 	State string `json:"state"`
 	Name  string `json:"name"`
 	// Query represents Rule's `expression` field
@@ -237,6 +240,7 @@ func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 }
 
 func (r *ApiRule) ExtendState() {
+	// if alerting rule already has alerts, then state is already set to either "pending" or "firing" and we don't need to change it
 	if len(r.Alerts) > 0 {
 		return
 	}

app/vmalert/web.go

@@ -115,7 +115,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		WriteRule(w, r, rule)
 		return true
-	// current used by old vmalert UI and Grafana Alerts
+	// used by old vmalert UI
 	case "/vmalert/groups", "/rules":
 		rf, err := newRulesFilter(r)
 		if err != nil {
@@ -128,6 +128,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			state = rf.states[0]
 			rf.states = rf.states[:1]
 		}
+		// enable extendedStates by default for vmalert UI
+		rf.extendedStates = true
 		lr := rh.groups(rf)
 		WriteListGroups(w, r, lr.Data.Groups, state)
 		return true
@@ -543,6 +545,8 @@ func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 			if !groupFound && !strings.Contains(strings.ToLower(rule.Name), rf.search) {
 				continue
 			}
+			// extendedStates is used by the vmalert UI to extend the rule state with values such as "nomatch" and "unhealthy".
+			// those states are not supported by Grafana and not part of the Prometheus API spec yet
 			if rf.extendedStates {
 				rule.ExtendState()
 			}

app/vmalert/web.qtpl

@@ -130,9 +130,12 @@
                             data-bs-target="#item-{%s g.ID %}"
                         >
                             <span class="d-flex gap-2">
-                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.States["unhealthy"] %}</span> {% endif %}
-                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.States["nomatch"] %}</span> {% endif %}
-                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.States["ok"] %}</span>
+                                {% if g.States["inactive"] > 0 %}<span class="badge bg-light text-success border border-success" title="None of the alert instances is in a pending or firing state">{%d g.States["inactive"] %} inactive</span> {% endif %}
+                                {% if g.States["pending"] > 0 %}<span class="badge bg-warning text-dark" title="At least one alert instance is pending">{%d g.States["pending"] %} pending</span> {% endif %}
+                                {% if g.States["firing"] > 0 %}<span class="badge bg-danger" title="At least one alert instance is firing">{%d g.States["firing"] %} firing</span> {% endif %}
+                                {% if g.States["ok"] > 0 %}<span class="badge bg-success" title="Recording rule last evaluation succeeded">{%d g.States["ok"] %} ok</span>{% endif %}
+                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Last evaluation failed with an error">{%d g.States["unhealthy"] %} unhealthy</span> {% endif %}
+                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Rule expression matched no time series">{%d g.States["nomatch"] %} nomatch</span> {% endif %}
                             </span>
                         </span>
                     </span>
@@ -169,7 +172,7 @@
                             <thead>
                                 <tr>
                                     <th scope="col" class="w-60">Rule</th>
-                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
+                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series Returned</th>
                                     <th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
                                 </tr>
                             </thead>
@@ -188,8 +191,21 @@
                                                     {% else %}
                                                         <b>record:</b> {%s r.Name %}
                                                     {% endif %}
-                                                    |
+                                                    {% if r.State == "inactive" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "pending" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "firing" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "ok" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "unhealthy" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "nomatch" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% endif %}
                                                     {%= seriesFetchedWarn(prefix, &r) %}
+                                                    |
                                                     <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
                                                 </div>
                                                 <div class="col-12">

app/vmauth/main.go

@@ -386,7 +386,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 			return nil, &httpserver.ErrorWithStatusCode{
 				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
 					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
-				StatusCode: http.StatusBadRequest,
+				StatusCode: http.StatusRequestTimeout,
 			}
 		}
 

app/vmauth/main_synctest_test.go

@@ -0,0 +1,54 @@
+//go:build synctest
+
+package main
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+func TestBufferRequestBody_Timeout(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+
+		defaultMaxQueueDuration := *maxQueueDuration
+		defer func() {
+			*maxQueueDuration = defaultMaxQueueDuration
+		}()
+
+		*maxQueueDuration = 100 * time.Millisecond
+
+		ctx, cancel := context.WithTimeout(context.Background(), *maxQueueDuration)
+		defer cancel()
+
+		_, err := bufferRequestBody(ctx, &timeoutBody{delay: *maxQueueDuration + 100*time.Millisecond}, "foo")
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		esc, ok := err.(*httpserver.ErrorWithStatusCode)
+		if !ok {
+			t.Fatalf("unexpected error type: %s", err)
+		}
+		if esc.StatusCode != http.StatusRequestTimeout {
+			t.Fatalf("read request body timeout meet unexpected status code; got %d; want %d", esc.StatusCode, http.StatusRequestTimeout)
+		}
+	})
+}
+
+type timeoutBody struct {
+	delay time.Duration
+}
+
+func (r *timeoutBody) Read(_ []byte) (int, error) {
+	time.Sleep(r.delay)
+	return 0, context.DeadlineExceeded
+}
+
+func (r *timeoutBody) Close() error {
+	return nil
+}

app/vmctl/remoteread/remoteread.go

@@ -8,12 +8,14 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -234,9 +236,29 @@ func processResponse(body io.ReadCloser, callback StreamCallback) error {
 	// shouldn't be accounted as an error.
 	for _, res := range readResp.Results {
 		for _, ts := range res.Timeseries {
-			vmTs := convertSamples(ts.Samples, ts.Labels)
-			if err := callback(vmTs); err != nil {
-				return err
+			// A series contains either float samples or native histogram samples.
+			// Both fields are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain samples of both types.
+			if len(ts.Samples) > 0 {
+				vmTs := convertSamples(ts.Samples, ts.Labels)
+				if err := callback(vmTs); err != nil {
+					return err
+				}
+			}
+			if len(ts.Histograms) > 0 {
+				hSamples := make([]histogramSample, 0, len(ts.Histograms))
+				for _, h := range ts.Histograms {
+					hSamples = append(hSamples, histogramSample{
+						timestamp: h.Timestamp,
+						fh:        h.ToFloatHistogram(),
+					})
+				}
+				for _, vmTs := range convertHistograms(hSamples, ts.Labels) {
+					if err := callback(vmTs); err != nil {
+						return err
+					}
+				}
 			}
 		}
 	}
@@ -263,17 +285,45 @@ func processStreamResponse(body io.ReadCloser, callback StreamCallback) error {
 
 		for _, series := range res.ChunkedSeries {
 			samples := make([]prompb.Sample, 0)
+			var hSamples []histogramSample
 			for _, chunk := range series.Chunks {
-				s, err := parseSamples(chunk.Data)
-				if err != nil {
-					return err
+				switch chunk.Type {
+				case prompb.Chunk_XOR, prompb.Chunk_UNKNOWN:
+					// In proto3 the `type` field may be left unset (UNKNOWN) for XOR chunks.
+					// Prometheus remote.proto: "REQUIREMENT: when using proto3, this field
+					// MUST be set when using anything else than XOR". Senders before native
+					// histograms support (Prometheus < 2.40) do not set this field at all,
+					// so UNKNOWN chunks must be parsed as XOR ones.
+					s, err := parseSamples(chunk.Data)
+					if err != nil {
+						return err
+					}
+					samples = append(samples, s...)
+				case prompb.Chunk_HISTOGRAM, prompb.Chunk_FLOAT_HISTOGRAM:
+					hs, err := parseHistograms(chunk.Type, chunk.Data)
+					if err != nil {
+						return err
+					}
+					hSamples = append(hSamples, hs...)
+				default:
+					return fmt.Errorf("unsupported chunk encoding %q", chunk.Type)
 				}
-				samples = append(samples, s...)
 			}
 
-			ts := convertSamples(samples, series.Labels)
-			if err := callback(ts); err != nil {
-				return err
+			// A series contains either XOR chunks or native histogram chunks.
+			// Both are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain chunks of both types.
+			if len(samples) > 0 {
+				ts := convertSamples(samples, series.Labels)
+				if err := callback(ts); err != nil {
+					return err
+				}
+			}
+			for _, ts := range convertHistograms(hSamples, series.Labels) {
+				if err := callback(ts); err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -312,6 +362,151 @@ func parseSamples(chunk []byte) ([]prompb.Sample, error) {
 	return samples, it.Err()
 }
 
+// histogramSample represents a single native histogram sample.
+type histogramSample struct {
+	timestamp int64
+	fh        *histogram.FloatHistogram
+}
+
+func parseHistograms(encoding prompb.Chunk_Encoding, chunk []byte) ([]histogramSample, error) {
+	var enc chunkenc.Encoding
+	switch encoding {
+	case prompb.Chunk_HISTOGRAM:
+		enc = chunkenc.EncHistogram
+	case prompb.Chunk_FLOAT_HISTOGRAM:
+		enc = chunkenc.EncFloatHistogram
+	default:
+		return nil, fmt.Errorf("unsupported histogram chunk encoding %q", encoding)
+	}
+	c, err := chunkenc.FromData(enc, chunk)
+	if err != nil {
+		return nil, fmt.Errorf("error read chunk: %w", err)
+	}
+
+	var hSamples []histogramSample
+	it := c.Iterator(nil)
+	for {
+		typ := it.Next()
+		if typ == chunkenc.ValNone {
+			break
+		}
+		switch typ {
+		case chunkenc.ValHistogram:
+			ts, h := it.AtHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        h.ToFloat(nil),
+			})
+		case chunkenc.ValFloatHistogram:
+			ts, fh := it.AtFloatHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        fh,
+			})
+		default:
+			// Skip unsupported values
+			continue
+		}
+	}
+	if err := it.Err(); err != nil {
+		return nil, fmt.Errorf("error iterate over chunks: %w", err)
+	}
+
+	return hSamples, nil
+}
+
+// convertHistograms converts native histogram samples into VictoriaMetrics histogram
+// time series in the same way as VictoriaMetrics converts native histograms
+// received via Prometheus remote write protocol: every native histogram sample
+// is converted into `<name>_count` and `<name>_sum` series plus a set of
+// `<name>_bucket` series with `vmrange` labels containing non-cumulative bucket counts.
+// The only difference is that for native histograms with custom buckets (NHCB)
+// bucket bounds are taken from the custom values, while the remote write protocol
+// parser ignores custom values and estimates the bounds with the exponential formula.
+// See https://prometheus.io/docs/specs/native_histograms/#data-model
+func convertHistograms(hSamples []histogramSample, labels []prompb.Label) []*vm.TimeSeries {
+	if len(hSamples) == 0 {
+		return nil
+	}
+
+	labelPairs := make([]vm.LabelPair, 0, len(labels))
+	nameValue := ""
+	for _, label := range labels {
+		if label.Name == "__name__" {
+			nameValue = label.Value
+			continue
+		}
+		labelPairs = append(labelPairs, vm.LabelPair{Name: label.Name, Value: label.Value})
+	}
+	// the metric has no name, skip it in the same way as VictoriaMetrics does
+	// when it receives a native histogram without the metric name via remote write protocol.
+	if nameValue == "" {
+		return nil
+	}
+
+	countSeries := &vm.TimeSeries{
+		Name:       nameValue + "_count",
+		LabelPairs: labelPairs,
+	}
+	sumSeries := &vm.TimeSeries{
+		Name:       nameValue + "_sum",
+		LabelPairs: labelPairs,
+	}
+	bucketSeries := make(map[string]*vm.TimeSeries)
+	// vmranges preserves the order of bucketSeries creation
+	// in order to get deterministic results.
+	var vmranges []string
+
+	for _, hs := range hSamples {
+		fh := hs.fh
+		countSeries.Timestamps = append(countSeries.Timestamps, hs.timestamp)
+		countSeries.Values = append(countSeries.Values, fh.Count)
+		sumSeries.Timestamps = append(sumSeries.Timestamps, hs.timestamp)
+		sumSeries.Values = append(sumSeries.Values, fh.Sum)
+
+		it := fh.AllBucketIterator()
+		for it.Next() {
+			b := it.At()
+			if b.Count <= 0 {
+				continue
+			}
+			vmrange := formatVmrange(b.Lower, b.Upper)
+			s := bucketSeries[vmrange]
+			if s == nil {
+				bucketLabelPairs := make([]vm.LabelPair, len(labelPairs), len(labelPairs)+1)
+				copy(bucketLabelPairs, labelPairs)
+				bucketLabelPairs = append(bucketLabelPairs, vm.LabelPair{Name: "vmrange", Value: vmrange})
+				s = &vm.TimeSeries{
+					Name:       nameValue + "_bucket",
+					LabelPairs: bucketLabelPairs,
+				}
+				bucketSeries[vmrange] = s
+				vmranges = append(vmranges, vmrange)
+			}
+			s.Timestamps = append(s.Timestamps, hs.timestamp)
+			s.Values = append(s.Values, b.Count)
+		}
+	}
+
+	tss := make([]*vm.TimeSeries, 0, 2+len(vmranges))
+	tss = append(tss, countSeries, sumSeries)
+	for _, vmrange := range vmranges {
+		tss = append(tss, bucketSeries[vmrange])
+	}
+	return tss
+}
+
+// formatVmrange formats the given bucket bounds into `vmrange` label value
+// in the same way as VictoriaMetrics does for native histograms
+// received via Prometheus remote write protocol.
+func formatVmrange(lower, upper float64) string {
+	b := make([]byte, 0, 24)
+	b = strconv.AppendFloat(b, lower, 'e', 3, 64)
+	b = append(b, "..."...)
+	b = strconv.AppendFloat(b, upper, 'e', 3, 64)
+	return string(b)
+}
+
 type keyValue struct {
 	key   string
 	value string

app/vmctl/remoteread/remoteread_test.go

@@ -0,0 +1,334 @@
+package remoteread
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage/remote"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+)
+
+func testHistogram(mul int64) *histogram.Histogram {
+	return &histogram.Histogram{
+		Schema:          0,
+		Count:           uint64(10 * mul),
+		Sum:             25.5 * float64(mul),
+		ZeroThreshold:   0.001,
+		ZeroCount:       uint64(2 * mul),
+		PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}},
+		PositiveBuckets: []int64{1 * mul, 2 * mul},
+		NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+		NegativeBuckets: []int64{4 * mul},
+	}
+}
+
+func TestConvertHistograms(t *testing.T) {
+	f := func(hSamples []histogramSample, labels []prompb.Label, expected []*vm.TimeSeries) {
+		t.Helper()
+
+		tss := convertHistograms(hSamples, labels)
+		if !reflect.DeepEqual(tss, expected) {
+			t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, expected)
+		}
+	}
+
+	// series without samples
+	f(nil, []prompb.Label{{Name: "__name__", Value: "foo"}}, nil)
+
+	// series without the metric name must be skipped
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+	}, []prompb.Label{{Name: "job", Value: "bar"}}, nil)
+
+	// native histogram must be converted to _count, _sum and _bucket series
+	// in the same way as VictoriaMetrics does for Prometheus remote write protocol
+	labels := []prompb.Label{
+		{Name: "__name__", Value: "request_duration_seconds"},
+		{Name: "job", Value: "bar"},
+	}
+	jobLabel := []vm.LabelPair{{Name: "job", Value: "bar"}}
+	bucketLabels := func(vmrange string) []vm.LabelPair {
+		return []vm.LabelPair{
+			{Name: "job", Value: "bar"},
+			{Name: "vmrange", Value: vmrange},
+		}
+	}
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+		{timestamp: 2000, fh: testHistogram(2).ToFloat(nil)},
+	}, labels, []*vm.TimeSeries{
+		{
+			Name:       "request_duration_seconds_count",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{10, 20},
+		},
+		{
+			Name:       "request_duration_seconds_sum",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{25.5, 51},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e+00...-5.000e-01"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{4, 8},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e-03...1.000e-03"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{2, 4},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("5.000e-01...1.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{1, 2},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("1.000e+00...2.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{3, 6},
+		},
+	})
+}
+
+func TestParseHistograms(t *testing.T) {
+	c := chunkenc.NewHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 2000, testHistogram(2), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse histogram chunk: %s", err)
+	}
+	if len(hSamples) != 2 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 2", len(hSamples))
+	}
+	for i, expected := range []struct {
+		timestamp int64
+		count     float64
+		sum       float64
+	}{
+		{timestamp: 1000, count: 10, sum: 25.5},
+		{timestamp: 2000, count: 20, sum: 51},
+	} {
+		if hSamples[i].timestamp != expected.timestamp {
+			t.Fatalf("unexpected timestamp; got %d; want %d", hSamples[i].timestamp, expected.timestamp)
+		}
+		if hSamples[i].fh.Count != expected.count {
+			t.Fatalf("unexpected count; got %f; want %f", hSamples[i].fh.Count, expected.count)
+		}
+		if hSamples[i].fh.Sum != expected.sum {
+			t.Fatalf("unexpected sum; got %f; want %f", hSamples[i].fh.Sum, expected.sum)
+		}
+	}
+
+	// unsupported chunk encoding must return error
+	if _, err := parseHistograms(prompb.Chunk_XOR, c.Bytes()); err == nil {
+		t.Fatalf("expecting non-nil error for unsupported chunk encoding")
+	}
+}
+
+func TestProcessResponse(t *testing.T) {
+	readResp := &prompb.ReadResponse{
+		Results: []*prompb.QueryResult{
+			{
+				Timeseries: []*prompb.TimeSeries{
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "cpu_usage"},
+							{Name: "job", Value: "bar"},
+						},
+						Samples: []prompb.Sample{
+							{Timestamp: 1000, Value: 1.5},
+						},
+					},
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "request_duration_seconds"},
+							{Name: "job", Value: "bar"},
+						},
+						Histograms: []prompb.Histogram{
+							prompb.FromIntHistogram(1000, testHistogram(1)),
+						},
+					},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(readResp)
+	if err != nil {
+		t.Fatalf("cannot marshal ReadResponse: %s", err)
+	}
+	compressed := snappy.Encode(nil, data)
+
+	var tss []*vm.TimeSeries
+	err = processResponse(io.NopCloser(bytes.NewReader(compressed)), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process response: %s", err)
+	}
+
+	// 1 float series + _count + _sum + 4 buckets
+	if len(tss) != 7 {
+		t.Fatalf("unexpected number of time series; got %d; want 7", len(tss))
+	}
+	if tss[0].Name != "cpu_usage" || !reflect.DeepEqual(tss[0].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[1].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[1])
+	}
+	if tss[2].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[2].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[2])
+	}
+	for _, ts := range tss[3:] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+}
+
+type nopFlusher struct{}
+
+func (nopFlusher) Flush() {}
+
+func TestProcessStreamResponse(t *testing.T) {
+	// build a histogram chunk
+	hc := chunkenc.NewHistogramChunk()
+	hApp, err := hc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create histogram chunk appender: %s", err)
+	}
+	if _, _, _, err := hApp.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	// build a float chunk
+	xc := chunkenc.NewXORChunk()
+	xApp, err := xc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create xor chunk appender: %s", err)
+	}
+	xApp.Append(0, 1000, 1.5)
+
+	res := &prompb.ChunkedReadResponse{
+		ChunkedSeries: []*prompb.ChunkedSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "request_duration_seconds"},
+					{Name: "job", Value: "bar"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_HISTOGRAM, Data: hc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "cpu_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_XOR, Data: xc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "memory_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					// the `type` field may be unset for XOR chunks,
+					// such chunks must be parsed as XOR ones
+					{Type: prompb.Chunk_UNKNOWN, Data: xc.Bytes()},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(res)
+	if err != nil {
+		t.Fatalf("cannot marshal ChunkedReadResponse: %s", err)
+	}
+	var buf bytes.Buffer
+	cw := remote.NewChunkedWriter(&buf, nopFlusher{})
+	if _, err := cw.Write(data); err != nil {
+		t.Fatalf("cannot write chunked response: %s", err)
+	}
+
+	var tss []*vm.TimeSeries
+	err = processStreamResponse(io.NopCloser(&buf), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process stream response: %s", err)
+	}
+
+	// _count + _sum + 4 buckets + 1 float series + 1 float series from UNKNOWN chunk
+	if len(tss) != 8 {
+		t.Fatalf("unexpected number of time series; got %d; want 8", len(tss))
+	}
+	if tss[0].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[0].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[1].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[1])
+	}
+	for _, ts := range tss[2:6] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+	if tss[6].Name != "cpu_usage" || !reflect.DeepEqual(tss[6].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[6])
+	}
+	if tss[7].Name != "memory_usage" || !reflect.DeepEqual(tss[7].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series from UNKNOWN chunk: %v", tss[7])
+	}
+}
+
+func TestParseFloatHistograms(t *testing.T) {
+	c := chunkenc.NewFloatHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	fh := testHistogram(1).ToFloat(nil)
+	if _, _, _, err := app.AppendFloatHistogram(nil, 0, 1000, fh, true); err != nil {
+		t.Fatalf("cannot append float histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_FLOAT_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse float histogram chunk: %s", err)
+	}
+	if len(hSamples) != 1 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 1", len(hSamples))
+	}
+	if hSamples[0].timestamp != 1000 {
+		t.Fatalf("unexpected timestamp; got %d; want 1000", hSamples[0].timestamp)
+	}
+	if hSamples[0].fh.Count != 10 {
+		t.Fatalf("unexpected count; got %f; want 10", hSamples[0].fh.Count)
+	}
+}

app/vmselect/main.go

@@ -369,6 +369,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -388,6 +392,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -541,7 +549,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		promscrape.WriteMetricRelabelDebug(w, r, "", nil)
 		return true
 	case "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmselect/promql/binary_op.go

@@ -164,15 +164,21 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		left := bfa.left
 		right := bfa.right
 		op := bfa.be.Op
-		switch true {
-		case metricsql.IsBinaryOpCmp(op):
-			// Do not remove empty series for comparison operations,
-			// since this may lead to missing result.
-		default:
+		isCmpOp := metricsql.IsBinaryOpCmp(op)
+		if !isCmpOp {
+			// Do not remove empty series for comparison operations, since NaN can be an
+			// explicitly present sample value. In particular, `value != NaN` is true.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
-		if len(left) == 0 || len(right) == 0 {
+		if len(left) == 0 && len(right) == 0 {
+			return nil, nil
+		}
+		if len(left) == 0 && bfa.be.FillLeft == nil {
+			return nil, nil
+		}
+		if len(right) == 0 && bfa.be.FillRight == nil {
 			return nil, nil
 		}
 		left, right, dst, err := adjustBinaryOpTags(bfa.be, left, right)
@@ -183,6 +189,16 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			logger.Panicf("BUG: len(left) must match len(right) and len(dst); got %d vs %d vs %d", len(left), len(right), len(dst))
 		}
 		isBool := bfa.be.Bool
+		fillLeft := bfa.be.FillLeft
+		fillRight := bfa.be.FillRight
+		// A NaN produced by a vector comparison denotes a filtered-out sample rather
+		// than an explicitly present NaN value. Drop it when this result is used as
+		// the right-hand side of another comparison.
+		// Note that filtered-out samples surviving as NaN inside other wrappers such as
+		// transform functions or arithmetic operations aren't detected, since such NaN
+		// is indistinguishable from an explicitly present NaN value there.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
+		dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -193,6 +209,24 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			}
 			for j, a := range leftValues {
 				b := rightValues[j]
+				leftIsNaN := math.IsNaN(a)
+				rightIsNaN := math.IsNaN(b)
+				// Both sides are NaN.
+				if leftIsNaN && rightIsNaN {
+					dstValues[j] = bf(a, b, isBool)
+					continue
+				}
+				if dropNaNRight && rightIsNaN && fillRight == nil {
+					dstValues[j] = nan
+					continue
+				}
+				// Apply the fill value when either the left or right side is NaN, but not both.
+				if leftIsNaN && fillLeft != nil {
+					a = fillLeft.N
+				}
+				if rightIsNaN && fillRight != nil {
+					b = fillRight.N
+				}
 				dstValues[j] = bf(a, b, isBool)
 			}
 		}
@@ -202,6 +236,38 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
+// isVectorComparisonExpr returns whether e is a comparison operation
+// with at least one non-scalar operand.
+func isVectorComparisonExpr(e metricsql.Expr) bool {
+	if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
+		// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
+		// the shape of the result. Subqueries are left as is, since rollup
+		// functions skip NaN values on their own.
+		e = re.Expr
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
+		return false
+	}
+	return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
+}
+
+// isScalarLikeExpr returns whether e always evaluates to a scalar value.
+func isScalarLikeExpr(e metricsql.Expr) bool {
+	switch e := e.(type) {
+	case *metricsql.NumberExpr, *metricsql.DurationExpr:
+		return true
+	case *metricsql.BinaryOpExpr:
+		return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
+	case *metricsql.FuncExpr:
+		switch strings.ToLower(e.Name) {
+		case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
+			return true
+		}
+	}
+	return false
+}
+
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {
@@ -226,7 +292,7 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		}
 	}
 
-	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
+	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} {fill|fill_left|fill_right} b`
 	var rvsLeft, rvsRight []*timeseries
 	mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
 	joinOp := strings.ToLower(be.JoinModifier.Op)
@@ -239,10 +305,27 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		// Add __name__ to groupTags if metric name must be preserved.
 		groupTags = append(groupTags[:len(groupTags):len(groupTags)], "__name__")
 	}
+	// Add missing keys from mRight to mLeft when fill_left()/fill() modifier is used
+	if be.FillLeft != nil {
+		for k := range mRight {
+			if _, ok := mLeft[k]; !ok {
+				mLeft[k] = nil
+			}
+		}
+	}
 	for k, tssLeft := range mLeft {
 		tssRight := mRight[k]
+		if len(tssLeft) == 0 {
+			if be.FillLeft == nil {
+				logger.Panicf("BUG: unexpected empty tssLeft for key %q when FillLeft is nil", k)
+			}
+			tssLeft = []*timeseries{newFillTimeseries(be, tssRight[0])}
+		}
 		if len(tssRight) == 0 {
-			continue
+			if be.FillRight == nil {
+				continue
+			}
+			tssRight = []*timeseries{newFillTimeseries(be, tssLeft[0])}
 		}
 		switch joinOp {
 		case "group_left":
@@ -287,6 +370,28 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 	return rvsLeft, rvsRight, dst, nil
 }
 
+// newFillTimeseries returns a time series filled with NaN values for the fill_left()/fill_right()/fill() modifiers.
+// These NaN values will be replaced later with the fill value if needed.
+func newFillTimeseries(be *metricsql.BinaryOpExpr, src *timeseries) *timeseries {
+	var ts timeseries
+	ts.CopyFromShallowTimestamps(src)
+	if !be.KeepMetricNames {
+		ts.MetricName.ResetMetricGroup()
+	}
+	groupTags := be.GroupModifier.Args
+	switch strings.ToLower(be.GroupModifier.Op) {
+	case "on":
+		ts.MetricName.RemoveTagsOn(groupTags)
+	default:
+		ts.MetricName.RemoveTagsIgnoring(groupTags)
+	}
+	values := ts.Values
+	for i := range values {
+		values[i] = math.NaN()
+	}
+	return &ts
+}
+
 func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
 	if len(tss) == 0 {
 		logger.Panicf("BUG: tss must contain at least one value")

app/vmselect/promql/eval.go

@@ -424,18 +424,7 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	if bf == nil {
 		return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
 	}
-	var err error
-	var tssLeft, tssRight []*timeseries
-	switch strings.ToLower(be.Op) {
-	case "and", "if":
-		// Fetch right-side series at first, since it usually contains
-		// lower number of time series for `and` and `if` operator.
-		// This should produce more specific label filters for the left side of the query.
-		// This, in turn, should reduce the time to select series for the left side of the query.
-		tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
-	default:
-		tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
-	}
+	tssLeft, tssRight, err := execBinaryOpArgs(qt, ec, be)
 	if err != nil {
 		return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
 	}
@@ -451,6 +440,29 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	return rv, nil
 }
 
+// binaryOpEvalOrder might change the order of evaluation of the left and right sides of a binary operation,
+// when there is chance to push down common label filters from exprFirst to exprSecond in the following executions.
+func binaryOpEvalOrder(be *metricsql.BinaryOpExpr) (exprFirst, exprSecond metricsql.Expr) {
+	exprFirst, exprSecond = be.Left, be.Right
+	switch strings.ToLower(be.Op) {
+	case "and", "if":
+		// For `and` and `if`, fetch the right-side series first, since it usually contains
+		// fewer time series and yields more specific filters for the left side.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	if be.FillLeft != nil && be.FillRight == nil {
+		// For `fill_left(<value>)`, the unmatched series can only come from the right side, so evaluate it first.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	return exprFirst, exprSecond
+}
+
+// canPushdownCommonFilters decides if common label filters can be pushed down from one side of a binary operation to the other.
+//
+// Common filters cannot be pushed down when:
+// - the operator is `or` or `default`;
+// - either side is an aggregation function without explicit grouping;
+// - fill(<value>) modifier is used.
 func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	switch strings.ToLower(be.Op) {
 	case "or", "default":
@@ -459,6 +471,10 @@ func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	if isAggrFuncWithoutGrouping(be.Left) || isAggrFuncWithoutGrouping(be.Right) {
 		return false
 	}
+	// Filters cannot be propagated when fill(<value>) modifier is used.
+	if be.FillLeft != nil && be.FillRight != nil {
+		return false
+	}
 	return true
 }
 
@@ -470,7 +486,15 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
 	return len(afe.Modifier.Args) == 0
 }
 
-func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+	exprFirst, exprSecond := binaryOpEvalOrder(be)
+	firstIsLeft := exprFirst == be.Left
+	sortResult := func(tssFirst, tssSecond []*timeseries) ([]*timeseries, []*timeseries, error) {
+		if firstIsLeft {
+			return tssFirst, tssSecond, nil
+		}
+		return tssSecond, tssFirst, nil
+	}
 	if canPushdownCommonFilters(be) {
 		// Execute binary operation in the following way:
 		//
@@ -512,7 +536,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		if err != nil {
 			return nil, nil, err
 		}
-		return tssFirst, tssSecond, nil
+		return sortResult(tssFirst, tssSecond)
 	}
 
 	// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
@@ -568,7 +592,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 	if errSecond != nil {
 		return nil, nil, errSecond
 	}
-	return tssFirst, tssSecond, nil
+	return sortResult(tssFirst, tssSecond)
 }
 
 func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {

app/vmselect/promql/exec_test.go

@@ -2994,6 +2994,136 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != NaN`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != 1200`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (1 > 2)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
+		// A missing sample on the left side results in NaN for every comparison
+		// operation, so no special handling is needed there.
+		t.Parallel()
+		q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
+		// Prometheus drops the timestamps filtered out by the comparison on the right.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`-1 < 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `-1 < 2`
@@ -4006,6 +4136,275 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`vector + vector fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill(0) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill() both sides NaN case`, func(t *testing.T) {
+		t.Parallel()
+		q := `(
+			label_set(time() <= 1200, "foo", "common")
+		) + fill(10) (
+			label_set(time() >= 1600, "foo", "common")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1010, 1210, nan, 1610, 1810, 2010},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_left(10) fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_right() only`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common", "extra", "l")
+			or label_set(2, "foo", "left_only", "extra", "l")
+		) + on(foo) fill(0) (
+			label_set(3, "foo", "common", "extra", "r")
+			or label_set(4, "foo", "right_only", "extra", "r")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() group_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "method", "get", "code", "500")
+			or label_set(2, "method", "get", "code", "404")
+			or label_set(3, "method", "put", "code", "501")
+		) + on(method) group_left() fill_right(0) (
+			label_set(10, "method", "get")
+		), "method", "code")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12, 12, 12, 12, 12, 12},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("404"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11, 11, 11, 11, 11, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("500"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("501"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector / vector ignoring() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(6, "method", "get", "code", "500")
+			or label_set(1, "method", "put", "code", "500")
+		) / ignoring(code) fill(0) (
+			label_set(12, "method", "get")
+			or label_set(5, "method", "post")
+			or label_set(10, "method", "put")
+		), "method")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("post"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(scalar)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, time())`

app/vmselect/vmui/assets/rolldown-runtime-CNC7AqOf.js

@@ -0,0 +1 @@
+var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(e){throw n=[e],e}},s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/assets/rolldown-runtime-Cyuzqnbw.js

@@ -1 +0,0 @@
-var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t)=>()=>(e&&(t=e(e=0)),t),s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-xYKUiOTH.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Cyuzqnbw.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-B83wxFqK.js">
+  <script type="module" crossorigin src="./assets/index-D5egN2id.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BBUnmLOr.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -30,7 +30,15 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
 		"The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
-	vmselectAddr                  = flag.String("vmselectAddr", "", "TCP address to accept connections from vmselect services")
+	maxBackfillAge = flagutil.NewRetentionDuration("maxBackfillAge", "0", "The maximum allowed age for the ingested samples with historical timestamps. "+
+		"Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. "+
+		"By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. "+
+		"This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
+	vmselectAddr = flag.String("vmselectAddr", "", "TCP address to listen for incoming connections from vmselect. "+
+		"When set, the node will be able to accept cluster-native vmselect RPC requests as if it were vmstorage. "+
+		"The tenant ID assigned to this node's data is controlled by -accountID and -projectID flags. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy")
 	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
 		"This reduces CPU usage at the cost of higher network bandwidth usage")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
@@ -146,6 +154,7 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
 	opts := storage.OpenOptions{
 		Retention:                   retentionPeriod.Duration(),
 		FutureRetention:             futureRetention.Duration(),
+		MaxBackfillAge:              maxBackfillAge.Duration(),
 		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
 		MaxHourlySeries:             getMaxHourlySeries(),
 		MaxDailySeries:              getMaxDailySeries(),
@@ -467,8 +476,10 @@ func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/inmemory"}`, tm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/small"}`, tm.SmallSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/big"}`, tm.BigSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="storage/metaindex"}`, tm.MetaindexSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/inmemory"}`, idbm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/file"}`, idbm.FileSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/metaindex"}`, idbm.MetaindexSizeBytes)
 
 	metrics.WriteCounterUint64(w, `vm_rows_received_by_storage_total`, m.RowsReceivedTotal)
 	metrics.WriteCounterUint64(w, `vm_rows_added_to_storage_total`, m.RowsAddedTotal)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.4 AS build-web-stage
+FROM golang:1.26.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/package.json

@@ -9,7 +9,7 @@
     "start": "vite",
     "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
+    "lint": "eslint --format stylish 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
     "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
@@ -23,40 +23,40 @@
     "classnames": "^2.5.1",
     "dayjs": "^1.11.21",
     "lodash.debounce": "^4.0.8",
-    "marked": "^18.0.5",
-    "preact": "^10.29.2",
-    "qs": "^6.15.2",
+    "marked": "^18.0.6",
+    "preact": "^10.29.7",
+    "qs": "^6.15.3",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.17.0",
+    "react-router-dom": "^7.18.1",
     "uplot": "^1.6.32",
-    "vite": "^8.0.16",
+    "vite": "^8.1.4",
     "web-vitals": "^5.3.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.5",
+    "@eslint/eslintrc": "^3.3.6",
     "@eslint/js": "^9.39.2",
     "@preact/preset-vite": "^2.10.5",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^25.9.2",
+    "@types/node": "^26.1.1",
     "@types/qs": "^6.15.1",
     "@types/react": "^19.2.17",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.61.0",
-    "@typescript-eslint/parser": "^8.61.0",
+    "@typescript-eslint/eslint-plugin": "^8.64.0",
+    "@typescript-eslint/parser": "^8.64.0",
     "cross-env": "^10.1.0",
     "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-unused-imports": "^4.4.1",
-    "globals": "^17.6.0",
-    "http-proxy-middleware": "^4.1.0",
+    "globals": "^17.7.0",
+    "http-proxy-middleware": "^4.2.0",
     "jsdom": "^29.1.1",
-    "postcss": "^8.5.15",
+    "postcss": "^8.5.19",
     "sass-embedded": "^1.100.0",
     "typescript": "^6.0.3",
-    "vitest": "^4.1.8"
+    "vitest": "^4.1.10"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls.tsx

@@ -1,4 +1,4 @@
-import { FC, useEffect, useRef, useState } from "preact/compat";
+import { FC, useEffect, useRef } from "preact/compat";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { getAppModeEnable } from "../../../../utils/app-mode";
 import Button from "../../../Main/Button/Button";
@@ -9,27 +9,38 @@ import classNames from "classnames";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import useBoolean from "../../../../hooks/useBoolean";
+import { getMillisecondsFromDuration } from "../../../../utils/time";
+import { useSearchParams } from "react-router";
 
-interface AutoRefreshOption {
-  seconds: number
-  title: string
-}
-
-const delayOptions: AutoRefreshOption[] = [
-  { seconds: 0, title: "Off" },
-  { seconds: 1, title: "1s" },
-  { seconds: 2, title: "2s" },
-  { seconds: 5, title: "5s" },
-  { seconds: 10, title: "10s" },
-  { seconds: 30, title: "30s" },
-  { seconds: 60, title: "1m" },
-  { seconds: 300, title: "5m" },
-  { seconds: 900, title: "15m" },
-  { seconds: 1800, title: "30m" },
-  { seconds: 3600, title: "1h" },
-  { seconds: 7200, title: "2h" }
+const delayOptions = [
+  "Off",
+  "1s",
+  "2s",
+  "5s",
+  "10s",
+  "30s",
+  "1m",
+  "5m",
+  "15m",
+  "30m",
+  "1h",
+  "2h"
 ];
 
+const DEFAULT_OPTION = delayOptions[0];
+
+const MIN_REFRESH_MS = 1000;
+const MAX_REFRESH_MS = getMillisecondsFromDuration(delayOptions[delayOptions.length - 1]);
+const REFRESH_URL_PARAM = "refresh";
+
+const durationToMs = (dur: string | null) => {
+  return dur ? getMillisecondsFromDuration(dur) : 0;
+};
+
+const isValidDelay = (ms: number) => {
+  return ms >= MIN_REFRESH_MS && ms <= MAX_REFRESH_MS;
+};
+
 interface ExecutionControlsProps {
   tooltip: string;
   useAutorefresh?: boolean;
@@ -38,12 +49,14 @@ interface ExecutionControlsProps {
 
 export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAutorefresh, closeModal }) => {
   const { isMobile } = useDeviceDetect();
+  const [searchParams, setSearchParams] = useSearchParams();
 
   const dispatch = useTimeDispatch();
   const appModeEnable = getAppModeEnable();
-  const [autoRefresh, setAutoRefresh] = useState(false);
 
-  const [selectedDelay, setSelectedDelay] = useState<AutoRefreshOption>(delayOptions[0]);
+  const rawDelay = searchParams.get(REFRESH_URL_PARAM);
+  const msDelay = durationToMs(rawDelay);
+  const selectedDelay = isValidDelay(msDelay) ? rawDelay : DEFAULT_OPTION;
 
   const {
     value: openOptions,
@@ -52,11 +65,20 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   } = useBoolean(false);
   const optionsButtonRef = useRef<HTMLDivElement>(null);
 
-  const handleChange = (d: AutoRefreshOption) => {
-    if ((autoRefresh && !d.seconds) || (!autoRefresh && d.seconds)) {
-      setAutoRefresh(prev => !prev);
-    }
-    setSelectedDelay(d);
+  const handleChange = (dur: string) => () => {
+    setSearchParams(prev => {
+      const nextParams = new URLSearchParams(prev);
+      const ms = durationToMs(dur);
+
+      if (ms) {
+        nextParams.set(REFRESH_URL_PARAM, `${dur}`);
+      } else {
+        nextParams.delete(REFRESH_URL_PARAM);
+      }
+
+      return nextParams;
+    });
+
     handleCloseOptions();
   };
 
@@ -68,23 +90,19 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   };
 
   useEffect(() => {
-    const delay = selectedDelay.seconds;
+    const ms = durationToMs(selectedDelay);
     let timer: number;
-    if (autoRefresh) {
+
+    if (useAutorefresh && isValidDelay(ms)) {
       timer = setInterval(() => {
         dispatch({ type: "RUN_QUERY" });
-      }, delay * 1000) as unknown as number;
-    } else {
-      setSelectedDelay(delayOptions[0]);
+      }, ms) as unknown as number;
     }
-    return () => {
-      timer && clearInterval(timer);
-    };
-  }, [selectedDelay, autoRefresh]);
 
-  const createHandlerChange = (d: AutoRefreshOption) => () => {
-    handleChange(d);
-  };
+    return () => {
+      clearInterval(timer);
+    };
+  }, [selectedDelay, useAutorefresh]);
 
   return (
     <>
@@ -106,7 +124,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 <span className="vm-mobile-option__icon"><RestartIcon/></span>
                 <div className="vm-mobile-option-text">
                   <span className="vm-mobile-option-text__label">Auto-refresh</span>
-                  <span className="vm-mobile-option-text__value">{selectedDelay.title}</span>
+                  <span className="vm-mobile-option-text__value">{selectedDelay}</span>
                 </div>
                 <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
               </div>
@@ -139,7 +157,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                       )}
                       onClick={toggleOpenOptions}
                     >
-                      {selectedDelay.title}
+                      {selectedDelay}
                     </Button>
                   </div>
                 </Tooltip>
@@ -187,12 +205,12 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 className={classNames({
                   "vm-list-item": true,
                   "vm-list-item_mobile": isMobile,
-                  "vm-list-item_active": d.seconds === selectedDelay.seconds
+                  "vm-list-item_active": d === selectedDelay
                 })}
-                key={d.seconds}
-                onClick={createHandlerChange(d)}
+                key={d}
+                onClick={handleChange(d)}
               >
-                {d.title}
+                {d}
               </div>
             ))}
           </div>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseAlert/index.tsx

@@ -98,7 +98,7 @@ const BaseAlert = ({ item, group }: BaseAlertProps) => {
       {!!Object.keys(item.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/index.tsx

@@ -121,7 +121,7 @@ const BaseRule = ({ item, group }: BaseRuleProps) => {
       {!!Object.keys(item?.annotations || {}).length && (
         <>
           <span className="vm-alerts-title">Annotations</span>
-          <table>
+          <table className="vm-annotations-table">
             <colgroup>
               <col className="vm-col-md"/>
               <col/>

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/HeaderNav.tsx

@@ -17,10 +17,12 @@ interface HeaderNavProps {
 const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
   const { pathname } = useLocation();
   const [activeMenu, setActiveMenu] = useState(pathname);
+  const [openMenu, setOpenMenu] = useState<string | null>(null);
   const menu = useNavigationMenu();
 
   useEffect(() => {
     setActiveMenu(pathname);
+    setOpenMenu(null);
   }, [pathname]);
 
   return (
@@ -41,6 +43,8 @@ const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
               color={color}
               background={background}
               direction={direction}
+              openMenu={openMenu}
+              setOpenMenu={setOpenMenu}
             />
           )
           : (

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/NavSubItem.tsx

@@ -1,11 +1,8 @@
-import { FC, useRef, useState } from "preact/compat";
-import { useLocation } from "react-router-dom";
+import { FC, useRef, useState, Dispatch, SetStateAction } from "preact/compat";
 import classNames from "classnames";
 import { ArrowDropDownIcon } from "../../../components/Main/Icons";
 import Popper from "../../../components/Main/Popper/Popper";
 import NavItem from "./NavItem";
-import { useEffect } from "react";
-import useBoolean from "../../../hooks/useBoolean";
 import { NavigationItem, NavigationItemType } from "../../../router/navigation";
 
 interface NavItemProps {
@@ -15,6 +12,8 @@ interface NavItemProps {
   color?: string
   background?: string
   direction?: "row" | "column"
+  openMenu: string | null,
+  setOpenMenu: Dispatch<SetStateAction<string | null>>,
 }
 
 const NavSubItem: FC<NavItemProps> = ({
@@ -23,21 +22,18 @@ const NavSubItem: FC<NavItemProps> = ({
   color,
   background,
   submenu,
-  direction = "row"
+  direction = "row",
+  openMenu,
+  setOpenMenu,
 }) => {
-  const { pathname } = useLocation();
-
   const [menuTimeout, setMenuTimeout] = useState<NodeJS.Timeout | null>(null);
   const buttonRef = useRef<HTMLDivElement>(null);
 
-  const {
-    value: openSubmenu,
-    setFalse: handleCloseSubmenu,
-    setTrue: setOpenSubmenu,
-  } = useBoolean(false);
+  const openSubmenu = openMenu === label;
+  const handleCloseSubmenu = () => setOpenMenu(prev => (prev === label ? null : prev));
 
   const handleOpenSubmenu = () => {
-    if (direction === "row" || !openSubmenu) setOpenSubmenu();
+    if (direction === "row" || !openSubmenu) setOpenMenu(label);
     if (direction === "column" && openSubmenu) handleCloseSubmenu();
     if (direction === "row" && menuTimeout) clearTimeout(menuTimeout);
   };
@@ -52,10 +48,6 @@ const NavSubItem: FC<NavItemProps> = ({
     if (menuTimeout) clearTimeout(menuTimeout);
   };
 
-  useEffect(() => {
-    handleCloseSubmenu();
-  }, [pathname]);
-
   return (
     <div
       className={classNames({

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -32,7 +32,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   const match = searchParams.get("match");
   const focusLabel = searchParams.get("focusLabel");
   const isMetric = /__name__/.test(match || "");
-
+  const showMetricNameStats = !(match || focusLabel);
   const progress = totalSeries / totalSeriesAll * 100;
   const diff = totalSeries - totalSeriesPrev;
   const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
@@ -56,7 +56,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
     },
   ].filter(t => t.display);
 
-  if (!totals.length) {
+  if (!totals.length && !showMetricNameStats) {
     return null;
   }
 
@@ -75,7 +75,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           <h4 className="vm-cardinality-totals-card__title">
             {info && (
               <Tooltip title={<p className="vm-cardinality-totals-card__tooltip">{info}</p>}>
-                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon/></div>
+                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon /></div>
               </Tooltip>
             )}
             {title}
@@ -99,7 +99,10 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           )}
         </div>
       ))}
-      <CardinalityMetricNameStats metricNameStats={metricNameStats}/>
+      {
+        showMetricNameStats &&
+        <CardinalityMetricNameStats metricNameStats={metricNameStats} />
+      }
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreAlerts/style.scss

@@ -62,6 +62,13 @@
     padding-right: 40px;
   }
 
+  .vm-annotations-table {
+    tbody > tr > td {
+      vertical-align: top;
+      white-space: pre-line;
+    }
+  }
+
   display: flex;
   flex-direction: column;
   align-items: stretch;

app/vmui/packages/vmui/src/pages/Relabel/index.tsx

@@ -72,7 +72,7 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header-configs">
           <TextField
             type="textarea"
-            label="Relabel configs"
+            label="Config"
             value={config}
             autofocus
             onChange={handleChangeConfig}
@@ -82,8 +82,9 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header__labels">
           <TextField
             type="textarea"
-            label="Labels"
+            label="A time series"
             value={labels}
+            placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}"
             onChange={handleChangeLabels}
             onEnter={handleRunQuery}
           />
@@ -97,22 +98,22 @@ const Relabel: FC = () => {
             <Button
               variant="text"
               color="gray"
-              startIcon={<InfoIcon/>}
+              startIcon={<WikiIcon/>}
             >
               Relabeling cookbook
             </Button>
           </a>
           <a
             target="_blank"
-            href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
+            href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages"
             rel="help noreferrer"
           >
             <Button
               variant="text"
               color="gray"
-              startIcon={<WikiIcon/>}
+              startIcon={<InfoIcon/>}
             >
-              Documentation
+              Relabeling Stages
             </Button>
           </a>
           <Button

app/vmui/packages/vmui/src/utils/time.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect } from "vitest";
 import dayjs from "dayjs";
-import { getNanoTimestamp, parseSupportedDuration } from "./time";
+import { getMillisecondsFromDuration, getNanoTimestamp, parseSupportedDuration } from "./time";
 
 describe("Time utils", () => {
   describe("getNanoTimestamp", () => {
@@ -82,4 +82,19 @@ describe("Time utils", () => {
       expect(parseSupportedDuration("   ")).toBeUndefined();
     });
   });
+
+  describe("getMillisecondsFromDuration", () => {
+    it("should convert valid durations to milliseconds", () => {
+      expect(getMillisecondsFromDuration("1s")).toBe(1000);
+      expect(getMillisecondsFromDuration("1.5s")).toBe(1500);
+      expect(getMillisecondsFromDuration("1m30s")).toBe(90000);
+      expect(getMillisecondsFromDuration("1h 30m")).toBe(5400000);
+      expect(getMillisecondsFromDuration("500ms")).toBe(500);
+    });
+
+    it("should return zero when duration cannot be parsed", () => {
+      expect(getMillisecondsFromDuration("garbage")).toBe(0);
+      expect(getMillisecondsFromDuration("")).toBe(0);
+    });
+  });
 });

app/vmui/packages/vmui/src/utils/time.ts

@@ -100,6 +100,10 @@ export const getSecondsFromDuration = (dur: string) => {
   return dayjs.duration(durObject).asSeconds();
 };
 
+export const getMillisecondsFromDuration = (dur: string): number => {
+  return getSecondsFromDuration(dur) * 1000;
+};
+
 const instantQueryViews = [DisplayType.table, DisplayType.code];
 export const getStepFromDuration = (dur: number, histogram?: boolean, displayType?: DisplayType): string => {
   if (displayType && instantQueryViews.includes(displayType)) return roundStep(dur);
@@ -276,4 +280,3 @@ export const getNanoTimestamp = (dateStr: string): bigint => {
   // Return the full timestamp in nanoseconds as a BigInt
   return BigInt(baseMs) * 1000000n + BigInt(extraNano);
 };
-

apptest/tests/max_backfill_age_test.go

@@ -0,0 +1,222 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestSingleMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle", []string{
+				"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
+				"-retentionPeriod=" + retentionPeriod,
+				"-maxBackfillAge=" + maxBackfillAge,
+			})
+		},
+		stop: func() {
+			tc.StopApp("vmsingle")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+func TestClusterMaxBackfillAge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	opts := maxBackfillAgeOpts{
+		start: func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&apptest.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage1"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				Vmstorage2Instance: "vmstorage2",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage2"),
+					"-retentionPeriod=" + retentionPeriod,
+					"-maxBackfillAge=" + maxBackfillAge,
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags:    []string{},
+			})
+		},
+		stop: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1")
+			tc.StopApp("vmstorage2")
+		},
+	}
+
+	testMaxBackfillAge(tc, opts)
+}
+
+type maxBackfillAgeOpts struct {
+	start func(retentionPeriod, maxBackfillAge string) apptest.PrometheusWriteQuerier
+	stop  func()
+}
+
+func testMaxBackfillAge(tc *apptest.TestCase, opts maxBackfillAgeOpts) {
+	t := tc.T()
+
+	assertSeries := func(app apptest.PrometheusQuerier, prefix string, start, end int64, want []map[string]string) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, apptest.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &apptest.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want,
+			},
+			FailNow: true,
+		})
+	}
+
+	assertQueryResults := func(app apptest.PrometheusQuerier, prefix string, start, end, step int64, want []*apptest.QueryResult) {
+		t.Helper()
+
+		query := fmt.Sprintf(`{__name__=~"metric_%s.*"}`, prefix)
+		tc.Assert(&apptest.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
+					Start:       fmt.Sprintf("%d", start),
+					End:         fmt.Sprintf("%d", end),
+					Step:        fmt.Sprintf("%dms", step),
+					MaxLookback: fmt.Sprintf("%dms", step-1),
+					NoCache:     "1",
+				})
+			},
+			Want: &apptest.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &apptest.QueryData{
+					ResultType: "matrix",
+					Result:     want,
+				},
+			},
+			FailNow: true,
+		})
+	}
+
+	const numMetrics = 1000
+	now := time.Now().UTC()
+	var start, end, step int64
+	emptySeries := []map[string]string{}
+	emptyQueryResults := []*apptest.QueryResult{}
+
+	// Start sut with the same -retentionPeriod and -maxBackfillAge.
+	sut := opts.start("1y", "1y")
+
+	// Verify that samples older than the retention period are rejected.
+	start = now.Add(-365 * 24 * time.Hour).Add(-time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	outsideRetention := genMaxBackfillAgeData("outside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, outsideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "outside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "outside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that samples within the retention period are accepted and
+	// searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention := genMaxBackfillAgeData("inside_retention", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Restart sut with -maxBackfillAge shorter than the -retentionPeriod.
+	opts.stop()
+	sut = opts.start("1y", "6M")
+
+	// Verify that new samples older than max backfill age but still within the
+	// retention period are rejected but existing samples are still searcheable.
+	start = now.Add(-365 * 24 * time.Hour).Add(time.Hour).UnixMilli()
+	end = now.Add(-365 * 24 * time.Hour).Add(2 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideRetention2 := genMaxBackfillAgeData("inside_retention2", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideRetention2.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention2", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention2", start, end, step, emptyQueryResults)
+	assertSeries(sut, "inside_retention", start, end, insideRetention.wantSeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, insideRetention.wantQueryResults)
+
+	// Verify that the metrics that are outside the backfill window can still
+	// be deleted.
+	sut.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*inside_retention.*"}`, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_retention", start, end, emptySeries)
+	assertQueryResults(sut, "inside_retention", start, end, step, emptyQueryResults)
+
+	// Verify that the samples that are within the backfill window are accepted
+	// and searchable.
+	start = now.Add(-180 * 24 * time.Hour).UnixMilli()
+	end = now.Add(-180 * 24 * time.Hour).Add(1 * time.Hour).UnixMilli()
+	step = (end - start) / numMetrics
+	insideMaxBackfillAge := genMaxBackfillAgeData("inside_max_backfill_age", numMetrics, start, step)
+	sut.PrometheusAPIV1ImportPrometheus(t, insideMaxBackfillAge.samples, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, "inside_max_backfill_age", start, end, insideMaxBackfillAge.wantSeries)
+	assertQueryResults(sut, "inside_max_backfill_age", start, end, step, insideMaxBackfillAge.wantQueryResults)
+
+	opts.stop()
+}
+
+type maxBackfillAgeData struct {
+	samples          []string
+	wantSeries       []map[string]string
+	wantQueryResults []*apptest.QueryResult
+}
+
+func genMaxBackfillAgeData(prefix string, numMetrics, start, step int64) maxBackfillAgeData {
+	samples := make([]string, numMetrics)
+	wantSeries := make([]map[string]string, numMetrics)
+	wantQueryResults := make([]*apptest.QueryResult, numMetrics)
+	for i := range numMetrics {
+		metricName := fmt.Sprintf("metric_%s_%04d", prefix, i)
+		labelName := fmt.Sprintf("label_%s_%04d", prefix, i)
+		labelValue := fmt.Sprintf("value_%s_%04d", prefix, i)
+		value := i
+		timestamp := start + i*step
+		samples[i] = fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
+		wantSeries[i] = map[string]string{
+			"__name__": metricName,
+			labelName:  "value",
+			"label":    labelValue,
+		}
+		wantQueryResults[i] = &apptest.QueryResult{
+			Metric: map[string]string{
+				"__name__": metricName,
+				labelName:  "value",
+				"label":    labelValue,
+			},
+			Samples: []*apptest.Sample{{Timestamp: timestamp, Value: float64(value)}},
+		}
+	}
+	return maxBackfillAgeData{samples, wantSeries, wantQueryResults}
+}

apptest/tests/prometheus_mock_storage.go

@@ -59,7 +59,7 @@ func (ms *PrometheusMockStorage) Read(_ context.Context, query *prompb.Query, so
 		}
 
 		if !notMatch {
-			q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples})
+			q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples, Histograms: s.Histograms})
 		}
 	}
 

apptest/tests/remote_read_server.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
@@ -86,10 +87,17 @@ func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
 						samples = append(samples, sample)
 					}
 				}
+				var histograms []prompb.Histogram
+				for _, h := range s.Histograms {
+					if h.Timestamp >= startTs && h.Timestamp < endTs {
+						histograms = append(histograms, h)
+					}
+				}
 				var series prompb.TimeSeries
-				if len(samples) > 0 {
+				if len(samples) > 0 || len(histograms) > 0 {
 					series.Labels = s.Labels
 					series.Samples = samples
+					series.Histograms = histograms
 				}
 				ts[i] = &series
 			}
@@ -317,6 +325,37 @@ func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64)
 	return samples
 }
 
+// GenerateRemoteReadHistogramSeries generates a remote read series
+// with native histogram samples within the given time range.
+func GenerateRemoteReadHistogramSeries(start, end, numOfSamples int64) []*prompb.TimeSeries {
+	timeSeries := &prompb.TimeSeries{
+		Labels: []prompb.Label{
+			{Name: labels.MetricName, Value: "vm_histogram_metric"},
+			{Name: "job", Value: "0"},
+		},
+	}
+
+	delta := (end - start) / numOfSamples
+	mul := int64(0)
+	for t := start; t != end; t += delta {
+		mul++
+		h := &histogram.Histogram{
+			Schema:          0,
+			Count:           uint64(10 * mul),
+			Sum:             25.5 * float64(mul),
+			ZeroThreshold:   0.001,
+			ZeroCount:       uint64(2 * mul),
+			PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}},
+			PositiveBuckets: []int64{mul, 2 * mul},
+			NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+			NegativeBuckets: []int64{4 * mul},
+		}
+		timeSeries.Histograms = append(timeSeries.Histograms, prompb.FromIntHistogram(t*1000, h))
+	}
+
+	return []*prompb.TimeSeries{timeSeries}
+}
+
 func labelsToLabelsProto(ls labels.Labels) []prompb.Label {
 	result := make([]prompb.Label, 0, ls.Len())
 	ls.Range(func(l labels.Label) {

apptest/tests/vmctl_remote_read_mogration_test.go

@@ -75,6 +75,118 @@ func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
 	testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
 }
 
+func TestSingleVmctlRemoteReadNativeHistograms(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadServer, vmctlFlags)
+}
+
+func TestSingleVmctlRemoteReadStreamNativeHistograms(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--remote-read-use-stream=true`,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadStreamServer, vmctlFlags)
+}
+
+// testRemoteReadNativeHistograms verifies that native histograms are migrated
+// as _count, _sum and _bucket series with vmrange labels in the same way
+// as VictoriaMetrics converts native histograms received via Prometheus remote write protocol.
+func testRemoteReadNativeHistograms(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T, series []*prompb.TimeSeries) *RemoteReadServer, vmctlFlags []string) {
+	t := tc.T()
+	t.Helper()
+
+	series := GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2)
+
+	rrs := newRemoteReadServer(t, series)
+	defer rrs.Close()
+
+	vmctlFlags = append(vmctlFlags, `--remote-read-src-addr=`+rrs.HTTPAddr())
+	tc.MustStartVmctl("vmctl", vmctlFlags)
+
+	sut.ForceFlush(t)
+
+	tc.Assert(&apptest.AssertOptions{
+		Retries: 300,
+		Msg:     `unexpected native histogram metrics stored on vmsingle via the prometheus protocol`,
+		Got: func() any {
+			got := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
+				Start: "2025-06-11T15:31:10Z",
+				End:   "2025-06-11T15:32:20Z",
+			})
+			got.Sort()
+			return got.Data.Result
+		},
+		Want: expectedNativeHistogramQueryResult(),
+		CmpOpts: []cmp.Option{
+			cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		},
+	})
+}
+
+// expectedNativeHistogramQueryResult returns the series expected to be stored in VictoriaMetrics
+// after migrating the series generated by GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2).
+func expectedNativeHistogramQueryResult() []*apptest.QueryResult {
+	metric := func(name, vmrange string) map[string]string {
+		m := map[string]string{
+			"__name__": name,
+			"job":      "0",
+		}
+		if vmrange != "" {
+			m["vmrange"] = vmrange
+		}
+		return m
+	}
+	samples := func(v1, v2 float64) []*apptest.Sample {
+		return []*apptest.Sample{
+			{Timestamp: 1749655870000, Value: v1},
+			{Timestamp: 1749655875000, Value: v2},
+		}
+	}
+	resp := &apptest.PrometheusAPIV1QueryResponse{
+		Data: &apptest.QueryData{
+			Result: []*apptest.QueryResult{
+				{Metric: metric("vm_histogram_metric_count", ""), Samples: samples(10, 20)},
+				{Metric: metric("vm_histogram_metric_sum", ""), Samples: samples(25.5, 51)},
+				{Metric: metric("vm_histogram_metric_bucket", "-1.000e+00...-5.000e-01"), Samples: samples(4, 8)},
+				{Metric: metric("vm_histogram_metric_bucket", "-1.000e-03...1.000e-03"), Samples: samples(2, 4)},
+				{Metric: metric("vm_histogram_metric_bucket", "5.000e-01...1.000e+00"), Samples: samples(1, 2)},
+				{Metric: metric("vm_histogram_metric_bucket", "1.000e+00...2.000e+00"), Samples: samples(3, 6)},
+			},
+		},
+	}
+	// sort in the same way as the exported result
+	resp.Sort()
+	return resp.Data.Result
+}
+
 func testRemoteReadProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
 	t := tc.T()
 	t.Helper()

dashboards/victoriametrics-cluster.json

@@ -5136,6 +5136,110 @@
           ],
           "title": "Major page faults rate  ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 305
+          },
+          "id": 230,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

dashboards/victoriametrics.json

@@ -5181,6 +5181,110 @@
           ],
           "title": "Major page faults rate",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 81
+          },
+          "id": 159,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
@@ -8742,4 +8846,4 @@
   "uid": "wNf0q_kZk",
   "version": 1,
   "weekStart": ""
-}
+}

dashboards/vm/victoriametrics-cluster.json

@@ -5137,6 +5137,110 @@
           ],
           "title": "Major page faults rate  ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 305
+          },
+          "id": 230,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

dashboards/vm/victoriametrics.json

@@ -5182,6 +5182,110 @@
           ],
           "title": "Major page faults rate",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 81
+          },
+          "id": 159,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
@@ -8743,4 +8847,4 @@
   "uid": "wNf0q_kZk_vm",
   "version": 1,
   "weekStart": ""
-}
+}

dashboards/vm/vmagent.json

@@ -4581,6 +4581,110 @@
           ],
           "title": "Rows ignored for last 1h ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 189
+          },
+          "id": 171,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

dashboards/vmagent.json

@@ -4580,6 +4580,110 @@
           ],
           "title": "Rows ignored for last 1h ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 189
+          },
+          "id": 171,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

deployment/docker/Makefile

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.24.1
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.24.1
 
-GO_BUILDER_IMAGE := golang:1.26.4
+GO_BUILDER_IMAGE := golang:1.26.5
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/README.md

@@ -163,6 +163,8 @@ The list of alerting rules is the following:
   alerting rules related to [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) component;
 * [alerts-vmanomaly.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmanomaly.yml):
   alerting rules related to [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/);
+* [alerts-vmbackupmanager.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml):
+  alerting rules related to [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) component;
 
 Please, also see [how to monitor VictoriaMetrics installations](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring).
 ## Troubleshooting

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.148.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.146.0-cluster
+    image: victoriametrics/vmstorage:v1.148.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.146.0-cluster
+    image: victoriametrics/vminsert:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.146.0-cluster
+    image: victoriametrics/vmselect:v1.148.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.146.0
+    image: victoriametrics/vmauth:v1.148.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - "vmauth"
     ports:
@@ -129,6 +129,7 @@ services:
       - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
       - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
       - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
+      - ./rules/alerts-vmbackupmanager.yml:/etc/alerts/alerts-vmbackupmanager.yml
     command:
       - "--datasource.url=http://vmauth:8427/select/0/prometheus"
       - "--datasource.basicAuth.username=foo"

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"
@@ -70,6 +70,7 @@ services:
       - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
       - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
       - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
+      - ./rules/alerts-vmbackupmanager.yml:/etc/alerts/alerts-vmbackupmanager.yml
     command:
       - "--datasource.url=http://victoriametrics:8428/"
       - "--remoteRead.url=http://victoriametrics:8428/"

deployment/docker/rules/alerts-vmagent.yml

@@ -10,7 +10,7 @@ groups:
     concurrency: 2
     rules:
       - alert: PersistentQueueIsDroppingData
-        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path) > 0
+        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path,name) > 0
         for: 10m
         labels:
           severity: critical
@@ -201,4 +201,4 @@ groups:
         annotations:
           summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours."
           description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
-            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
+            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."

deployment/docker/rules/alerts-vmanomaly.yml

@@ -30,6 +30,35 @@ groups:
           summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
           description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5m"
 
+      - alert: SchedulerWorkerDown
+        expr: >
+          (vmanomaly_scheduler_alive{job=~".*vmanomaly.*"} == 0)
+          and on (job, instance, scheduler_alias, preset)
+          (sum(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"})
+          by (job, instance, scheduler_alias, preset) > 0)
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Scheduler {{ $labels.scheduler_alias }} is down in {{ $labels.job }} ({{ $labels.instance }})"
+          description: |
+            Periodic scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} has remained down for more than 2 minutes
+            after entering automatic recovery. The restart policy may be exhausted.
+            Check vmanomaly logs and the vmanomaly_scheduler_restarts_total metric for restart failures.
+
+      - alert: FrequentSchedulerRestarts
+        expr: >
+          sum(increase(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"}[15m]))
+          by (job, instance, scheduler_alias, preset) > 2
+        labels:
+          severity: warning
+        annotations:
+          summary: "Scheduler {{ $labels.scheduler_alias }} is restarting frequently in {{ $labels.job }} ({{ $labels.instance }})"
+          description: |
+            Scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} had more than two restart attempts
+            during the last 15 minutes. The service may be self-healing, but repeated restarts indicate an unstable
+            scheduler, configuration, datasource, or runtime dependency. Check vmanomaly logs for the original failure.
+
       # default value of 900 Should be changed to the scrape_interval for pull metrics. For push metrics this should be the lowest fit_every or infer_every in your vmanomaly config.
       - alert: NoSelfMonitoringMetrics
         expr: >

deployment/docker/rules/alerts-vmbackupmanager.yml

@@ -0,0 +1,110 @@
+# File contains default list of alerts for vmbackupmanager.
+# The alerts below are just recommendations and may require some updates
+# and threshold calibration according to every specific setup.
+groups:
+  - name: vmbackupmanager
+    rules:
+      - alert: NoLatestBackupWithinLastDay
+        expr: |
+          (
+            vm_backup_last_success_at{type="latest"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="latest"} > 0
+            and time() - vm_backup_last_success_at{type="latest"} > 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "latest backup hasn't completed successfully for more than 1 day on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed the latest backup for more than 1 day.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoHourlyBackupWithinLastDay
+        expr: |
+          (
+            vm_backup_last_success_at{type="hourly"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="hourly"} > 0
+            and time() - vm_backup_last_success_at{type="hourly"} > 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "hourly backup hasn't completed successfully for more than 1 day on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed an hourly backup for more than 1 day.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoDailyBackupWithinLast3Days
+        expr: |
+          (
+            vm_backup_last_success_at{type="daily"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 3 * 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="daily"} > 0
+            and time() - vm_backup_last_success_at{type="daily"} > 3 * 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "daily backup hasn't completed successfully for more than 3 days on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed a daily backup for more than 3 days.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoWeeklyBackupWithinLast14Days
+        expr: |
+          (
+            vm_backup_last_success_at{type="weekly"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 14 * 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="weekly"} > 0
+            and time() - vm_backup_last_success_at{type="weekly"} > 14 * 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "weekly backup hasn't completed successfully for more than 14 days on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed a weekly backup for more than 14 days.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoMonthlyBackupWithinLast62Days
+        expr: |
+          (
+            vm_backup_last_success_at{type="monthly"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 62 * 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="monthly"} > 0
+            and time() - vm_backup_last_success_at{type="monthly"} > 62 * 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "monthly backup hasn't completed successfully for more than 62 days on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed a monthly backup for more than 62 days.
+            Check error logs on vmbackupmanager for root cause.

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.146.0
+    image: victoriametrics/vmagent:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.146.0
+    image: victoriametrics/victoria-metrics:v1.148.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.146.0
+    image: victoriametrics/vmalert:v1.148.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.29.7
+    image: victoriametrics/vmanomaly:v1.30.0
     depends_on:
       - "victoriametrics"
     ports:

deployment/docker/vmanomaly/vmanomaly-integration/vmanomaly_config.yml

@@ -1,16 +1,18 @@
 schedulers:
   periodic:
     infer_every: "1m"
-    fit_every: "1h"
-    fit_window: "2d" # 2d-14d based on the presence of weekly seasonality in your data
+    fit_every: "100w" # the online model keeps learning during inference
+    fit_window: "2w"
 
 models:
-  prophet:
-    class: "prophet"
-    args:
-      interval_width: 0.98
-      weekly_seasonality: False  # comment it if your data has weekly seasonality
-      yearly_seasonality: False
+  temporal_envelope:
+    class: "temporal_envelope"
+    queries: "all"
+    schedulers: "all"
+    seasonalities: ["hod_smooth", "dow_smooth"]
+    alpha: 0.005
+    loss_reactivity: 5
+    iqr_threshold: 2
 
 reader:
   datasource_url: "http://victoriametrics:8428/"
@@ -26,4 +28,4 @@ writer:
 monitoring:
   pull: # Enable /metrics endpoint.
     addr: "0.0.0.0"
-    port: 8490
+    port: 8490

docs/ai-tools/README.md

@@ -69,18 +69,21 @@ See more details at [VictoriaMetrics/mcp-victoriatraces](https://github.com/Vict
 REST API and documentation for AI-assisted anomaly detection, model management, and observability insights.
 
 Capabilities include:
-- Health Monitoring: Check `vmanomaly` server health and build information
-- Model Management: List, validate, and configure anomaly detection models (like `zscore_online`, `prophet`, and more)
-- Configuration Generation: Generate complete `vmanomaly` YAML configurations
-- Alert Rule Generation: Generate [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules) based on [anomaly score metrics](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) to simplify alerting setup
-- Documentation Search: Full-text search across embedded `vmanomaly` documentation with fuzzy matching
+- Check `vmanomaly` health, build information, compatibility, and self-monitoring metrics
+- Inspect model schemas and validate model or complete service configurations
+- Profile sampled query results for trends, seasonalities, changepoints, gaps, and intermittent behavior
+- Run asynchronous autotune tasks and turn their results into data-driven model recommendations
+- Generate complete `vmanomaly` YAML configurations and [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules)
+- Search embedded `vmanomaly` documentation with fuzzy matching
 
 See more details at [VictoriaMetrics/mcp-vmanomaly](https://github.com/VictoriaMetrics/mcp-vmanomaly).
+# vmanomaly UI Copilot
+The vmanomaly UI includes an [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) that can assist users with anomaly detection tasks, model configuration, and troubleshooting, changing the UI state based on user queries and providing actionable suggestions through automated data profiling and validation. The AI Copilot is powered by respective [MCP Server](#vmanomaly-mcp-server) and [Agent Skills](#agent-skills), enabling it to understand the context of the user's actions and provide relevant guidance.
 
 # Agent Skills
 
-[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate, 
-and troubleshoot VictoriaMetrics observability components, including metrics, logs, and traces.
+[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate,
+and troubleshoot VictoriaMetrics observability components, including metrics, logs, traces, and [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/).
 
 These skills provide predefined workflows and capabilities such as:
 * Query metrics, logs, traces and alerts
@@ -89,6 +92,9 @@ These skills provide predefined workflows and capabilities such as:
 * Cardinality optimization 
 * Unused metric detection
 * Stream aggregation configuration
+* Build validated `vmanomaly` configurations from measured time-series characteristics (e.g., seasonality, changepoints, trends)
+* Query and operate the `vmanomaly` API
+* Review existing anomaly detection configurations against real data and identify false-positive or model-data fit issues
 
 To install the available skills for AI agents, run:
 ```sh
@@ -108,4 +114,4 @@ AI code assistants like Claude Code, OpenAI Codex, Gemini CLI, Qwen Code, and Op
 helps to monitor cost usage, analytics, performance, compliance and improves troubleshooting experience. All major
 AI coding tools support OpenTelemetry and can be easily integrated into VictoriaMetrics Observability Stack.
 Please see more details in [Vibe coding tools observability with VictoriaMetrics Stack and OpenTelemetry
-](https://victoriametrics.com/blog/vibe-coding-observability/).
+](https://victoriametrics.com/blog/vibe-coding-observability/).

docs/ai-tools/_index.md

@@ -9,6 +9,7 @@ tags:
   - metrics
   - logs
   - traces
+  - anomaly detection
   - AI
   - AI integration
   - agent

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,31 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+{{% collapse name="2026" open=true %}}
+
+## v1.30.0
+Released: 2026-07-23
+
+- FEATURE: Added univariate and multivariate online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) models for complex time-series profiles. They combine robust causal trend, compact calendar and learned holiday behavior, changepoint adaptation, residual prediction intervals, future-horizon forecasts, and optional cross-series dependency detection in bounded state.
+
+- FEATURE: Added asynchronous [`/api/v1/autotune/tasks`](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow) endpoints and improved unsupervised tuning with model-capability-aware objectives, optional causal exact validation for online models, constrained alert-volume selection, fitted-state complexity tie-breaking, and frozen model-specific parameters.
+
+- FEATURE: Added `/api/v1/timeseries/characteristics` for bounded analysis of trend, calendar seasonality, changepoints, gaps, and intermittent or spiky behavior across sampled query results. This improves automated model selection and search-space suggestions in both agentic workflows and [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance)-backed [UI](https://docs.victoriametrics.com/anomaly-detection/ui/) experiments.
+
+- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v172) to [v1.8.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v180). Notable mentions are model and query configuration, query prettification, fullscreen charts, out-of-date results alert and improved [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) suggestion and cancellation behavior.
+
+- FEATURE: Made AI-assisted configuration more reliable with reusable [vmanomaly workflow skills](https://github.com/VictoriaMetrics/skills), data-aware model suggestions, protocol-safe cancellation and synchronized query, model, and anomaly-setting updates in [UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
+
+- IMPROVEMENT: Fitted history can be retained as a stronger prior through `history_strength` argument, requiring a reduced number of observations (say 2 weeks for weekly seasonality) to be effective if the history window is a good indicator of "normal" data patterns. Models supported: `mad_online`, `zscore_online`, `quantile_online`.
+
+- IMPROVEMENT: Improved bounded datasource operation with optional ad-hoc series limits, stale-series lookback caps, and propagation of configured per-query sampling periods to models and autotune trials.
+
+- IMPROVEMENT: Added `reader.fetch_timeout` and `reader.processing_timeout` for [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [`VLogsReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#victorialogs-reader), allowing datasource reads and post-fetch processing to be tuned *independently* while preserving `reader.timeout` as the backward-compatible default for both phases.
+
+- BUGFIX: Fixed [backtesting runs](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) across multiple fit cycles and for auto-tuned online wrappers, preventing duplicate or missing predictions while retaining causal model updates and restoration of compatible legacy auto-tuned state.
+
+- BUGFIX: Prevented service shutdown after a [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) worker dies by adding bounded restart attempts with backoff and exposing restart health through [startup metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics).
+
 ## v1.29.7
 Released: 2026-06-25
 
@@ -124,6 +149,10 @@ Released: 2026-01-12
 
 - BUGFIX: Restored expected behavior when `fit_every` equals `infer_every` in [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) - now full data range `fit_window` is fetched for model trainings instead of a  last point from that interval.
 
+{{% /collapse %}}
+
+{{% collapse name="2025" %}}
+
 ## v1.28.3
 Released: 2025-12-17
 
@@ -395,6 +424,10 @@ Released: 2025-01-20
 - BUGFIX: Now [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) properly handles the cases where the number of queries processed in parallel (up to `reader.queries` cardinality) exceeds the default limit of 10 HTTP(S) connections, preventing potential data loss from discarded queries. The pool limit will automatically adjust to match `reader.queries` cardinality.
 - BUGFIX: Corrected the construction of write endpoints for cluster VictoriaMetrics `url`s (`tenant_id` arg is set) in `monitoring.push` [section configurations](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters).
 
+{{% /collapse %}}
+
+{{% collapse name="2024" %}}
+
 ## v1.18.8
 Released: 2024-12-03
 
@@ -709,6 +742,10 @@ Released: 2024-01-15
 - IMPROVEMENT: Don't check /health endpoint, check the real /query_range or /import endpoints directly. Users kept getting problems with /health.
 - DEPRECATION: "health_path" param is deprecated and doesn't do anything in config ([reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), [monitoring.push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters)).
 
+{{% /collapse %}}
+
+{{% collapse name="2023" %}}
+
 
 ## v1.7.2
 Released: 2023-12-21
@@ -803,6 +840,12 @@ Released: 2023-01-23
 Released: 2023-01-06
 - BUGFIX: prophet model incorrectly predicted two points in case of only one
 
+{{% /collapse %}}
+
+{{% collapse name="2022" %}}
+
 ## v1.0.0-beta
 Released: 2022-12-08
 - First public release is available
+
+{{% /collapse %}}

docs/anomaly-detection/FAQ.md

@@ -118,34 +118,17 @@ To visualize and interact with both [self-monitoring metrics](https://docs.victo
 - {{% available_from "v1.26.0" anomaly %}} For rapid exploration of how different models, their configurations and included domain knowledge impacts the results of anomaly detection, use the built-in [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 
-## Is vmanomaly stateful?
-By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
-
-Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
-
-## Config hot-reloading
-
-`vmanomaly` supports [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to automatically reload configurations on config files changes. It can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) and allows for configuration updates without explicit service restarts.
-
-## Environment variables
-
-`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
-
-## Deploying vmanomaly
-
-`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
-
-## Migration
-
-For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
-
 ## Choosing the right model for vmanomaly
 
-> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on selecting the best model and its configuration for your use case. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
+Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect:
 
-Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet).
+- Use [Online MAD](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) for simple, mostly stationary data with no-to-slow trend, when robustness to outliers is important.
+- Use [Online Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) for simple, light-tailed data where standard-deviation units are meaningful.
+- Use [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} for complex data with trends, calendar patterns, holidays, or persistent shifts. It is the preferred *online* alternative to Prophet (which will be deprecated in the future releases).
+- Use multivariate Temporal Envelope when normal relationships between aligned metrics matter. This should replace [Isolation Forest](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate) used in previous versions of `vmanomaly`, which will be deprecated in future releases.
+- Use [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) when Prophet-specific decomposition outputs, or offline batch behavior are required. Consider using Temporal Envelope instead, as it is more efficient and provides better results in most cases.
 
-Also, there is an option to auto-tune the most important (hyper)parameters of selected model class {{% available_from "v1.12.0" anomaly %}}, find [the details here](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned).
+There is also an option to auto-tune the most important parameters of a selected model class {{% available_from "v1.12.0" anomaly %}}. {{% available_from "v1.30.0" anomaly %}} The asynchronous autotune API can first profile a bounded sample through `/api/v1/timeseries/characteristics`, then tune a shared concrete configuration through `/api/v1/autotune/tasks`. See the [autotune workflow](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow).
 
 Please refer to [respective blogpost on anomaly types and alerting heuristics](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/) for more details.
 
@@ -153,9 +136,6 @@ Still not 100% sure what to use? We are [here to help](https://docs.victoriametr
 
 ## Incorporating domain knowledge
 
-> [!TIP]
-> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on incorporating domain knowledge into your anomaly detection models. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
-
 Anomaly detection models can significantly improve when incorporating business-specific assumptions about the data and what constitutes an anomaly. `vmanomaly` supports various [business-side configuration parameters](https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args) across all built-in models to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** and **align model behavior with business needs**, for example:
 
 - **Setting `detection_direction`** - use [`detection_direction`](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) to specify whether anomalies occur **above or below expectations**:
@@ -211,39 +191,15 @@ models:
     provide_series: ['anomaly_score', 'y', 'yhat', 'yhat_lower', 'yhat_upper']
 ```
 
-## Alert generation in vmanomaly
-While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
+## Can AI help configure vmanomaly?
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
+Yes. The available tools serve different workflows:
 
-Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
+- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
+- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
+- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
 
-```yaml
-groups:
-- name: VMAnomalyAlerts
-  interval: 60s
-  rules:
-  - alert: HighAnomalyScore
-    expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
-    for: 5m  # adjust to your needs based on data frequency and alerting policies
-    labels:
-      severity: warning
-      query_alias: explore
-      model_alias: default
-      scheduler_alias: periodic
-      preset: ui
-    annotations:
-      summary: High anomaly score detected.
-      description: Anomaly score exceeded threshold ({{ $value }}) for more than
-        {{ $for }} for query {{ $labels.for }}.
-```
-
-> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
-
-> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
-
-## Preventing alert fatigue
-Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
+Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
 
 ## How to backtest particular configuration on historical data?
 
@@ -298,7 +254,7 @@ Configuration above will produce N intervals of full length (`fit_window`=14d +
 
 ## Forecasting
 
-`vmanomaly` can generate future forecasts (e.g. using [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}), which is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
+`vmanomaly` can generate future forecasts using [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} or [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}. This is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
 
 > However, please note that this mode should be used with care, as the model will produce `yhat_{h}` (and probably `yhat_lower_{h}`, and `yhat_upper_{h}`) time series **for each timeseries returned by input queries and for each forecasting horizon specified in `forecast_at` argument, which can lead to a significant increase in the number of active timeseries in VictoriaMetrics TSDB**.
 
@@ -309,12 +265,12 @@ Here's an example of how to produce forecasts using `vmanomaly` and combine it w
 schedulers:
   periodic_5m:  # this scheduler will be used to produce anomaly scores each 5 minutes using "regular" simple model
     class: 'periodic'
-    fit_every: '30d'
+    fit_every: '100w'
     fit_window: '3d'
     infer_every: '5m'
   periodic_forecast:  # this scheduler will be used to produce forecasts each 24h using "daily" model
     class: 'periodic'
-    fit_every: '7d'
+    fit_every: '1000w'
     fit_window: '730d' # to fit the model on 2 years of data to account for seasonality and holidays
     infer_every: '24h'
 # https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader
@@ -356,20 +312,18 @@ models:
     quantiles: [0.25, 0.5, 0.75]  # to produce median and upper quartiles
     iqr_threshold: 2.0
 
-  prophet_1d:
-    class: 'prophet'
+  envelope_1d:
+    class: 'temporal_envelope'
     queries: ['disk_usage_perc_1d']
     schedulers: ['periodic_forecast']
     clip_predictions: True
     detection_direction: 'above_expected'  # as we are interested in spikes in capacity planning
     forecast_at: ['3d', '7d']  # this will produce forecasts for 3 and 7 days ahead
     provide_series: ['yhat', 'yhat_upper']  # to write forecasts back to VictoriaMetrics, omitting `yhat_lower` as it is not needed in this example
-    # other model params, yearly_seasonality may stay
-
-    # https://facebook.github.io/prophet/docs/quick_start#python-api
-    args:
-      interval_width: 0.98  # see https://facebook.github.io/prophet/docs/uncertainty_intervals
-      country_holidays: 'US'
+    seasonalities: [dow_smooth]
+    holidays:
+      countries: [US]
+      group: true
 # https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer
 writer:
   class: 'vm'
@@ -408,6 +362,61 @@ groups:
       description: "Disk usage is forecasted to exceed 95% in the next 3 days for instance {{ $labels.instance }}. Forecasted value: {{ $value }}."
 ```
 
+## Alert generation in vmanomaly
+While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
+
+<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
+
+Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
+
+```yaml
+groups:
+- name: VMAnomalyAlerts
+  interval: 60s
+  rules:
+  - alert: HighAnomalyScore
+    expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
+    for: 5m  # adjust to your needs based on data frequency and alerting policies
+    labels:
+      severity: warning
+      query_alias: explore
+      model_alias: default
+      scheduler_alias: periodic
+      preset: ui
+    annotations:
+      summary: High anomaly score detected.
+      description: Anomaly score exceeded threshold ({{ $value }}) for more than
+        {{ $for }} for query {{ $labels.for }}.
+```
+
+> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
+
+> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
+
+## Preventing alert fatigue
+Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
+
+## Deploying vmanomaly
+
+`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
+
+## Environment variables
+
+`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
+
+## Is vmanomaly stateful?
+By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
+
+Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
+
+## Config hot-reloading
+
+`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
+
+## Migration
+
+For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
+
 ## Resource consumption of vmanomaly
 `vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), and the complexity of the employed [models](https://docs.victoriametrics.com/anomaly-detection/components/models/). Its resource usage is directly related to these factors, making it adaptable to various operational scales. Various optimizations are available to balance between RAM usage, processing speed, and model capacity. These options are described in the sections below.
 
@@ -423,7 +432,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.7
+    image: victoriametrics/vmanomaly:v1.30.0
     # ...
     restart: always
     volumes:
@@ -460,6 +469,8 @@ With the introduction of [online models](https://docs.victoriametrics.com/anomal
 - **Optimized resource utilization**: By spreading the computational load over time and reducing peak demands, online models make more efficient use of resources and inducing less data transfer from VictoriaMetrics TSDB, improving overall system performance.
 - **Faster convergence**: Online models can adapt {{% available_from "v1.23.0" anomaly %}} to changes in data patterns more quickly, which is particularly beneficial in dynamic environments where data characteristics may shift frequently. See `decay` argument description [here](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay).
 
+{{% available_from "v1.30.0" anomaly %}} Online models expose a **common** `min_n_samples_seen` warmup: they continue learning, but emit `anomaly_score: 0` until enough observations have been seen. Online MAD, Z-score, and Seasonal Quantile also support `history_strength`, which retains fitted history as a stronger prior while inference continues to adapt the state.
+
 > {{% available_from "v1.24.0" anomaly %}} Online models are best used in conjunction with [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the model state across service restarts. This allows the model to continue adapting to new data without losing previously learned patterns, thus avoiding the need for a full `fit` stage to start working again. {{% available_from "v1.28.1" anomaly %}} Additionally, setting [retention policies](https://docs.victoriametrics.com/anomaly-detection/components/settings/#retention) helps manage disk space or RAM used by periodical cleanup of old model instances.
 
 Here's an example of how we can switch from (offline) [Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score) to [Online Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score):
@@ -641,7 +652,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.7 && docker image tag victoriametrics/vmanomaly:v1.29.7 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.30.0 && docker image tag victoriametrics/vmanomaly:v1.30.0 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/Migration.md

@@ -45,7 +45,7 @@ There are 2 types of compatibility to consider when migrating in stateful mode:
 
 | Group start | Group end | Compatibility | Notes |
 |---------|--------- |------------|-------|
-| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1297) | Fully Compatible | - |
+| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300) | Fully Compatible | v1.30.0 adds new [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model state without changing the compatibility of existing model and data artifacts. |
 | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Partially compatible* | Dumped models of class [prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) and [seasonal quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) have problems with loading to [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) due to dropped `pytz` library. **Upgrading directly from v1.28.7 to [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) with a fix is suggested** |
 | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
 | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |

docs/anomaly-detection/QuickStart.md

@@ -132,7 +132,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.7
+docker pull victoriametrics/vmanomaly:v1.30.0
 ```
 
 2. Create the license file with your license key.
@@ -152,7 +152,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.7 \
+    victoriametrics/vmanomaly:v1.30.0 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -169,7 +169,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.7 \
+    victoriametrics/vmanomaly:v1.30.0 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -182,7 +182,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.7
+    image: victoriametrics/vmanomaly:v1.30.0
     # ...
     restart: always
     volumes:
@@ -245,7 +245,7 @@ Before deploying, check the correctness of your configuration validate config fi
 
 ### Example
 
-Here is an example of a config file that will run the [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) model on `vm_cache_entries` metric, with periodic scheduler that runs inference every minute and fits the model every day. The model will be trained on the last 2 weeks of data each time it is (re)fitted. The model will produce `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging purposes. The model will be timezone-aware and will use cyclical encoding for the hour of the day and day of the week seasonality.
+Here is an example of a config file that runs the online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model on a CPU metric. The scheduler runs inference every five minutes and performs a full refit only every 100 weeks; between refits the model updates causally from each inference batch. The initial fit uses four weeks of data. The model produces `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging, and its hour-of-day and day-of-week profiles follow the query timezone and daylight-saving-time changes.
 
 ```yaml
 settings:
@@ -260,38 +260,30 @@ settings:
     # scheduler: INFO
     # reader: INFO
     # writer: INFO
-    model.prophet: WARNING
+    model.online.temporal_envelope: WARNING
 
 schedulers:
-  1d_5m:
+  100w_5m:
     # https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler
     class: 'periodic'
     infer_every: '5m'
     scatter_infer_jobs: true
-    fit_every: '1d'
+    # Temporal Envelope learns online between full refits.
+    fit_every: '100w'
     fit_window: '4w'
 
 models:
-  # https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet
-  prophet_model:
-    class: 'prophet'
+  # https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope
+  temporal_envelope_model:
+    class: 'temporal_envelope'
+    queries: ['cpu_user']
+    schedulers: ['100w_5m']
     provide_series: ['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']  # for debugging
-    tz_aware: True  # set to True if your data is timezone-aware, to deal with DST changes correctly
-    tz_use_cyclical_encoding: True
-    tz_seasonalities: # intra-day + intra-week seasonality
-      - name: 'hod'  # intra-day seasonality, hour of the day
-        fourier_order: 4  # keep it 3-8 based on intraday pattern complexity
-        prior_scale: 10
-      - name: 'dow'  # intra-week seasonality, time of the week
-        fourier_order: 2  # keep it 2-4, as dependencies are learned separately for each weekday
-    compression:  # available since v1.28.1
-      window: "30m"               # downsample 5m data into 30m intervals before fitting
-      agg_method: "mean"          # use mean aggregation within each window
-      adjust_boundaries: true     # adjust confidence intervals after downsampling
-    # inner model args (key-value pairs) accepted by
-    # https://facebook.github.io/prophet/docs/quick_start#python-api
-    args:
-      interval_width: 0.98  # see https://facebook.github.io/prophet/docs/uncertainty_intervals
+    seasonalities: ['hod_smooth', 'dow_smooth']
+    alpha: 0.005  # trend reactivity; try 0.0025-0.02
+    loss_reactivity: 5  # try 1-5; lower values reduce the influence of spikes
+    iqr_threshold: 2  # try 1-4 to adjust data-driven interval width
+    min_n_samples_seen: 16
 
 reader:
   class: 'vm'  # use VictoriaMetrics as a data source
@@ -299,6 +291,7 @@ reader:
   datasource_url: "https://play.victoriametrics.com/" # [YOUR_DATASOURCE_URL]
   tenant_id: '0:0'
   sampling_period: "5m"
+  tz: 'UTC'  # set the IANA timezone that defines local calendar patterns, e.g. 'America/New_York'
   series_processing_batch_size: 8  # number of time series to process together while preparing data for fit or infer stages
   queries:
     # define your queries with MetricsQL - https://docs.victoriametrics.com/victoriametrics/metricsql/
@@ -317,7 +310,7 @@ writer:
 
 ### UI
 
-{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI can be used for prototyping and interactive experimenting to produce vmanomaly's and vmalert's configuration files. Please refer to the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for detailed instructions and examples. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the UI to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
+{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI supports prototyping and interactive generation of `vmanomaly` and `vmalert` configuration files. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for instructions and examples. For optional AI-assisted workflows, use the [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance), connect the [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server), or follow the published [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills).
 
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 > [!TIP]
@@ -409,7 +402,7 @@ For optimal service behavior, consider the following tweaks when configuring `vm
 
 - Set up **state restoration** {{% available_from "v1.24.0" anomaly %}} to resume from the last known state for long-term stability. This is controlled by the `settings.restore_state` boolean [arg](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration).
 
-- Set up **config hot-reloading** {{% available_from "v1.25.0" anomaly %}} to automatically reload configurations on config files changes. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) and allows for configuration updates without explicit service restarts.
+- Set up **configuration hot reload** {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
 
 **Schedulers**:
 - Configure the **inference frequency** in the [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) section of the configuration file.

docs/anomaly-detection/README.md

@@ -27,7 +27,7 @@ Key functions:
 
 The diagram below illustrates how `vmanomaly` fits into an observability setup, such as detecting anomalies in metrics collected by `node_exporter`:
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
+<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
 
 ## How does it work?
 
@@ -40,7 +40,7 @@ VictoriaMetrics Anomaly Detection **continuously re-fit and apply machine learni
   - **Confidence intervals** (`[yhat_lower, yhat_upper]`)  
   These outputs integrate seamlessly into downstream applications, making it easier to **visually inspect anomalies**, e.g. in respective [Grafana dashboards](https://docs.victoriametrics.com/anomaly-detection/presets/#grafana-dashboard).
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/components/vmanomaly-components.webp" alt="node_exporter_example_diagram" style="width:80%"/>
+{{% content "components/vmanomaly-components-diagram.md" %}}
 
 ## Key benefits
 
@@ -58,7 +58,7 @@ Get started with VictoriaMetrics Anomaly Detection by following our guides and i
 
 - **Quickstart**: Learn how to quickly set up `vmanomaly` by following the [Quickstart Guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/).
 - **UI**: Explore anomaly detection configurations through the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
-- **MCP**: Allow AI to assist you in generating service and alerting configurations, answering questions, planning migration with the [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly). Find the setup guide how to setup and use it [here](https://github.com/VictoriaMetrics/mcp-vmanomaly?tab=readme-ov-file#installation).
+- **AI-assisted setup (optional)**: Use the [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance), [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server), or [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) to inspect time-series characteristics, select and autotune models, and generate validated configurations.
 - **Integration**: Integrate anomaly detection into your existing observability stack. Find detailed steps [here](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
 - **Anomaly Detection Presets**: Enable anomaly detection on predefined sets of metrics. Learn more [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
 

docs/anomaly-detection/Scaling-vmanomaly.md

@@ -78,9 +78,7 @@ These [sub-configurations](#sub-configuration) can be assigned to a specific sha
 
 Additionally, a replication factor `R ≥ 1` ensures [high availability](#high-availability) by enforcing redundancy across shards.
 
-<p></p>
-
-![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
+{{% content "vmanomaly-sharding-ha-diagram.md" %}}
 
 > Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
 
@@ -130,9 +128,7 @@ Similar to other VictoriaMetrics ecosystem components, like [VMAgent](https://do
 
 When `VMANOMALY_REPLICATION_FACTOR` > 1, each [sub-config](#sub-configuration) `n` from `{0, N-1}` is assigned to exactly `R` nodes. This ensures redundancy, preventing single-node failures from causing data loss.
 
-<p></p>
-
-![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
+{{% content "vmanomaly-sharding-ha-diagram.md" %}}
 
 > Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
 

docs/anomaly-detection/Self-monitoring.md

@@ -135,6 +135,8 @@ These alerting rules complements the [dashboard](#grafana-dashboard) to monitor
 `vmanomaly-health` alerting group:
 - **`TooManyRestarts`**: Triggers if an instance restarts more than twice within 15 minutes, suggesting the process might be crashlooping and needs investigation.
 - **`ServiceDown`**: Alerts if an instance is down for more than 5 minutes, indicating a service outage.
+- **`SchedulerWorkerDown`**: {{% available_from "v1.30.0" anomaly %}} Alerts when an individual periodic scheduler remains down for more than 2 minutes after automatic recovery attempts. This catches partial service failures that a process-level `ServiceDown` alert cannot detect.
+- **`FrequentSchedulerRestarts`**: {{% available_from "v1.30.0" anomaly %}} Warns when an individual scheduler has more than two restart attempts within 15 minutes. A single successful self-heal does not alert, while repeated attempts indicate a flapping scheduler or dependency.
 - **`ProcessNearFDLimits`**: Alerts when the number of available file descriptors falls below 100, which could lead to severe degradation if the limit is exhausted.
 - **`TooHighCPUUsage`**: Alerts when CPU usage exceeds 90% for a continuous 5-minute period, indicating possible resource exhaustion and the need to adjust resource allocation or load.
 - **`TooHighMemoryUsage`**: Alerts when RAM usage exceeds 85% for a continuous 5-minute period and the need to adjust resource allocation or load.

docs/anomaly-detection/UI.md

@@ -20,13 +20,13 @@ aliases:
 
 ## Introduction
 
-{{% available_from "v1.26.0" anomaly %}} `vmanomaly` is shipped with a built-in [vmui-like](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) [UI](https://en.wikipedia.org/wiki/Graphical_user_interface) that provides an intuitive interface for rapid exploration of how anomaly detection models, their configurations and included domain knowledge impacts the results of anomaly detection, before such configurations are deployed in production.
+{{% available_from "v1.26.0" anomaly %}} `vmanomaly` includes a built-in [vmui-like](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) [UI](https://en.wikipedia.org/wiki/Graphical_user_interface) for exploring queries, comparing anomaly detection models, and tuning model or domain settings before production deployment.
 
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 
 ## Accessing the UI
 
-The UI is available at `http://<vmanomaly-host>:8490` by default, however, the port can be changed in `server` [section](https://docs.victoriametrics.com/anomaly-detection/components/server/) of the [configuration file](https://docs.victoriametrics.com/anomaly-detection/components/) using the `port` parameter:
+The UI is available at `http://<vmanomaly-host>:8490` by default. Change the port with `server.port` in the [configuration file](https://docs.victoriametrics.com/anomaly-detection/components/):
 
 ```yaml
 server:
@@ -39,7 +39,7 @@ For impactful parameters please refer to [optimize resource usage](#optimize-res
 
 ## Playgrounds
 
-To start exploring the UI, you can use embedded demo with preconfigured queries and models down below on public playgrounds (VictoriaMetrics, VictoriaLogs and VictoriaTraces):
+Try the UI with preconfigured queries and models in the public VictoriaMetrics, VictoriaLogs, and VictoriaTraces playgrounds:
 
 {{% collapse name="Playground on VictoriaMetrics Datasource" %}}
 
@@ -151,10 +151,11 @@ users:
 Then, on [settings panel](#settings-panel) of the UI, set the URLs accordingly, also check the option to forward auth headers to the datasource:
 
 ![vmanomaly-ui-sections-settings](vmanomaly-ui-sections-settings.webp)
+{class="w-50 mx-auto"}
 
 ### Pre-configured Datasource
 
-{{% available_from "v1.28.2" anomaly %}} It is possible to disable the datasource selectors from UI (e.g. at purpose to serve internal teams) by using pre-configured one with respective environment variables at `vmanomaly` startup:
+{{% available_from "v1.28.2" anomaly %}} For a shared deployment with a fixed datasource, set these environment variables at `vmanomaly` startup. The UI then hides its datasource selectors:
 
 - `VMANOMALY_UI_DATASOURCE_URL` - to set static datasource URL
 - `VMANOMALY_UI_DATASOURCE_TYPE` - to set datasource type, supported options are `vm` for VictoriaMetrics, `vmlogs` for both VictoriaLogs and VictoriaTraces.
@@ -165,10 +166,9 @@ export VMANOMALY_UI_DATASOURCE_URL=https://play.victoriametrics.com/select/0:0/p
 export VMANOMALY_UI_DATASOURCE_TYPE=vm
 ```
 
-After that, start `vmanomaly` instance as usual, and the datasource selectors will be hidden from UI, while the pre-configured datasource will be used for all queries:
-![vmanomaly-ui-preconfigured-datasource-1](vmanomaly-ui-preconfigured-datasource-1.webp)
+Start `vmanomaly` as usual. All UI queries will use the configured datasource:
 
-![vmanomaly-ui-preconfigured-datasource-2](vmanomaly-ui-preconfigured-datasource-2.webp)
+![vmanomaly-ui-preconfigured-datasource-1](vmanomaly-ui-preconfigured-datasource-1.webp)
 
 ## Preset
 
@@ -193,9 +193,10 @@ The best applications of this mode are:
 
 ### What you can do with Copilot
 
-- **Ask questions** about any model (e.g. [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
+- **Ask questions** about any model (e.g. [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope), [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
 - **Improve detection quality** - describe what's wrong ("too many false positives", "missing spikes") and Copilot reads the config, searches the docs, and proposes a validated configuration change to fix the issue.
 - **Get config suggestions inline** - suggestions appear as interactive cards with an explanation and a YAML diff; click **Apply** to write the change directly to your current settings, or **Decline** to keep the conversation going.
+- {{% available_from "v1.30.0" anomaly %}} **Profile and tune the real query** - with [mcp-vmanomaly](#mcp-tools-server) connected, Copilot can inspect bounded time-series characteristics, recommend an online model, start an asynchronous autotune task, and apply its validated query and model suggestions.
 
 ### How it works
 
@@ -212,7 +213,7 @@ AI Assistant is disabled by default; enable it with `VMANOMALY_COPILOT_ENABLED=t
 Supported providers and model formats:
 
 - **Anthropic** - set `ANTHROPIC_API_KEY`; model format: `anthropic:<model>`
-  - Examples: `claude-haiku-4-5`, `claude-sonnet-4-6`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
+  - Examples: `claude-haiku-4-5`, `claude-sonnet-5`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
 - **OpenAI** - set `OPENAI_API_KEY`; model format: `openai:<model>` or `openai-responses:<model>`
   - Examples: `gpt-5-mini`, `gpt-5.2`; see [full list](https://platform.openai.com/docs/models)
   - {{% available_from "v1.29.1" anomaly %}} OpenAI-compatible non-OpenAI providers are supported through `OPENAI_BASE_URL` + `OPENAI_API_KEY`
@@ -315,14 +316,14 @@ docker run -it --rm \
   -e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
   -p 8080:8080 \
   -p 8490:8490 \
-  victoriametrics/vmanomaly:v1.29.7 \
+  victoriametrics/vmanomaly:v1.30.0 \
   vmanomaly_config.yaml
 ```
 
 
 ## UI Navigation
 
-The vmanomaly UI provides a user-friendly interface for exploring and configuring anomaly detection models. The main components of the UI include:
+The UI has four main areas:
 
 - [**Query Explorer**](#query-explorer): A vmui-like interface for typing and executing MetricsQL/LogsQL queries to visualize data.
 - [**Model Panel**](#model-panel): A form for editing anomaly detection model hyperparameters and applying domain knowledge settings.
@@ -331,7 +332,7 @@ The vmanomaly UI provides a user-friendly interface for exploring and configurin
 
 ### Query Explorer
 
-The Query Explorer provides a vmui-like interface for typing and executing MetricsQL/LogsQL queries to visualize data. 
+Use Query Explorer to run MetricsQL or LogsQL queries and visualize input data.
 
 ![vmanomaly-ui-sections-explore](vmanomaly-ui-sections-explore.webp)
 
@@ -345,23 +346,23 @@ Users can:
 
 ### Visualization Panel
 
-The Visualization Panel has 2 modes of displaying data - either raw queried data or data with detected anomalies, depending on the action taken in the Model Panel.
+The Visualization Panel displays either raw query results or model output, depending on the selected action.
 
-**Visualizations of the queried data** ("Execute Query" button)
+After selecting **Execute Query**:
 
 ![vmanomaly-ui-sections-plot-area-query-mode](vmanomaly-ui-sections-plot-area-query-mode.webp)
 
-> All the metrics are shown in a single plot, similar to vmui, with zooming and panning capabilities.
+All returned series appear in one vmui-like plot with zooming and panning.
 
-**Initial data with detected anomalies** ("Detect Anomalies" button)
+After selecting **Detect Anomalies**:
 
 ![vmanomaly-ui-sections-plot-area-detect-mode](vmanomaly-ui-sections-plot-area-detect-mode.webp)
 
-> The plot shows the queried data, **grouped by individual series**, iterated over legend, with the actual values (`y`) compared to the expected values (model predictions, `y_hat`), confidence intervals (`y_hat_lower`, `y_hat_upper`), and detected anomalies. The anomalies are marked with red circles, and hovering over them provides additional information such as the anomaly score and associated labels.
+The plot groups model output by input series and compares actual values (`y`) with predictions (`yhat`), confidence intervals (`yhat_lower`, `yhat_upper`), and detected anomalies. Hover over an anomaly marker to inspect its score and labels.
 
-Also, timeseries (such as `y`, `y_hat`, etc.) can be toggled on/off by clicking on the legend items.
+Toggle individual output series from the legend.
 
-{{% available_from "v1.29.2" anomaly %}} Seeing model [business-boundaries](https://docs.victoriametrics.com/anomaly-detection/faq/#incorporating-domain-knowledge), such as [detection direction](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) and minimal deviation from expected ([absolute](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-deviation-from-expected) and [relative](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-relative-deviation-from-expected) combined) can be turned on with "business boundaries" toggle. Showing/hiding individual bands can be done by clicking on the respective legend items, while showing/hiding all business boundaries at once can be done with "business boundaries" toggle.
+{{% available_from "v1.29.2" anomaly %}} Enable **Business Boundaries** to overlay the configured [detection direction](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) and combined [absolute](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-deviation-from-expected) and [relative](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-relative-deviation-from-expected) minimum-deviation bands. Toggle individual bands from the legend or all bands with the main control.
 
 [Back to UI navigation](#ui-navigation)
 
@@ -371,18 +372,18 @@ Also, timeseries (such as `y`, `y_hat`, etc.) can be toggled on/off by clicking
 
 The Model Panel provides:
 
-Parameters, such as "Fit Every", "Fit Window" and {{% available_from "v1.28.0" anomaly %}} "Infer Every" to imitate [production scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler), as well as overriding default [anomaly detection threshold](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) (1.0).
+- Scheduling controls such as **Fit Every**, **Fit Window**, and {{% available_from "v1.28.0" anomaly %}} **Infer Every** for imitating [production scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler).
+- An override for the default [anomaly detection threshold](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) of `1.0`.
+- Actions for running or canceling detection, downloading results, and exporting model configurations or example alerting rules.
 
-> {{% available_from "v1.28.0" anomaly %}} "Exact" mode checkbox is used in combination with "Infer Every" control for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) such as `mad_online` or `quantile_online`, to provide unbiased estimates of how production scheduler would perform anomaly detection on incoming data streams. In "exact" mode, the model is updated exactly at every "infer every" micro-batch interval, at a cost of increased computation time.
+> {{% available_from "v1.28.0" anomaly %}} For [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models), combine **Exact** mode with **Infer Every** to reproduce causal production micro-batches. This gives a more representative backtest at the cost of additional computation.
 
-Controls for running/canceling anomaly detection on the queried data, downloading the results as CSV/JSON, accessing and downloading the model configuration or example alerting rules in YAML format.
+A form-based menu configures model hyperparameters and domain knowledge:
 
-A form-based menu for finetuning model hyperparameters and applying domain knowledge settings:
-
-- Model type selection (e.g., rolling quantile, Prophet, etc.)
+- Model selection, for example Temporal Envelope or Online MAD.
 ![vmanomaly-ui-model-config-menu](vmanomaly-ui-model-config-menu.webp)
-- Wizard with **model-agnostic parameters** (e.g., detection direction, data range, scale, clipping, minimum deviation from expected, etc.) and **model-specific hyperparameters** for chosen model type (e.g., quantile and window steps for [rolling quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-quantile) model). {{% available_from "v1.27.0" anomaly %}} autocomplete of example parameters by hitting Tab key is supported.
-![vmanomaly-ui-model-config-wizard](vmanomaly-ui-model-config-wizard.webp)
+- A wizard with **model-agnostic settings** such as detection direction, data range, clipping, and minimum deviation, plus the selected model's hyperparameters. {{% available_from "v1.27.0" anomaly %}} Press **Tab** to autocomplete suggested values.
+  <a class="content-image d-flex justify-content-center" data-bs-target="#image-modal" data-bs-toggle="modal" href="/anomaly-detection/vmanomaly-ui-model-config-wizard.webp"><img alt="vmanomaly-ui-model-config-wizard" class="w-75 mx-auto" src="/anomaly-detection/vmanomaly-ui-model-config-wizard.webp" style="min-width: 0;" /></a>
 
 [Back to UI navigation](#ui-navigation)
 
@@ -397,6 +398,7 @@ The vmui-like "Settings" panel allows users to configure global settings and pre
 - {{% available_from "v1.27.0" anomaly %}} Auth Headers forwarding to datasource (VictoriaMetrics, VictoriaLogs). 
 
 ![vmanomaly-ui-sections-settings](vmanomaly-ui-sections-settings.webp)
+{class="w-50 mx-auto"}
 
 [Back to navigation](#ui-navigation)
 
@@ -585,6 +587,7 @@ Set the "Fit Every" and "Fit Window" parameters to control how often and over wh
 Tune the model hyperparameters and apply domain knowledge settings using the form-based menu in the Model Panel. See (i) tooltips for parameter descriptions and [model documentation](https://docs.victoriametrics.com/anomaly-detection/components/models/) link for recommended values and guidelines.
 
 ![vmanomaly-ui-model-config-wizard](vmanomaly-ui-model-config-wizard.webp)
+{class="w-75 mx-auto"}
 
 For example, for a **MAD online** [model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad), that should be run on a query, returning per-mode CPU utilization (as fractions of 1, data range `[0, 1]`), where you are interested in capturing **spikes of at least 6% deviations** from expected behavior:
 
@@ -640,6 +643,31 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
+{{% collapse name="Release history" %}}
+
+### v1.8.0
+Released: 2026-07-23
+
+vmanomaly version: [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300)
+
+- FEATURE: UX Improvements, refreshed interface with clearer model and query forms, inline configuration previews, and fullscreen chart controls.
+
+- FEATURE: Server (production-running) models and queries can be accessed and selected from the UI, with a new "Queries" button, while model selection now includes a drop-down for server-configured scheduled models in model wizard.
+
+- IMPROVEMENT: Boosted [AI Copilot](#ai-assistance) stability and suggestions quality, aligned with MCP/skills toolset, and proper handling of canceled or incomplete tool calls.
+
+- IMPROVEMENT: Added query prettification and stable value formatting, including platform-aware keyboard-shortcut hints.
+
+- IMPROVEMENT: Results now visibly switch to an out-of-date state after the query, time range, or model configuration changes. The action updates to rerun detection and remains visible in both expanded and collapsed model views.
+
+- IMPROVEMENT: Chart range navigation commits one query on interaction completion instead of issuing many intermediate requests.
+
+- IMPROVEMENT: Compact consecutive identical [AI Copilot](#ai-assistance) tool calls, keep query/model/anomaly suggestions synchronized, and recover cleanly from canceled or incomplete tool calls.
+
+- BUGFIX: Fixed exact UI backtesting across multiple fit cycles and for auto-tuned online wrappers, preventing *duplicate or missing predictions* while retaining causal model updates and restoration of compatible legacy auto-tuned state.
+
+- BUGFIX: Kept automatic trailing-slash redirects for configured path prefixes and `/vmui` relative to the public origin, preventing internal backend hostnames from leaking through reverse proxies such as `vmauth`.
+
 ### v1.7.2
 Released: 2026-06-25
 
@@ -809,3 +837,5 @@ Released: 2025-10-02
 vmanomaly version: [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1260)
 
 Initial public release of the vmanomaly UI.
+
+{{% /collapse %}}

docs/anomaly-detection/components/README.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-This chapter describes different components, that correspond to respective sections of a config to launch VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/) service:
+This chapter describes the configuration sections used to run VictoriaMetrics Anomaly Detection, or [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/):
 
 - [Model(s) section](https://docs.victoriametrics.com/anomaly-detection/components/models/) - Required
 - [Reader section](https://docs.victoriametrics.com/anomaly-detection/components/reader/) - Required
@@ -16,9 +16,9 @@ This chapter describes different components, that correspond to respective secti
 - [Settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/) - Optional
 - [Server section](https://docs.victoriametrics.com/anomaly-detection/components/server/) - Optional
 
-> Once the service starts, automated config validation is performed {{% available_from "v1.7.2" anomaly %}}. Please see container logs for errors that need to be fixed to create fully valid config, visiting sections above for examples and documentation.
+> The service validates its configuration at startup{{% available_from "v1.7.2" anomaly %}}. Check the container logs for validation errors and use the sections above for field descriptions and examples.
 
-> Components' class {{% available_from "v1.13.0" anomaly %}} can be referenced by a short alias instead of a full class path - i.e. `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, `scheduler.periodic.PeriodicScheduler` becomes `periodic`, etc. Please see according sections for the details.
+> Component classes{{% available_from "v1.13.0" anomaly %}} can be referenced by short aliases instead of full import paths. For example, `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, and `scheduler.periodic.PeriodicScheduler` becomes `periodic`.
 
 > `preset` modes are available {{% available_from "v1.13.0" anomaly %}} for `vmanomaly`. Please find the guide [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
 
@@ -28,11 +28,11 @@ Below, you will find an example illustrating how the components of `vmanomaly` i
 
 > [Reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [Writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) also support [multitenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy), so you can read/write from/to different locations - see `tenant_id` param description.
 
-![vmanomaly-components](vmanomaly-components.webp)
+{{% content "vmanomaly-components-diagram.md" %}}
 
 ## Example config
 
-Here's a minimalistic full config example, demonstrating many-to-many configuration (actual for [latest version](https://docs.victoriametrics.com/anomaly-detection/changelog/)):
+The following minimal configuration demonstrates current many-to-many model, query, and scheduler mapping:
 
 ```yaml
 settings:
@@ -52,7 +52,7 @@ schedulers:
     scatter_infer_jobs: true  # distribute infer jobs evenly across the infer interval to reduce synchronized bursts
     fit_every: "365d"  # how often to re-fit the models, for online models used effectively once, then they are updated with new data and won't require re-fit
     fit_window: "3d"  # how much historical data to use for fit stage
-    start_from: "00:00"  # start from specified time, i.e. 00:00 given timezone and do daily fits as `fit_every` is 1 day
+    start_from: "00:00"  # align the annual fit schedule to midnight in the configured timezone
     tz: "Europe/Kyiv"  # timezone to use for start_from
   periodic_offline_1w:
     class: 'periodic'
@@ -146,14 +146,14 @@ server:
 
 > This feature is better used in conjunction with [stateful service](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the state of the models and schedulers between restarts and reuse what can be reused, thus avoiding unnecessary re-training of models, re-initialization of schedulers and re-reading of data.
 
-{{% available_from "v1.25.0" anomaly %}} Service supports hot reload of configuration files, which allows for automatic reloading of configurations on config files change without the need of explicit service restart. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). `vmanomaly_config_reload_enabled` flag in [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) will be set to 1 (if enabled) or 0 (if disabled).
+{{% available_from "v1.25.0" anomaly %}} The service supports hot reload of configuration files, applying changes without an explicit restart. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). The `vmanomaly_config_reload_enabled` [self-monitoring metric](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) is `1` when hot reload is enabled and `0` otherwise.
 
 > [!NOTE]
 > {{% deprecated_from "v1.29.5" anomaly %}} File system event-based hot reload has been deprecated in favor of content-based polling with configurable `-configCheckInterval` due to reliability issues with Kubernetes ConfigMap symlink rotations and other filesystems where event delivery can be inconsistent. If you were using file system event-based hot reload, please switch to content-based polling by enabling `--watch` flag and configuring `-configCheckInterval` as needed.
 
 ### How it works
 
-It works by checking watched `.yml|.yaml` file contents in the specified files or directories on the configured interval `-configCheckInterval` (default is `30s`) {{% available_from "v1.29.5" anomaly %}}. When a content change is detected, the service will attempt to reload the configuration files after the existing debounce window, rebuild the [global config](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration) and reinitialize the components. If the reload is successful, the `vmanomaly_config_reloads_total` metric will be incremented for `status="success"` label, otherwise it will be incremented with `status="failure"` label and a respective error message on config validation failure(s) will be logged.
+The service checks watched `.yml` and `.yaml` files at the `-configCheckInterval` interval (default `30s`){{% available_from "v1.29.5" anomaly %}}. When it detects a content change, it waits for the debounce window, rebuilds the [global configuration](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration), and reinitializes the components. The `vmanomaly_config_reloads_total` metric is incremented with `status="success"` or `status="failure"`; validation failures are also logged.
 
 > If the reload fails, the service will log an error message indicating the reason for the failure, and the **previous configuration will remain active until a successful reload occurs** to preserve the service's stability. This means that if there are errors in the new configuration, the service will continue to operate with the last valid configuration until the issues are resolved.
 

docs/anomaly-detection/components/autotune.svg

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 968 420" role="img" aria-labelledby="title description">
+  <title id="title">AutoTunedModel tuning and inference lifecycle</title>
+  <desc id="description">The tuning process tests and scores model candidates across n time-series splits until the trial count or timeout is reached. Each fold fits a candidate on training data and predicts anomalies on its validation segment. The best model is then used on inference data until the next fit call.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M0 0 10 5 0 10Z" fill="#202124"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #202124; font-size: 18px; }
+      .line { fill: none; stroke: #202124; stroke-width: 2.5; }
+      .arrow { fill: none; stroke: #202124; stroke-width: 2.5; marker-end: url(#arrow); }
+      .dash { fill: none; stroke: #202124; stroke-width: 2.5; stroke-dasharray: 8 9; }
+      .box { fill: #fff; stroke: #202124; stroke-width: 1.5; }
+    </style>
+  </defs>
+
+  <rect width="968" height="420" fill="#fff"/>
+  <rect x="112" y="10" width="242" height="83" class="box"/>
+  <text x="233" y="38" text-anchor="middle">
+    <tspan x="233" dy="0">test and score candidates</tspan>
+    <tspan x="233" dy="21">until `n_trials` or `timeout`</tspan>
+    <tspan x="233" dy="21">is reached</tspan>
+  </text>
+  <path d="M354 49 V67" class="arrow"/>
+  <text x="355" y="90" text-anchor="middle">Tuning</text>
+  <text x="355" y="112" text-anchor="middle">process</text>
+
+  <path d="M162 157 V136 H549 V157" class="line"/>
+  <path d="M355 112 V136" class="line"/>
+  <path d="M399 93 H653 V152" class="line"/>
+
+  <path d="M113 161 H75 V350 H113" class="line"/>
+  <path d="M75 236 H39" class="line"/>
+  <text x="28" y="250" transform="rotate(-90 28 250)" text-anchor="middle">n splits</text>
+
+  <text x="102" y="227">Fold 1</text>
+  <text x="102" y="255">Fold 2</text>
+  <text x="102" y="283">Fold 3</text>
+  <text x="120" y="327">...</text>
+
+  <text x="260" y="179" text-anchor="middle">Fit</text>
+  <text x="260" y="202" text-anchor="middle">candidate</text>
+  <text x="413" y="179" text-anchor="middle">Predict</text>
+  <text x="413" y="202" text-anchor="middle">anomalies</text>
+
+  <path d="M173 221 H354 V207" class="line"/>
+  <path d="M232 249 H412 V235" class="line"/>
+  <path d="M293 280 H472 V265" class="line"/>
+  <path d="M354 221 H412" class="dash"/>
+  <path d="M412 249 H474" class="dash"/>
+  <path d="M472 280 H513" class="dash"/>
+
+  <path d="M548 144 V415" class="dash" style="stroke-width:1.5;stroke-dasharray:4 6"/>
+  <rect x="577" y="152" width="136" height="61" rx="12" class="box"/>
+  <text x="645" y="187" text-anchor="middle">best model</text>
+  <path d="M653 213 V348" class="arrow"/>
+  <text x="815" y="168" text-anchor="middle">
+    <tspan x="815" dy="0">used to predict</tspan>
+    <tspan x="815" dy="21">on inference data</tspan>
+    <tspan x="815" dy="21">until the next `fit` call</tspan>
+  </text>
+
+  <path d="M40 350 H895" class="arrow"/>
+  <text x="455" y="402" text-anchor="middle">Training data</text>
+  <text x="623" y="402" text-anchor="middle">Inference data</text>
+  <text x="856" y="402">Time axis, t</text>
+</svg>

docs/anomaly-detection/components/model-lifecycle-multivariate.svg

@@ -0,0 +1,139 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
+  <title id="title">Multivariate models lifecycle</title>
+  <desc id="description">MetricsQL queries retrieve an aligned set of series from VictoriaMetrics. Reader data fits one shared multivariate model. The exact same series set produces one anomaly score series with the intersected label set; inference is skipped when the fit and inference series sets differ. Writer stores produced scores in VictoriaMetrics.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
+      .title { font-size: 50px; font-weight: 400; }
+      .node { font-size: 34px; font-weight: 400; }
+      .label { font-size: 29px; }
+      .blue-label { font-size: 26px; }
+      .small { font-size: 25px; }
+      .box { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .group { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
+      .line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
+      .blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
+      .blue { fill: #1478c9; }
+    </style>
+  </defs>
+
+  <rect width="1920" height="1295" fill="#fff"/>
+  <text x="20" y="70" class="title">Multivariate Models Lifecycle</text>
+
+  <!-- VictoriaMetrics and query configuration -->
+  <rect x="535" y="175" width="420" height="215" class="box"/>
+  <text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
+  <text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
+  <rect x="270" y="270" width="150" height="155" class="box"/>
+  <text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
+  <text x="285" y="360" class="label">MetricsQL</text>
+  <text x="285" y="395" class="label">queries</text>
+  <path d="M420 346 H535" class="line"/>
+
+  <!-- Reader and datasource exchange -->
+  <rect x="580" y="500" width="310" height="100" class="box"/>
+  <text x="735" y="562" class="node" text-anchor="middle">Reader</text>
+  <path d="M580 545 H455 V365 H535" class="line"/>
+  <text x="465" y="478" class="label">1. Request data</text>
+  <path d="M955 270 H1040 V550 H890" class="line"/>
+  <text x="810" y="478" class="label">2. Get metrics</text>
+
+  <!-- Historical fit data returned by the configured queries -->
+  <g aria-label="Historical fit data">
+    <rect x="1090" y="85" width="430" height="505" class="group"/>
+    <rect x="1120" y="135" width="340" height="170" class="group"/>
+    <text x="1140" y="180" class="node">Query 1</text>
+    <rect x="1135" y="195" width="300" height="42" class="box"/>
+    <text x="1150" y="228" class="label">Metric 1.1</text>
+    <text x="1150" y="262" class="label">...</text>
+    <rect x="1135" y="265" width="300" height="42" class="box"/>
+    <text x="1150" y="298" class="label">Metric 1.M₁</text>
+    <text x="1305" y="355" class="node" text-anchor="middle">...</text>
+    <rect x="1120" y="390" width="340" height="170" class="group"/>
+    <text x="1140" y="435" class="node">Query N</text>
+    <rect x="1135" y="450" width="300" height="42" class="box"/>
+    <text x="1150" y="483" class="label">Metric N.1</text>
+    <text x="1150" y="517" class="label">...</text>
+    <rect x="1135" y="520" width="300" height="42" class="box"/>
+    <text x="1150" y="553" class="label">Metric N.Mₙ</text>
+  </g>
+
+  <!-- One shared model is fitted on the complete aligned set -->
+  <rect x="1215" y="795" width="310" height="115" class="box"/>
+  <text x="1370" y="865" class="node" text-anchor="middle">Model</text>
+  <rect x="1365" y="635" width="155" height="130" class="box"/>
+  <text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1382" y="723" class="label">Model</text>
+  <text x="1382" y="757" class="label">config</text>
+  <path d="M1305 590 V795" class="line"/>
+  <text x="1055" y="672" class="label">
+    <tspan x="1055" dy="0">3. Fit one model</tspan>
+    <tspan x="1055" dy="38">on all historical series</tspan>
+  </text>
+  <path d="M1443 765 V795" class="line"/>
+
+  <!-- Inference data must contain the same set of series -->
+  <g aria-label="Inference data">
+    <rect x="25" y="615" width="430" height="500" class="group"/>
+    <rect x="55" y="645" width="340" height="170" class="group"/>
+    <text x="75" y="690" class="node">Query 1</text>
+    <rect x="70" y="705" width="300" height="42" class="box"/>
+    <text x="85" y="738" class="label">Metric 1.1</text>
+    <text x="85" y="772" class="label">...</text>
+    <rect x="70" y="775" width="300" height="42" class="box"/>
+    <text x="85" y="808" class="label">Metric 1.M₁</text>
+    <text x="225" y="870" class="node" text-anchor="middle">...</text>
+    <rect x="55" y="900" width="340" height="175" class="group"/>
+    <text x="75" y="945" class="node">Query N</text>
+    <rect x="70" y="960" width="300" height="42" class="box"/>
+    <text x="85" y="993" class="label">Metric N.1</text>
+    <text x="85" y="1027" class="label">...</text>
+    <rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
+    <text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
+  </g>
+  <path d="M580 575 H500 V650 H455" class="line"/>
+  <text x="465" y="680" class="label">4. Provide inference data</text>
+
+  <!-- Single multivariate model registry -->
+  <g aria-label="Multivariate model registry">
+    <rect x="580" y="750" width="420" height="480" class="group"/>
+    <text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
+    <rect x="610" y="835" width="360" height="220" class="model-group"/>
+    <rect x="640" y="885" width="300" height="115" class="box"/>
+    <text x="790" y="955" class="node" text-anchor="middle">Model (single)</text>
+  </g>
+  <path d="M455 1050 H580" class="line"/>
+  <path d="M1215 852 H1000" class="line"/>
+
+  <!-- Joint output and mismatched-series skip path -->
+  <rect x="1600" y="1015" width="285" height="105" class="box"/>
+  <text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
+  <rect x="1740" y="805" width="145" height="145" class="box"/>
+  <text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1758" y="900" class="label">Writer</text>
+  <text x="1758" y="935" class="label">config</text>
+  <path d="M1813 950 V1015" class="line"/>
+  <path d="M1000 1040 H1600" class="line"/>
+  <text x="1295" y="965" class="label" text-anchor="middle">
+    <tspan x="1295" dy="0">5.a Produce one anomaly-score series</tspan>
+    <tspan x="1295" dy="38">with the intersected label set</tspan>
+  </text>
+  <path d="M1000 1110 H1600" class="blue-line"/>
+  <text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
+    <tspan x="1320" dy="0">5.b Skip inference when the fit and inference</tspan>
+    <tspan x="1320" dy="32">series sets differ;</tspan>
+    <tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
+  </text>
+
+  <!-- Persist the joint anomaly score -->
+  <path d="M1743 1015 V80 H745 V175" class="line"/>
+  <text x="1170" y="55" class="label" text-anchor="middle">6. Write one anomaly-score series (label set = intersection)</text>
+</svg>