build(deps): bump github/codeql-action/init from 4.36.3 to 4.37.0

Bumps [github/codeql-action/init](https://github.com/github/codeql-action) from 4.36.3 to 4.37.0.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](54f647b7e1...99df26d4f1)

---
updated-dependencies:
- dependency-name: github/codeql-action/init
  dependency-version: 4.37.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/codeql-analysis-go.yml

@@ -54,14 +54,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9  # v4.37.0
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/autobuild@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a  # v4.36.3
         with:
           category: 'language:go'

.github/workflows/vmui.yml

@@ -26,9 +26,7 @@ jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
     permissions:
-      checks: write
       contents: read
-      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
@@ -70,12 +68,6 @@ jobs:
         env:
           VMUI_SKIP_INSTALL: true
 
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
       - name: Check overall status
         run: |
           echo "Lint status: ${{ steps.lint.outcome }}"

app/vmagent/remotewrite/client.go

@@ -151,17 +151,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
+	rwURL, err := url.Parse(remoteWriteURL)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	retryMaxIntervalFlag := retryMaxTime
 	if retryMaxInterval.String() != "" {
 		retryMaxIntervalFlag = retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
+		remoteWriteURL:   rwURL.String(),
 		authCfg:          authCfg,
 		awsCfg:           awsCfg,
 		fq:               fq,

app/vmagent/remotewrite/obfuscate.go

@@ -0,0 +1,108 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+type obfuscateLabelsCtx struct {
+	labels []prompb.Label
+
+	// buf holds allocations for cached results below
+	buf []byte
+
+	// cacheResults maps original label values to their SHA-256 hex digests,
+	// avoiding redundant hashing of repeated values within a single batch.
+	cacheResults map[string]string
+}
+
+func (olctx *obfuscateLabelsCtx) reset() {
+	promrelabel.CleanLabels(olctx.labels)
+	olctx.labels = olctx.labels[:0]
+	clear(olctx.cacheResults)
+	olctx.buf = olctx.buf[:0]
+}
+
+var obfuscateLabelsCtxPool = &sync.Pool{
+	New: func() any {
+		return &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+	},
+}
+
+func getObfuscateLabelsCtx() *obfuscateLabelsCtx {
+	return obfuscateLabelsCtxPool.Get().(*obfuscateLabelsCtx)
+}
+
+func putObfuscateLabelsCtx(ctx *obfuscateLabelsCtx) {
+	ctx.reset()
+	obfuscateLabelsCtxPool.Put(ctx)
+}
+
+func (olctx *obfuscateLabelsCtx) obfuscate(tss []prompb.TimeSeries, obfuscateLabels []string) []prompb.TimeSeries {
+	if len(obfuscateLabels) == 0 || len(tss) == 0 {
+		return tss
+	}
+	labels := olctx.labels
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		found := false
+		for _, labelName := range obfuscateLabels {
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], labelName)
+			if tmp == nil {
+				continue
+			}
+			found = true
+			if obfuscatedValue, ok := olctx.cacheResults[tmp.Value]; ok {
+				// fast path: the obfuscated result was calculated before
+				tmp.Value = obfuscatedValue
+				continue
+			}
+
+			digest := sha256.Sum256(bytesutil.ToUnsafeBytes(tmp.Value))
+			buf := olctx.buf
+			bufLen := len(buf)
+			buf = hex.AppendEncode(buf, digest[:])
+			obfuscatedValue := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			olctx.buf = buf
+			olctx.cacheResults[tmp.Value] = obfuscatedValue
+			tmp.Value = obfuscatedValue
+		}
+		if found {
+			ts.Labels = labels[labelsLen:]
+		} else {
+			labels = labels[:labelsLen]
+		}
+	}
+	olctx.labels = labels
+	return tss
+}
+
+func (rwctx *remoteWriteCtx) initObfuscateLabels() {
+	if len(*obfuscateLabels) == 0 {
+		return
+	}
+	idx := rwctx.idx
+	rwObfuscateLabels := obfuscateLabels.GetOptionalArg(idx)
+	rwObfuscateLabelsList := strings.Split(rwObfuscateLabels, "^^")
+
+	for _, label := range rwObfuscateLabelsList {
+		if label == "" {
+			continue
+		}
+		if !slices.Contains(rwctx.obfuscateLabels, label) {
+			rwctx.obfuscateLabels = append(rwctx.obfuscateLabels, label)
+		}
+	}
+}

app/vmagent/remotewrite/obfuscate_test.go

@@ -0,0 +1,186 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func TestRemoteWriteObfuscateLabels(t *testing.T) {
+	f := func(obfuscateLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
+		t.Helper()
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		defer metrics.UnregisterAllMetrics()
+		originValue := *obfuscateLabels
+		defer func() {
+			*obfuscateLabels = originValue
+		}()
+		*obfuscateLabels = []string{obfuscateLabelList}
+		rwctx.initObfuscateLabels()
+
+		outputTss := olctx.obfuscate(inputTss, rwctx.obfuscateLabels)
+
+		if !reflect.DeepEqual(expectedTss, outputTss) {
+			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
+		}
+	}
+
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+
+	// 1. obfuscation is not set.
+	f("",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 1. obfuscation is set for another rwctx.
+	f(",ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 2. obfuscate the value of "ip" label
+	f("ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 3. obfuscate the values of "ip" and "instance"
+	f("ip^^instance",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: sha256Result("1234")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 4. duplicate label names in config must produce single SHA-256, not double
+	f("ip^^ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+}

app/vmagent/remotewrite/obfuscate_timing_test.go

@@ -0,0 +1,94 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func BenchmarkRemoteWriteObfuscateLabels(b *testing.B) {
+	originValue := *obfuscateLabels
+	defer func() {
+		*obfuscateLabels = originValue
+	}()
+	*obfuscateLabels = []string{"ip^^instance"}
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+	expected := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("123")},
+				{Name: "instance", Value: sha256Result("12345")},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("1236")},
+				{Name: "instance", Value: sha256Result("some-long-instante-string")},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	defer metrics.UnregisterAllMetrics()
+
+	inputTss := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "123"},
+				{Name: "instance", Value: "12345"},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "1236"},
+				{Name: "instance", Value: "some-long-instante-string"},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	b.RunParallel(func(pb *testing.PB) {
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		rwctx.initObfuscateLabels()
+		var localTss []prompb.TimeSeries
+		for pb.Next() {
+			// always make a shallow copy because obfuscate changes input
+			localTss = localTss[:0]
+			localTss = append(localTss, inputTss...)
+			olctx.reset()
+			outputTss := olctx.obfuscate(localTss, rwctx.obfuscateLabels)
+			if !reflect.DeepEqual(expected, outputTss) {
+				b.Fatalf("unexpected output: got: \n%v\n want: \n%v\n", outputTss, expected)
+			}
+		}
+	})
+
+}

app/vmagent/remotewrite/remotewrite.go

@@ -107,7 +107,12 @@ var (
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
 
 	enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
+		"Can be combined with -remoteWrite.obfuscateLabels to hide sensitive label values in the forwarded metrics. "+
 		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
+	obfuscateLabels = flagutil.NewArrayString("remoteWrite.obfuscateLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
+		"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
+		"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
 )
 
 var (
@@ -881,6 +886,8 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
 
+	obfuscateLabels []string
+
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
 	mdxRowsPreserved       *metrics.Counter
@@ -995,6 +1002,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 		rowsDroppedOnPushFailure:     metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
 	rwctx.initStreamAggrConfig()
+	rwctx.initObfuscateLabels()
 
 	if enableMdx.GetOptionalArg(argIdx) {
 		mdxFilter := mdx.NewFilter()
@@ -1198,24 +1206,41 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
 func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompb.TimeSeries
+	var olctx *obfuscateLabelsCtx
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if olctx != nil {
+			putObfuscateLabelsCtx(olctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
 
+	if len(rwctx.obfuscateLabels) != 0 {
+		copyTimeSeriesIfNeeded()
+		olctx = getObfuscateLabelsCtx()
+		tss = olctx.obfuscate(tss, rwctx.obfuscateLabels)
+	}
+
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -94,6 +95,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	hc := &http.Client{Transport: tr}
+	datasourceURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
+	}
+	hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
 
 	if extraParams == nil {
 		extraParams = url.Values{}
@@ -120,9 +127,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	}
 
 	return &Client{
-		c:                &http.Client{Transport: tr},
+		c:                hc,
 		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		datasourceURL:    strings.TrimSuffix(datasourceURL.String(), "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
 		extraParams:      extraParams,

app/vmalert/remoteread/init.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -76,7 +78,13 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	c := &http.Client{Transport: tr}
+	rrURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
+	}
 
+	c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +97,5 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
-	c := &http.Client{Transport: tr}
-	return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
+	return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Concurrency > 0 {
 		cc = cfg.Concurrency
 	}
+	hc := &http.Client{
+		Timeout:   *sendTimeout,
+		Transport: cfg.Transport,
+	}
+	rwURL, err := url.Parse(cfg.Addr)
+	if err != nil {
+		logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	c := &Client{
-		c: &http.Client{
-			Timeout:   *sendTimeout,
-			Transport: cfg.Transport,
-		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		c:             hc,
+		addr:          strings.TrimSuffix(rwURL.String(), "/"),
 		authCfg:       cfg.AuthCfg,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,

app/vmalert/rule/web.go

@@ -75,10 +75,13 @@ type GroupAlerts struct {
 // ApiRule represents a Rule for web view
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type ApiRule struct {
-	// State must be one of these under following scenarios
-	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing ruleState.
-	//  "firing": at least 1 alert in the rule in firing state.
-	//  "inactive": no alert in the rule in firing or pending state.
+	// Rule state must be one of these under following scenarios:
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing state. (only for alerting rules)
+	//  "firing": at least 1 alert in the rule in firing state. (only for alerting rules)
+	//  "inactive": rule's last evaluation was successful but no alert in the rule in firing or pending state. (only for alerting rules)
+	//  "unhealthy": rule's last evaluation was failed with error. (for both alerting and recording rules)
+	// 	"nomatch": rule's last evaluation was successful but no time series matched the rule's expression. (for both alerting and recording rules)
+	//  "ok": the recording rule's last evaluation was successful. (only for recording rules)
 	State string `json:"state"`
 	Name  string `json:"name"`
 	// Query represents Rule's `expression` field
@@ -237,6 +240,7 @@ func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 }
 
 func (r *ApiRule) ExtendState() {
+	// if alerting rule already has alerts, then state is already set to either "pending" or "firing" and we don't need to change it
 	if len(r.Alerts) > 0 {
 		return
 	}

app/vmalert/web.go

@@ -115,7 +115,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		WriteRule(w, r, rule)
 		return true
-	// current used by old vmalert UI and Grafana Alerts
+	// used by old vmalert UI
 	case "/vmalert/groups", "/rules":
 		rf, err := newRulesFilter(r)
 		if err != nil {
@@ -128,6 +128,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			state = rf.states[0]
 			rf.states = rf.states[:1]
 		}
+		// enable extendedStates by default for vmalert UI
+		rf.extendedStates = true
 		lr := rh.groups(rf)
 		WriteListGroups(w, r, lr.Data.Groups, state)
 		return true
@@ -543,6 +545,8 @@ func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 			if !groupFound && !strings.Contains(strings.ToLower(rule.Name), rf.search) {
 				continue
 			}
+			// extendedStates is used by the vmalert UI to extend the rule state with values such as "nomatch" and "unhealthy".
+			// those states are not supported by Grafana and not part of the Prometheus API spec yet
 			if rf.extendedStates {
 				rule.ExtendState()
 			}

app/vmalert/web.qtpl

@@ -130,9 +130,12 @@
                             data-bs-target="#item-{%s g.ID %}"
                         >
                             <span class="d-flex gap-2">
-                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.States["unhealthy"] %}</span> {% endif %}
-                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.States["nomatch"] %}</span> {% endif %}
-                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.States["ok"] %}</span>
+                                {% if g.States["inactive"] > 0 %}<span class="badge bg-light text-success border border-success" title="None of the alert instances is in a pending or firing state">{%d g.States["inactive"] %} inactive</span> {% endif %}
+                                {% if g.States["pending"] > 0 %}<span class="badge bg-warning text-dark" title="At least one alert instance is pending">{%d g.States["pending"] %} pending</span> {% endif %}
+                                {% if g.States["firing"] > 0 %}<span class="badge bg-danger" title="At least one alert instance is firing">{%d g.States["firing"] %} firing</span> {% endif %}
+                                {% if g.States["ok"] > 0 %}<span class="badge bg-success" title="Recording rule last evaluation succeeded">{%d g.States["ok"] %} ok</span>{% endif %}
+                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Last evaluation failed with an error">{%d g.States["unhealthy"] %} unhealthy</span> {% endif %}
+                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Rule expression matched no time series">{%d g.States["nomatch"] %} nomatch</span> {% endif %}
                             </span>
                         </span>
                     </span>
@@ -169,7 +172,7 @@
                             <thead>
                                 <tr>
                                     <th scope="col" class="w-60">Rule</th>
-                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
+                                    <th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series Returned</th>
                                     <th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
                                 </tr>
                             </thead>
@@ -188,8 +191,21 @@
                                                     {% else %}
                                                         <b>record:</b> {%s r.Name %}
                                                     {% endif %}
-                                                    |
+                                                    {% if r.State == "inactive" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "pending" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "firing" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "ok" %}
+                                                        <span><a class="badge bg-success">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "unhealthy" %}
+                                                        <span><a class="badge bg-danger">{%s r.State %}</a></span>
+                                                    {% elseif  r.State == "nomatch" %}
+                                                        <span><a class="badge bg-warning">{%s r.State %}</a></span>
+                                                    {% endif %}
                                                     {%= seriesFetchedWarn(prefix, &r) %}
+                                                    |
                                                     <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
                                                 </div>
                                                 <div class="col-12">

app/vmauth/auth_config.go

@@ -73,15 +73,22 @@ type UserInfo struct {
 	Username    string     `yaml:"username,omitempty"`
 	Password    string     `yaml:"password,omitempty"`
 
-	URLPrefix              *URLPrefix      `yaml:"url_prefix,omitempty"`
-	URLMaps                []URLMap        `yaml:"url_map,omitempty"`
-	DumpRequestOnErrors    bool            `yaml:"dump_request_on_errors,omitempty"`
-	HeadersConf            HeadersConf     `yaml:",inline"`
-	BackendSettings        BackendSettings `yaml:",inline"`
-	DefaultURL             *URLPrefix      `yaml:"default_url,omitempty"`
-	RetryStatusCodes       []int           `yaml:"retry_status_codes,omitempty"`
-	MergeQueryArgs         []string        `yaml:"merge_query_args,omitempty"`
-	DropSrcPathPrefixParts *int            `yaml:"drop_src_path_prefix_parts,omitempty"`
+	URLPrefix              *URLPrefix  `yaml:"url_prefix,omitempty"`
+	DiscoverBackendIPs     *bool       `yaml:"discover_backend_ips,omitempty"`
+	URLMaps                []URLMap    `yaml:"url_map,omitempty"`
+	DumpRequestOnErrors    bool        `yaml:"dump_request_on_errors,omitempty"`
+	HeadersConf            HeadersConf `yaml:",inline"`
+	MaxConcurrentRequests  int         `yaml:"max_concurrent_requests,omitempty"`
+	DefaultURL             *URLPrefix  `yaml:"default_url,omitempty"`
+	RetryStatusCodes       []int       `yaml:"retry_status_codes,omitempty"`
+	LoadBalancingPolicy    string      `yaml:"load_balancing_policy,omitempty"`
+	MergeQueryArgs         []string    `yaml:"merge_query_args,omitempty"`
+	DropSrcPathPrefixParts *int        `yaml:"drop_src_path_prefix_parts,omitempty"`
+	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
+	TLSCertFile            string      `yaml:"tls_cert_file,omitempty"`
+	TLSKeyFile             string      `yaml:"tls_key_file,omitempty"`
+	TLSServerName          string      `yaml:"tls_server_name,omitempty"`
+	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
 
 	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
 
@@ -147,19 +154,6 @@ type HeadersConf struct {
 	hasAnyPlaceHolders bool
 }
 
-// BackendSettings holds settings shared between UserInfo and BackendGroupConfig for controlling
-// how vmauth selects and connects to backends.
-type BackendSettings struct {
-	LoadBalancingPolicy   string `yaml:"load_balancing_policy,omitempty"`
-	DiscoverBackendIPs    *bool  `yaml:"discover_backend_ips,omitempty"`
-	MaxConcurrentRequests int    `yaml:"max_concurrent_requests,omitempty"`
-	TLSCAFile             string `yaml:"tls_ca_file,omitempty"`
-	TLSCertFile           string `yaml:"tls_cert_file,omitempty"`
-	TLSKeyFile            string `yaml:"tls_key_file,omitempty"`
-	TLSServerName         string `yaml:"tls_server_name,omitempty"`
-	TLSInsecureSkipVerify *bool  `yaml:"tls_insecure_skip_verify,omitempty"`
-}
-
 func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	select {
 	case ui.concurrencyLimitCh <- struct{}{}:
@@ -191,7 +185,7 @@ func (ui *UserInfo) endConcurrencyLimit() {
 }
 
 func (ui *UserInfo) getMaxConcurrentRequests() int {
-	mcr := ui.BackendSettings.MaxConcurrentRequests
+	mcr := ui.MaxConcurrentRequests
 	if mcr <= 0 {
 		mcr = *maxConcurrentPerUserRequests
 	}
@@ -346,30 +340,20 @@ type URLPrefix struct {
 	// how many request path prefix parts to drop before routing the request to backendURL
 	dropSrcPathPrefixParts int
 
-	// busOriginal contains the original list of backend groups specified in yaml config.
-	busOriginal []*backendGroupSpec
+	// busOriginal contains the original list of backends specified in yaml config.
+	busOriginal []*url.URL
 
 	// n is an atomic counter, which is used for balancing load among available backends.
 	n atomic.Uint32
 
-	// backendGroupCounters holds one atomic counter per busOriginal entry, used for balancing load.
-	backendGroupCounters []atomic.Uint32
-
 	// the list of backend urls
 	//
 	// the list can be dynamically updated if `discover_backend_ips` option is set.
 	bus atomic.Pointer[backendURLs]
 
-	// if this option is set by default, then backend ips for busOriginal are periodically re-discovered and put to bus.
-	//
-	// individual busOriginal entries may override this via their own discover_backend_ips setting.
+	// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
 	discoverBackendIPs bool
 
-	// hasAnyBackendDiscovery is true if discovery is effectively enabled for at least one busOriginal entry.
-	// It is computed once in sanitizeAndInitialize, so discoverBackendAddrsIfNeeded can cheaply skip
-	// the whole discovery machinery on the common no-discovery path.
-	hasAnyBackendDiscovery bool
-
 	// The next deadline for DNS-based discovery of backend IPs
 	nextDiscoveryDeadline atomic.Uint64
 
@@ -377,208 +361,21 @@ type URLPrefix struct {
 	vOriginal any
 }
 
-// backendGroupSpec represents a single parsed `url_prefix` list item.
-//
-// It is built either from a plain url string (no overrides, inherits everything from the
-// enclosing scope) or from a BackendGroupConfig mapping.
-type backendGroupSpec struct {
-	// name identifies this group in metrics. Falls back to its ordinal position in url_prefix when empty.
-	name string
-
-	urls []*url.URL
-
-	// per-group overrides; zero values mean "inherit from the enclosing url_prefix / user / url_map scope".
-	loadBalancingPolicy   string
-	discoverBackendIPs    *bool
-	maxConcurrentRequests int
-	tlsCAFile             string
-	tlsCertFile           string
-	tlsKeyFile            string
-	tlsServerName         string
-	tlsInsecureSkipVerify *bool
-}
-
-func (spec *backendGroupSpec) hasTLSOverride() bool {
-	return spec.tlsCAFile != "" || spec.tlsCertFile != "" || spec.tlsKeyFile != "" ||
-		spec.tlsServerName != "" || spec.tlsInsecureSkipVerify != nil
-}
-
-// BackendGroupConfig represents a `url_prefix` list item specified as a YAML mapping instead of a plain string.
-//
-// It lets a group of backend urls override load_balancing_policy, discover_backend_ips,
-// max_concurrent_requests and tls_* settings, which are otherwise inherited from the enclosing
-// `user` / `url_map` scope. See https://docs.victoriametrics.com/victoriametrics/vmauth/#load-balancing
-type BackendGroupConfig struct {
-	// Name optionally identifies this backend group in metrics. Falls back to its ordinal
-	// position in url_prefix when not set.
-	Name            string          `yaml:"name,omitempty"`
-	URLPrefix       stringOrSlice   `yaml:"url_prefix"`
-	BackendSettings BackendSettings `yaml:",inline"`
-}
-
-// stringOrSlice unmarshals a YAML value that is either a single string or a list of strings.
-type stringOrSlice []string
-
-func (s *stringOrSlice) UnmarshalYAML(f func(any) error) error {
-	var v any
-	if err := f(&v); err != nil {
-		return err
-	}
-	urls, err := parseStringOrSliceValue(v)
-	if err != nil {
-		return fmt.Errorf("cannot unmarshal `url_prefix`: %w", err)
-	}
-	*s = urls
-	return nil
-}
-
-func parseStringOrSliceValue(v any) ([]string, error) {
-	switch x := v.(type) {
-	case string:
-		return []string{x}, nil
-	case []any:
-		if len(x) == 0 {
-			return nil, fmt.Errorf("must contain at least a single url")
-		}
-		us := make([]string, len(x))
-		for i, xx := range x {
-			s, ok := xx.(string)
-			if !ok {
-				return nil, fmt.Errorf("must contain array of strings; got %T", xx)
-			}
-			us[i] = s
-		}
-		return us, nil
-	default:
-		return nil, fmt.Errorf("unexpected type: %T; want string or []string", v)
-	}
-}
-
-func validateLoadBalancingPolicyValue(policy string) error {
-	switch policy {
+func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
+	switch loadBalancingPolicy {
 	case "", // empty string is equivalent to least_loaded
 		"least_loaded",
 		"first_available":
+		up.loadBalancingPolicy = loadBalancingPolicy
 		return nil
 	default:
-		return fmt.Errorf("unexpected load_balancing_policy: %q; want least_loaded or first_available", policy)
+		return fmt.Errorf("unexpected load_balancing_policy: %q; want least_loaded or first_available", loadBalancingPolicy)
 	}
 }
 
-func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
-	if err := validateLoadBalancingPolicyValue(loadBalancingPolicy); err != nil {
-		return err
-	}
-	up.loadBalancingPolicy = loadBalancingPolicy
-	return nil
-}
-
-// backendUserSettings holds the resolved user-level settings needed for building backendURLGroups.
-//
-// It lets URLPrefix.sanitizeAndInitialize apply per-group overrides (load_balancing_policy, tls_*,
-// max_concurrent_requests) on top of the settings inherited from the enclosing user.
-type backendUserSettings struct {
-	tlsCAFile             string
-	tlsCertFile           string
-	tlsKeyFile            string
-	tlsServerName         string
-	tlsInsecureSkipVerify *bool
-
-	ms           *metrics.Set
-	metricLabels string
-}
-
-func backendGroupMetricLabels(userLabels, groupID string) string {
-	label := fmt.Sprintf(`backend_group=%q`, groupID)
-	if userLabels == "" {
-		return "{" + label + "}"
-	}
-	return userLabels[:len(userLabels)-1] + "," + label + "}"
-}
-
 type backendURLs struct {
-	bhc    backendHealthCheck
-	n      *atomic.Uint32
-	groups []*backendURLGroup
-}
-
-// backendURLGroup holds the backend urls discovered for a single busOriginal entry.
-type backendURLGroup struct {
-	// n is an atomic counter, which is used for balancing load among bus.
-	//
-	// It points into URLPrefix.backendGroupCounters, so it survives across the
-	// group being recreated by backend IP rediscovery.
-	n *atomic.Uint32
-
+	bhc backendHealthCheck
 	bus []*backendURL
-
-	loadBalancingPolicy string
-
-	// rt is a per-group HTTP RoundTripper. nil means inherit the enclosing user's RoundTripper.
-	rt http.RoundTripper
-
-	// concurrencyLimitCh is a per-group concurrency limiter. nil means no group-level limit is configured.
-	concurrencyLimitCh      chan struct{}
-	concurrencyLimitReached *metrics.Counter
-}
-
-func (g *backendURLGroup) isBroken() bool {
-	for _, bu := range g.bus {
-		if !bu.isBroken() {
-			return false
-		}
-	}
-	return true
-}
-
-func (g *backendURLGroup) isAtConcurrencyLimit() bool {
-	return g.concurrencyLimitCh != nil && len(g.concurrencyLimitCh) >= cap(g.concurrencyLimitCh)
-}
-
-func (g *backendURLGroup) isUnavailable() bool {
-	return g.isBroken() || g.isAtConcurrencyLimit()
-}
-
-func (g *backendURLGroup) minConcurrentRequests() int32 {
-	minReqs := int32(math.MaxInt32)
-	for _, bu := range g.bus {
-		if bu.isBroken() {
-			continue
-		}
-		if n := bu.concurrentRequests.Load(); n < minReqs {
-			minReqs = n
-		}
-	}
-	return minReqs
-}
-
-func (g *backendURLGroup) getBackendURL() *backendURL {
-	if g.loadBalancingPolicy == "first_available" {
-		return g.getFirstAvailable()
-	}
-	return g.getLeastLoaded()
-}
-
-func (g *backendURLGroup) beginConcurrencyLimit() bool {
-	if g.concurrencyLimitCh == nil {
-		return true
-	}
-	select {
-	case g.concurrencyLimitCh <- struct{}{}:
-		return true
-	default:
-		if g.concurrencyLimitReached != nil {
-			g.concurrencyLimitReached.Inc()
-		}
-		return false
-	}
-}
-
-func (g *backendURLGroup) endConcurrencyLimit() {
-	if g.concurrencyLimitCh == nil {
-		return
-	}
-	<-g.concurrencyLimitCh
 }
 
 type backendHealthCheck struct {
@@ -607,34 +404,22 @@ func (bhc *backendHealthCheck) stop() {
 	bhc.wg.Wait()
 }
 
-func newBackendURLs(n *atomic.Uint32) *backendURLs {
+func newBackendURLs() *backendURLs {
 	ctx, cancel := context.WithCancel(context.Background())
 	return &backendURLs{
 		bhc: backendHealthCheck{
 			ctx:    ctx,
 			cancel: cancel,
 		},
-		n: n,
 	}
 }
 
-// addGroup appends a new backendURLGroup to bus, containing a backendURL for every url in urls,
-// and returns the newly created group so the caller can apply per-group settings to it.
-func (bus *backendURLs) addGroup(urls []*url.URL, n *atomic.Uint32) *backendURLGroup {
-	g := &backendURLGroup{
-		n:   n,
-		bus: make([]*backendURL, len(urls)),
-	}
-	for i, u := range urls {
-		g.bus[i] = &backendURL{
-			url:             u,
-			bhc:             &bus.bhc,
-			hasPlaceHolders: hasAnyPlaceholders(u),
-			group:           g,
-		}
-	}
-	bus.groups = append(bus.groups, g)
-	return g
+func (bus *backendURLs) add(u *url.URL) {
+	bus.bus = append(bus.bus, &backendURL{
+		url:             u,
+		bhc:             &bus.bhc,
+		hasPlaceHolders: hasAnyPlaceholders(u),
+	})
 }
 
 func (bus *backendURLs) stopHealthChecks() {
@@ -651,8 +436,6 @@ type backendURL struct {
 	url *url.URL
 
 	hasPlaceHolders bool
-
-	group *backendURLGroup
 }
 
 func (bu *backendURL) isBroken() bool {
@@ -712,11 +495,7 @@ func (bu *backendURL) put() {
 
 func (up *URLPrefix) getBackendsCount() int {
 	bus := up.bus.Load()
-	n := 0
-	for _, g := range bus.groups {
-		n += len(g.bus)
-	}
-	return n
+	return len(bus.bus)
 }
 
 // getBackendURL returns the backendURL depending on the load balance policy.
@@ -728,34 +507,19 @@ func (up *URLPrefix) getBackendURL() *backendURL {
 	up.discoverBackendAddrsIfNeeded()
 
 	bus := up.bus.Load()
-	if len(bus.groups) == 0 {
+	if len(bus.bus) == 0 {
 		return nil
 	}
 
-	var g *backendURLGroup
 	if up.loadBalancingPolicy == "first_available" {
-		g = bus.getFirstAvailable()
-	} else {
-		g = bus.getLeastLoaded()
+		return getFirstAvailableBackendURL(bus.bus)
 	}
-	if len(g.bus) == 0 {
-		return nil
-	}
-
-	return g.getBackendURL()
-}
-
-// effectiveDiscoverBackendIPs returns whether backend IP discovery is enabled for spec,
-// taking into account spec's own override of up.discoverBackendIPs.
-func (up *URLPrefix) effectiveDiscoverBackendIPs(spec *backendGroupSpec) bool {
-	if spec.discoverBackendIPs != nil {
-		return *spec.discoverBackendIPs
-	}
-	return up.discoverBackendIPs
+	return getLeastLoadedBackendURL(bus.bus, &up.n)
 }
 
 func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
-	if !up.hasAnyBackendDiscovery {
+	if !up.discoverBackendIPs {
+		// The discovery is disabled.
 		return
 	}
 
@@ -776,89 +540,66 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 		return
 	}
 
-	// Discover ips for all the backendURLs which need it.
+	// Discover ips for all the backendURLs
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(intervalSec))
 	hostToAddrs := make(map[string][]string)
-	for _, spec := range up.busOriginal {
-		if !up.effectiveDiscoverBackendIPs(spec) {
+	for _, bu := range up.busOriginal {
+		host := bu.Hostname()
+		port := bu.Port()
+		if hostToAddrs[host] != nil {
+			// ips for the given host have been already discovered
 			continue
 		}
-		for _, bu := range spec.urls {
-			host := bu.Hostname()
-			port := bu.Port()
-			if hostToAddrs[host] != nil {
-				// ips for the given host have been already discovered
-				continue
-			}
 
-			var resolvedAddrs []string
-			if strings.HasPrefix(host, "srv+") {
-				// The host has the format 'srv+realhost'. Strip 'srv+' prefix before performing the lookup.
-				srvHost := strings.TrimPrefix(host, "srv+")
-				_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", srvHost)
-				if err != nil {
-					logger.Warnf("cannot discover backend SRV records for %s: %s; use it literally", bu, err)
-					resolvedAddrs = []string{host}
-				} else {
-					resolvedAddrs = make([]string, len(addrs))
-					for i, addr := range addrs {
-						hostPort := port
-						if hostPort == "" && addr.Port > 0 {
-							hostPort = strconv.FormatUint(uint64(addr.Port), 10)
-						}
-						resolvedAddrs[i] = net.JoinHostPort(addr.Target, hostPort)
-					}
-				}
+		var resolvedAddrs []string
+		if strings.HasPrefix(host, "srv+") {
+			// The host has the format 'srv+realhost'. Strip 'srv+' prefix before performing the lookup.
+			srvHost := strings.TrimPrefix(host, "srv+")
+			_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", srvHost)
+			if err != nil {
+				logger.Warnf("cannot discover backend SRV records for %s: %s; use it literally", bu, err)
+				resolvedAddrs = []string{host}
 			} else {
-				addrs, err := netutil.Resolver.LookupIPAddr(ctx, host)
-				if err != nil {
-					logger.Warnf("cannot discover backend IPs for %s: %s; use it literally", bu, err)
-					resolvedAddrs = []string{host}
-				} else {
-					resolvedAddrs = make([]string, len(addrs))
-					for i, addr := range addrs {
-						resolvedAddrs[i] = net.JoinHostPort(addr.String(), port)
+				resolvedAddrs = make([]string, len(addrs))
+				for i, addr := range addrs {
+					hostPort := port
+					if hostPort == "" && addr.Port > 0 {
+						hostPort = strconv.FormatUint(uint64(addr.Port), 10)
 					}
-				}
-			}
-			// sort resolvedAddrs, so they could be compared below in areEqualBackendURLGroups()
-			sort.Strings(resolvedAddrs)
-			hostToAddrs[host] = resolvedAddrs
-		}
-	}
-	cancel()
-
-	// generate new backendURLs for the resolved IPs, one group per busOriginal entry
-	oldGroups := up.bus.Load().groups
-	busNew := newBackendURLs(&up.n)
-	for i, spec := range up.busOriginal {
-		var urls []*url.URL
-		if up.effectiveDiscoverBackendIPs(spec) {
-			for _, bu := range spec.urls {
-				host := bu.Hostname()
-				for _, addr := range hostToAddrs[host] {
-					buCopy := *bu
-					buCopy.Host = addr
-					urls = append(urls, &buCopy)
+					resolvedAddrs[i] = net.JoinHostPort(addr.Target, hostPort)
 				}
 			}
 		} else {
-			urls = spec.urls
+			addrs, err := netutil.Resolver.LookupIPAddr(ctx, host)
+			if err != nil {
+				logger.Warnf("cannot discover backend IPs for %s: %s; use it literally", bu, err)
+				resolvedAddrs = []string{host}
+			} else {
+				resolvedAddrs = make([]string, len(addrs))
+				for i, addr := range addrs {
+					resolvedAddrs[i] = net.JoinHostPort(addr.String(), port)
+				}
+			}
 		}
-		g := busNew.addGroup(urls, &up.backendGroupCounters[i])
-		if i < len(oldGroups) {
-			// Per-group settings are static for the lifetime of this URLPrefix - carry them over
-			// instead of recomputing, so a per-group RoundTripper / concurrency limiter isn't
-			// rebuilt (and in-flight concurrency accounting isn't lost) on every rediscovery.
-			g.loadBalancingPolicy = oldGroups[i].loadBalancingPolicy
-			g.rt = oldGroups[i].rt
-			g.concurrencyLimitCh = oldGroups[i].concurrencyLimitCh
-			g.concurrencyLimitReached = oldGroups[i].concurrencyLimitReached
+		// sort resolvedAddrs, so they could be compared below in areEqualBackendURLs()
+		sort.Strings(resolvedAddrs)
+		hostToAddrs[host] = resolvedAddrs
+	}
+	cancel()
+
+	// generate new backendURLs for the resolved IPs
+	busNew := newBackendURLs()
+	for _, bu := range up.busOriginal {
+		host := bu.Hostname()
+		for _, addr := range hostToAddrs[host] {
+			buCopy := *bu
+			buCopy.Host = addr
+			busNew.add(&buCopy)
 		}
 	}
 
 	bus := up.bus.Load()
-	if areEqualBackendURLGroups(bus.groups, busNew.groups) {
+	if areEqualBackendURLs(bus.bus, busNew.bus) {
 		return
 	}
 
@@ -867,18 +608,6 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	bus.stopHealthChecks()
 }
 
-func areEqualBackendURLGroups(a, b []*backendURLGroup) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i, g := range a {
-		if !areEqualBackendURLs(g.bus, b[i].bus) {
-			return false
-		}
-	}
-	return true
-}
-
 func areEqualBackendURLs(a, b []*backendURL) bool {
 	if len(a) != len(b) {
 		return false
@@ -892,12 +621,11 @@ func areEqualBackendURLs(a, b []*backendURL) bool {
 	return true
 }
 
-// getFirstAvailable returns the first available backendURL in g, which isn't broken.
-// If all backendURLs in g are broken, then returns the first one.
+// getFirstAvailableBackendURL returns the first available backendURL, which isn't broken.
+// If all backendURLs are broken, then returns the first backendURL.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
-func (g *backendURLGroup) getFirstAvailable() *backendURL {
-	bus := g.bus
+func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 	bu := bus[0]
 	if !bu.isBroken() {
 		// Fast path - send the request to the first url.
@@ -920,10 +648,11 @@ func (g *backendURLGroup) getFirstAvailable() *backendURL {
 	return bu
 }
 
-func (g *backendURLGroup) getLeastLoaded() *backendURL {
-	bus := g.bus
-	atomicCounter := g.n
-
+// getLeastLoadedBackendURL returns a non-broken backendURL with the lowest number of concurrent requests.
+// If all backendURLs are broken, then returns the first backendURL.
+//
+// backendURL.put() must be called on the returned backendURL after the request is complete.
+func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
 	firstBu := bus[0]
 	if len(bus) == 1 {
 		firstBu.get()
@@ -975,62 +704,6 @@ func (g *backendURLGroup) getLeastLoaded() *backendURL {
 	return buMin
 }
 
-// getFirstAvailable returns the first backendURLGroup in bus, which has at least a single non-broken backend url.
-// If all groups are fully broken, then returns the first one.
-func (bus *backendURLs) getFirstAvailable() *backendURLGroup {
-	groups := bus.groups
-	g := groups[0]
-	if !g.isUnavailable() {
-		// Fast path - use the first group.
-		return g
-	}
-
-	// Slow path - the first group is temporarily unavailable. Fall back to the remaining groups.
-	for i := 1; i < len(groups); i++ {
-		if !groups[i].isUnavailable() {
-			return groups[i]
-		}
-	}
-
-	// All groups are unavailable, then return the first one, it could help increase the success rate of the requests.
-	return g
-}
-
-// getLeastLoaded returns a non-broken backendURLGroup in bus with the lowest number of concurrent requests
-// among its non-broken backend urls. If all groups are broken, then returns the first one.
-func (bus *backendURLs) getLeastLoaded() *backendURLGroup {
-	groups := bus.groups
-	atomicCounter := bus.n
-	firstGroup := groups[0]
-	if len(groups) == 1 {
-		return firstGroup
-	}
-
-	n := atomicCounter.Add(1) - 1
-	gMinIdx := n % uint32(len(groups))
-	minRequests := groups[gMinIdx].minConcurrentRequests()
-	for i := uint32(1); i < uint32(len(groups)); i++ {
-		idx := (n + i) % uint32(len(groups))
-		g := groups[idx]
-		if g.isUnavailable() {
-			continue
-		}
-
-		reqs := g.minConcurrentRequests()
-		if reqs < minRequests || groups[gMinIdx].isUnavailable() {
-			gMinIdx = idx
-			minRequests = reqs
-		}
-	}
-	gMin := groups[gMinIdx]
-	if gMin.isUnavailable() {
-		// If all groups are unavailable, then returns the first group.
-		return firstGroup
-	}
-	atomicCounter.CompareAndSwap(n+1, gMinIdx+1)
-	return gMin
-}
-
 // UnmarshalYAML unmarshals up from yaml.
 func (up *URLPrefix) UnmarshalYAML(f func(any) error) error {
 	var v any
@@ -1039,77 +712,37 @@ func (up *URLPrefix) UnmarshalYAML(f func(any) error) error {
 	}
 	up.vOriginal = v
 
-	var items []any
+	var urls []string
 	switch x := v.(type) {
 	case string:
-		items = []any{x}
+		urls = []string{x}
 	case []any:
 		if len(x) == 0 {
 			return fmt.Errorf("`url_prefix` must contain at least a single url")
 		}
-		items = x
-	default:
-		return fmt.Errorf("unexpected type for `url_prefix`: %T; want string, []string or a list containing backend group mappings", v)
-	}
-
-	specs := make([]*backendGroupSpec, len(items))
-	for i, item := range items {
-		spec, err := parseBackendGroupSpecItem(item)
-		if err != nil {
-			return fmt.Errorf("cannot unmarshal `url_prefix` item #%d: %w", i+1, err)
-		}
-		specs[i] = spec
-	}
-	up.busOriginal = specs
-	return nil
-}
-
-func parseBackendGroupSpecItem(item any) (*backendGroupSpec, error) {
-	switch x := item.(type) {
-	case string:
-		pu, err := url.Parse(x)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unmarshal %q into url: %w", x, err)
-		}
-		return &backendGroupSpec{urls: []*url.URL{pu}}, nil
-	case map[interface{}]interface{}:
-		data, err := yaml.Marshal(x)
-		if err != nil {
-			return nil, fmt.Errorf("cannot re-marshal backend group mapping: %w", err)
-		}
-		var bgc BackendGroupConfig
-		if err := yaml.UnmarshalStrict(data, &bgc); err != nil {
-			return nil, fmt.Errorf("cannot unmarshal backend group mapping: %w", err)
-		}
-		if len(bgc.URLPrefix) == 0 {
-			return nil, fmt.Errorf("missing `url_prefix` in backend group mapping")
-		}
-		if err := validateLoadBalancingPolicyValue(bgc.BackendSettings.LoadBalancingPolicy); err != nil {
-			return nil, err
-		}
-		urls := make([]*url.URL, len(bgc.URLPrefix))
-		for i, u := range bgc.URLPrefix {
-			pu, err := url.Parse(u)
-			if err != nil {
-				return nil, fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
+		us := make([]string, len(x))
+		for i, xx := range x {
+			s, ok := xx.(string)
+			if !ok {
+				return fmt.Errorf("`url_prefix` must contain array of strings; got %T", xx)
 			}
-			urls[i] = pu
+			us[i] = s
 		}
-		return &backendGroupSpec{
-			name:                  bgc.Name,
-			urls:                  urls,
-			loadBalancingPolicy:   bgc.BackendSettings.LoadBalancingPolicy,
-			discoverBackendIPs:    bgc.BackendSettings.DiscoverBackendIPs,
-			maxConcurrentRequests: bgc.BackendSettings.MaxConcurrentRequests,
-			tlsCAFile:             bgc.BackendSettings.TLSCAFile,
-			tlsCertFile:           bgc.BackendSettings.TLSCertFile,
-			tlsKeyFile:            bgc.BackendSettings.TLSKeyFile,
-			tlsServerName:         bgc.BackendSettings.TLSServerName,
-			tlsInsecureSkipVerify: bgc.BackendSettings.TLSInsecureSkipVerify,
-		}, nil
+		urls = us
 	default:
-		return nil, fmt.Errorf("unexpected type for `url_prefix` item: %T; want a string or a mapping with url_prefix/load_balancing_policy/discover_backend_ips/etc", item)
+		return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v)
 	}
+
+	bus := make([]*url.URL, len(urls))
+	for i, u := range urls {
+		pu, err := url.Parse(u)
+		if err != nil {
+			return fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
+		}
+		bus[i] = pu
+	}
+	up.busOriginal = bus
+	return nil
 }
 
 // MarshalYAML marshals up to yaml.
@@ -1364,7 +997,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		}
 
 		if ui.hasAnyURLs() {
-			if err := ui.initURLs(ac.ms); err != nil {
+			if err := ui.initURLs(); err != nil {
 				return nil, err
 			}
 		}
@@ -1387,7 +1020,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
-		rt, err := newRoundTripper(ui.BackendSettings)
+		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
 		}
@@ -1426,7 +1059,7 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
 			return nil, err
 		}
-		if err := ui.initURLs(ac.ms); err != nil {
+		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
 
@@ -1449,7 +1082,7 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
-		rt, err := newRoundTripper(ui.BackendSettings)
+		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
 		}
@@ -1485,7 +1118,7 @@ func (ui *UserInfo) getMetricLabels() (string, error) {
 	return labelsStr, nil
 }
 
-func (ui *UserInfo) initURLs(ms *metrics.Set) error {
+func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
 	mergeQueryArgs := *defaultMergeQueryArgs
@@ -1494,8 +1127,8 @@ func (ui *UserInfo) initURLs(ms *metrics.Set) error {
 	if ui.RetryStatusCodes != nil {
 		retryStatusCodes = ui.RetryStatusCodes
 	}
-	if ui.BackendSettings.LoadBalancingPolicy != "" {
-		loadBalancingPolicy = ui.BackendSettings.LoadBalancingPolicy
+	if ui.LoadBalancingPolicy != "" {
+		loadBalancingPolicy = ui.LoadBalancingPolicy
 	}
 	if len(ui.MergeQueryArgs) != 0 {
 		mergeQueryArgs = ui.MergeQueryArgs
@@ -1503,26 +1136,15 @@ func (ui *UserInfo) initURLs(ms *metrics.Set) error {
 	if ui.DropSrcPathPrefixParts != nil {
 		dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts
 	}
-	if ui.BackendSettings.DiscoverBackendIPs != nil {
-		discoverBackendIPs = *ui.BackendSettings.DiscoverBackendIPs
-	}
-
-	metricLabels, err := ui.getMetricLabels()
-	if err != nil {
-		return err
-	}
-	userSettings := backendUserSettings{
-		tlsCAFile:             ui.BackendSettings.TLSCAFile,
-		tlsCertFile:           ui.BackendSettings.TLSCertFile,
-		tlsKeyFile:            ui.BackendSettings.TLSKeyFile,
-		tlsServerName:         ui.BackendSettings.TLSServerName,
-		tlsInsecureSkipVerify: ui.BackendSettings.TLSInsecureSkipVerify,
-		ms:                    ms,
-		metricLabels:          metricLabels,
+	if ui.DiscoverBackendIPs != nil {
+		discoverBackendIPs = *ui.DiscoverBackendIPs
 	}
 
 	up := ui.URLPrefix
 	if up != nil {
+		if err := up.sanitizeAndInitialize(); err != nil {
+			return err
+		}
 		up.retryStatusCodes = retryStatusCodes
 		up.dropSrcPathPrefixParts = dropSrcPathPrefixParts
 		up.discoverBackendIPs = discoverBackendIPs
@@ -1530,12 +1152,9 @@ func (ui *UserInfo) initURLs(ms *metrics.Set) error {
 			return err
 		}
 		up.mergeQueryArgs = mergeQueryArgs
-		if err := up.sanitizeAndInitialize(userSettings); err != nil {
-			return err
-		}
 	}
 	if ui.DefaultURL != nil {
-		if err := ui.DefaultURL.sanitizeAndInitialize(userSettings); err != nil {
+		if err := ui.DefaultURL.sanitizeAndInitialize(); err != nil {
 			return err
 		}
 	}
@@ -1547,6 +1166,9 @@ func (ui *UserInfo) initURLs(ms *metrics.Set) error {
 		if e.URLPrefix == nil {
 			return fmt.Errorf("missing `url_prefix` in `url_map`")
 		}
+		if err := e.URLPrefix.sanitizeAndInitialize(); err != nil {
+			return err
+		}
 		rscs := retryStatusCodes
 		lbp := loadBalancingPolicy
 		mqa := mergeQueryArgs
@@ -1574,9 +1196,6 @@ func (ui *UserInfo) initURLs(ms *metrics.Set) error {
 		e.URLPrefix.mergeQueryArgs = mqa
 		e.URLPrefix.dropSrcPathPrefixParts = dsp
 		e.URLPrefix.discoverBackendIPs = dbd
-		if err := e.URLPrefix.sanitizeAndInitialize(userSettings); err != nil {
-			return err
-		}
 	}
 	if len(ui.URLMaps) == 0 && ui.URLPrefix == nil {
 		return fmt.Errorf("missing `url_prefix` or `url_map`")
@@ -1679,89 +1298,19 @@ func getAuthTokensFromRequest(r *http.Request) []string {
 	return ats
 }
 
-// sanitizeAndInitialize validates up.busOriginal and (re)initializes up.bus from it,
-// applying per-group overrides on top of the settings inherited from userSettings.
-func (up *URLPrefix) sanitizeAndInitialize(userSettings backendUserSettings) error {
-	for _, spec := range up.busOriginal {
-		for i, bu := range spec.urls {
-			puNew, err := sanitizeURLPrefix(bu)
-			if err != nil {
-				return err
-			}
-			spec.urls[i] = puNew
+func (up *URLPrefix) sanitizeAndInitialize() error {
+	for i, bu := range up.busOriginal {
+		puNew, err := sanitizeURLPrefix(bu)
+		if err != nil {
+			return err
 		}
+		up.busOriginal[i] = puNew
 	}
 
-	up.backendGroupCounters = make([]atomic.Uint32, len(up.busOriginal))
-
-	up.hasAnyBackendDiscovery = false
-	for _, spec := range up.busOriginal {
-		if up.effectiveDiscoverBackendIPs(spec) {
-			up.hasAnyBackendDiscovery = true
-			break
-		}
-	}
-
-	// Initialize up.bus with a single group per busOriginal entry.
-	bus := newBackendURLs(&up.n)
-	for i, spec := range up.busOriginal {
-		g := bus.addGroup(spec.urls, &up.backendGroupCounters[i])
-
-		lbp := spec.loadBalancingPolicy
-		if lbp == "" {
-			lbp = up.loadBalancingPolicy
-		}
-		g.loadBalancingPolicy = lbp
-
-		if spec.hasTLSOverride() {
-			bs := BackendSettings{
-				TLSCAFile:             spec.tlsCAFile,
-				TLSCertFile:           spec.tlsCertFile,
-				TLSKeyFile:            spec.tlsKeyFile,
-				TLSServerName:         spec.tlsServerName,
-				TLSInsecureSkipVerify: spec.tlsInsecureSkipVerify,
-			}
-			if bs.TLSCAFile == "" {
-				bs.TLSCAFile = userSettings.tlsCAFile
-			}
-			if bs.TLSCertFile == "" {
-				bs.TLSCertFile = userSettings.tlsCertFile
-			}
-			if bs.TLSKeyFile == "" {
-				bs.TLSKeyFile = userSettings.tlsKeyFile
-			}
-			if bs.TLSServerName == "" {
-				bs.TLSServerName = userSettings.tlsServerName
-			}
-			if bs.TLSInsecureSkipVerify == nil {
-				bs.TLSInsecureSkipVerify = userSettings.tlsInsecureSkipVerify
-			}
-
-			rt, err := newRoundTripper(bs)
-			if err != nil {
-				return fmt.Errorf("cannot initialize HTTP RoundTripper for a backend group at `url_prefix` item #%d: %w", i+1, err)
-			}
-			g.rt = rt
-		}
-
-		if spec.maxConcurrentRequests > 0 {
-			g.concurrencyLimitCh = make(chan struct{}, spec.maxConcurrentRequests)
-			if userSettings.ms != nil {
-				groupID := spec.name
-				if groupID == "" {
-					groupID = strconv.Itoa(i)
-				}
-				groupLabels := backendGroupMetricLabels(userSettings.metricLabels, groupID)
-				g.concurrencyLimitReached = userSettings.ms.GetOrCreateCounter(`vmauth_backend_group_concurrent_requests_limit_reached_total` + groupLabels)
-				ch := g.concurrencyLimitCh
-				_ = userSettings.ms.GetOrCreateGauge(`vmauth_backend_group_concurrent_requests_capacity`+groupLabels, func() float64 {
-					return float64(cap(ch))
-				})
-				_ = userSettings.ms.GetOrCreateGauge(`vmauth_backend_group_concurrent_requests_current`+groupLabels, func() float64 {
-					return float64(len(ch))
-				})
-			}
-		}
+	// Initialize up.bus
+	bus := newBackendURLs()
+	for _, bu := range up.busOriginal {
+		bus.add(bu)
 	}
 	up.bus.Store(bus)
 

app/vmauth/auth_config_test.go

@@ -2,13 +2,11 @@ package main
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"net"
 	"net/http"
 	"net/url"
 	"strings"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -367,13 +365,11 @@ users:
   tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
 		getHTTPAuthBasicToken("foo", "bar"): {
-			Username:  "foo",
-			Password:  "bar",
-			URLPrefix: mustParseURL("http://aaa:343/bbb"),
-			BackendSettings: BackendSettings{
-				MaxConcurrentRequests: 5,
-				TLSInsecureSkipVerify: &insecureSkipVerifyTrue,
-			},
+			Username:              "foo",
+			Password:              "bar",
+			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
+			MaxConcurrentRequests: 5,
+			TLSInsecureSkipVerify: &insecureSkipVerifyTrue,
 		},
 	}, nil)
 
@@ -390,16 +386,14 @@ users:
   tls_key_file: "foo/foo"
 `, map[string]*UserInfo{
 		getHTTPAuthToken("foo"): {
-			AuthToken: "foo",
-			URLPrefix: mustParseURL("https://aaa:343/bbb"),
-			BackendSettings: BackendSettings{
-				MaxConcurrentRequests: 5,
-				TLSInsecureSkipVerify: &insecureSkipVerifyTrue,
-				TLSServerName:         "foo.bar",
-				TLSCAFile:             "foo/bar",
-				TLSCertFile:           "foo/baz",
-				TLSKeyFile:            "foo/foo",
-			},
+			AuthToken:             "foo",
+			URLPrefix:             mustParseURL("https://aaa:343/bbb"),
+			MaxConcurrentRequests: 5,
+			TLSInsecureSkipVerify: &insecureSkipVerifyTrue,
+			TLSServerName:         "foo.bar",
+			TLSCAFile:             "foo/bar",
+			TLSCertFile:           "foo/baz",
+			TLSKeyFile:            "foo/foo",
 		},
 	}, nil)
 
@@ -427,14 +421,12 @@ users:
 				"http://node1:343/bbb",
 				"http://srv+node2:343/bbb",
 			}),
-			BackendSettings: BackendSettings{
-				TLSInsecureSkipVerify: &insecureSkipVerifyFalse,
-				LoadBalancingPolicy:   "first_available",
-				DiscoverBackendIPs:    &discoverBackendIPsTrue,
-			},
+			TLSInsecureSkipVerify:  &insecureSkipVerifyFalse,
 			RetryStatusCodes:       []int{500, 501},
+			LoadBalancingPolicy:    "first_available",
 			MergeQueryArgs:         []string{"foo", "bar"},
 			DropSrcPathPrefixParts: new(1),
+			DiscoverBackendIPs:     &discoverBackendIPsTrue,
 		},
 	}, nil)
 
@@ -744,11 +736,11 @@ unauthorized_user:
 	}
 
 	ui := m[getHTTPAuthBasicToken("foo", "bar")]
-	if !isSetBool(ui.BackendSettings.TLSInsecureSkipVerify, true) {
+	if !isSetBool(ui.TLSInsecureSkipVerify, true) {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
 	}
 
-	if !isSetBool(ac.UnauthorizedUser.BackendSettings.TLSInsecureSkipVerify, false) {
+	if !isSetBool(ac.UnauthorizedUser.TLSInsecureSkipVerify, false) {
 		t.Fatalf("unexpected TLSInsecureSkipVerify value for unauthorized_user")
 	}
 }
@@ -849,7 +841,7 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	up.loadBalancingPolicy = "least_loaded"
 
 	pbus := up.bus.Load()
-	bus := pbus.groups[0].bus
+	bus := pbus.bus
 
 	fn := func(ns ...int) {
 		t.Helper()
@@ -921,7 +913,7 @@ func TestBrokenBackend(t *testing.T) {
 	})
 	up.loadBalancingPolicy = "least_loaded"
 	pbus := up.bus.Load()
-	bus := pbus.groups[0].bus
+	bus := pbus.bus
 
 	// explicitly mark one of the backends as broken
 	bus[1].setBroken()
@@ -940,12 +932,11 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 		t.Helper()
 		up := mustParseURL(actualUrl)
 		up.discoverBackendIPs = true
-		up.hasAnyBackendDiscovery = true
 		up.loadBalancingPolicy = "least_loaded"
 
 		up.discoverBackendAddrsIfNeeded()
 		pbus := up.bus.Load()
-		bus := pbus.groups[0].bus
+		bus := pbus.bus
 
 		if len(bus) != 1 {
 			t.Fatalf("expected url list to be of size 1; got %d instead", len(bus))
@@ -1005,114 +996,6 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 
 }
 
-func TestAreEqualBackendURLGroups(t *testing.T) {
-	newGroup := func(hosts ...string) *backendURLGroup {
-		g := &backendURLGroup{}
-		for _, h := range hosts {
-			g.bus = append(g.bus, &backendURL{url: &url.URL{Host: h}})
-		}
-		return g
-	}
-
-	f := func(a, b []*backendURLGroup, expected bool) {
-		t.Helper()
-		if got := areEqualBackendURLGroups(a, b); got != expected {
-			t.Fatalf("unexpected result; got %v; want %v", got, expected)
-		}
-	}
-
-	// identical grouping
-	f(
-		[]*backendURLGroup{newGroup("10.0.0.1", "10.0.0.2"), newGroup("10.0.0.3")},
-		[]*backendURLGroup{newGroup("10.0.0.1", "10.0.0.2"), newGroup("10.0.0.3")},
-		true,
-	)
-
-	// different number of groups
-	f(
-		[]*backendURLGroup{newGroup("10.0.0.1")},
-		[]*backendURLGroup{newGroup("10.0.0.1"), newGroup("10.0.0.2")},
-		false,
-	)
-
-	// the flattened address sequence is unchanged, but an address moved across the group boundary
-	f(
-		[]*backendURLGroup{newGroup("10.0.0.1", "10.0.0.2"), newGroup("10.0.0.3")},
-		[]*backendURLGroup{newGroup("10.0.0.1"), newGroup("10.0.0.2", "10.0.0.3")},
-		false,
-	)
-
-	// genuinely different content
-	f(
-		[]*backendURLGroup{newGroup("10.0.0.1")},
-		[]*backendURLGroup{newGroup("10.0.0.9")},
-		false,
-	)
-}
-
-func TestDiscoverBackendAddrsIfNeededDetectsGroupBoundaryShift(t *testing.T) {
-	customResolver := &fakeResolver{
-		Resolver: &net.Resolver{},
-		lookupIPAddrResults: map[string][]net.IPAddr{
-			"zonea": {
-				{IP: net.ParseIP("10.0.0.1")},
-				{IP: net.ParseIP("10.0.0.2")},
-			},
-			"zoneb": {
-				{IP: net.ParseIP("10.0.0.3")},
-			},
-		},
-	}
-	origResolver := netutil.Resolver
-	netutil.Resolver = customResolver
-	defer func() {
-		netutil.Resolver = origResolver
-	}()
-
-	up := &URLPrefix{}
-	up.busOriginal = []*backendGroupSpec{
-		{urls: []*url.URL{{Scheme: "http", Host: "zonea"}}},
-		{urls: []*url.URL{{Scheme: "http", Host: "zoneb"}}},
-	}
-	up.backendGroupCounters = make([]atomic.Uint32, 2)
-	bus0 := newBackendURLs(&up.n)
-	bus0.addGroup(up.busOriginal[0].urls, &up.backendGroupCounters[0])
-	bus0.addGroup(up.busOriginal[1].urls, &up.backendGroupCounters[1])
-	up.bus.Store(bus0)
-	up.discoverBackendIPs = true
-	up.hasAnyBackendDiscovery = true
-
-	up.discoverBackendAddrsIfNeeded()
-	bus := up.bus.Load()
-	if len(bus.groups) != 2 || len(bus.groups[0].bus) != 2 || len(bus.groups[1].bus) != 1 {
-		t.Fatalf("unexpected initial grouping: zonea=%d zoneb=%d", len(bus.groups[0].bus), len(bus.groups[1].bus))
-	}
-
-	// Simulate 10.0.0.2 moving from zonea to zoneb. The flattened address sequence
-	// stays [10.0.0.1, 10.0.0.2, 10.0.0.3], but the group boundaries shift, so the
-	// rediscovery must still be picked up.
-	customResolver.lookupIPAddrResults["zonea"] = []net.IPAddr{
-		{IP: net.ParseIP("10.0.0.1")},
-	}
-	customResolver.lookupIPAddrResults["zoneb"] = []net.IPAddr{
-		{IP: net.ParseIP("10.0.0.2")},
-		{IP: net.ParseIP("10.0.0.3")},
-	}
-	up.nextDiscoveryDeadline.Store(0)
-	up.discoverBackendAddrsIfNeeded()
-
-	bus = up.bus.Load()
-	if len(bus.groups) != 2 {
-		t.Fatalf("unexpected groups count after rediscovery: %d", len(bus.groups))
-	}
-	if len(bus.groups[0].bus) != 1 || bus.groups[0].bus[0].url.Host != "10.0.0.1:" {
-		t.Fatalf("unexpected zonea group after rediscovery: %v", bus.groups[0].bus)
-	}
-	if len(bus.groups[1].bus) != 2 {
-		t.Fatalf("unexpected zoneb group size after rediscovery; got %d; want 2", len(bus.groups[1].bus))
-	}
-}
-
 func TestLogRequest(t *testing.T) {
 	ui := &UserInfo{AccessLog: &AccessLog{}}
 
@@ -1158,8 +1041,7 @@ func TestGetFirstAvailableBackend(t *testing.T) {
 			}
 			bus[i].broken.Store(broken[i])
 		}
-		g := &backendURLGroup{bus: bus}
-		bu := g.getFirstAvailable()
+		bu := getFirstAvailableBackendURL(bus)
 		if bu == nil {
 			t.Fatalf("unexpected nil backend")
 		}
@@ -1176,355 +1058,6 @@ func TestGetFirstAvailableBackend(t *testing.T) {
 
 }
 
-func newTestGroup(broken ...bool) *backendURLGroup {
-	ctx, cancel := context.WithCancel(context.Background())
-	bhc := &backendHealthCheck{
-		ctx:    ctx,
-		cancel: cancel,
-	}
-	g := &backendURLGroup{
-		bus: make([]*backendURL, len(broken)),
-	}
-	for i, b := range broken {
-		g.bus[i] = &backendURL{
-			url: &url.URL{Host: fmt.Sprintf("target-%d", i)},
-			bhc: bhc,
-		}
-		g.bus[i].broken.Store(b)
-	}
-	return g
-}
-
-func TestGetFirstAvailableGroup(t *testing.T) {
-	// the first group is available
-	bus := &backendURLs{groups: []*backendURLGroup{newTestGroup(false), newTestGroup(false)}}
-	if g := bus.getFirstAvailable(); g != bus.groups[0] {
-		t.Fatalf("expecting the first group to be returned")
-	}
-
-	// the first group is fully broken - falls back to the second one
-	bus = &backendURLs{groups: []*backendURLGroup{newTestGroup(true), newTestGroup(false)}}
-	if g := bus.getFirstAvailable(); g != bus.groups[1] {
-		t.Fatalf("expecting the second group to be returned")
-	}
-
-	// the first group has a non-broken target - it is still considered available
-	bus = &backendURLs{groups: []*backendURLGroup{newTestGroup(true, false), newTestGroup(false)}}
-	if g := bus.getFirstAvailable(); g != bus.groups[0] {
-		t.Fatalf("expecting the first group to be returned, since it has a non-broken target")
-	}
-
-	// all groups are fully broken - the first one is returned
-	bus = &backendURLs{groups: []*backendURLGroup{newTestGroup(true), newTestGroup(true)}}
-	if g := bus.getFirstAvailable(); g != bus.groups[0] {
-		t.Fatalf("expecting the first group to be returned when all groups are broken")
-	}
-}
-
-func TestGetLeastLoadedGroup(t *testing.T) {
-	g0 := newTestGroup(false)
-	g1 := newTestGroup(false)
-	var n atomic.Uint32
-	bus := &backendURLs{groups: []*backendURLGroup{g0, g1}, n: &n}
-
-	// both groups are idle - the first one is picked
-	if g := bus.getLeastLoaded(); g != g0 {
-		t.Fatalf("expecting g0 to be returned when all groups are idle")
-	}
-
-	// g0 becomes more loaded than g1 - g1 must be picked
-	g0.bus[0].concurrentRequests.Add(5)
-	if g := bus.getLeastLoaded(); g != g1 {
-		t.Fatalf("expecting g1 to be returned when it is less loaded than g0")
-	}
-
-	// g1 is broken - g0 must be picked despite being more loaded
-	g1.bus[0].setBroken()
-	if g := bus.getLeastLoaded(); g != g0 {
-		t.Fatalf("expecting g0 to be returned when g1 is broken")
-	}
-
-	// all groups are broken - the first one is returned
-	g0.bus[0].setBroken()
-	if g := bus.getLeastLoaded(); g != g0 {
-		t.Fatalf("expecting g0 to be returned when all groups are broken")
-	}
-}
-
-func TestGetBackendURLTwoTier(t *testing.T) {
-	// Simulates two backends (busOriginal entries), each expanded into two discovered targets.
-	up := &URLPrefix{}
-	up.backendGroupCounters = make([]atomic.Uint32, 2)
-	bus := newBackendURLs(&up.n)
-	g0 := bus.addGroup([]*url.URL{{Host: "g0-a"}, {Host: "g0-b"}}, &up.backendGroupCounters[0])
-	g0.loadBalancingPolicy = "first_available"
-	g1 := bus.addGroup([]*url.URL{{Host: "g1-a"}, {Host: "g1-b"}}, &up.backendGroupCounters[1])
-	g1.loadBalancingPolicy = "first_available"
-
-	up.bus.Store(bus)
-	up.loadBalancingPolicy = "first_available"
-
-	if n := up.getBackendsCount(); n != 4 {
-		t.Fatalf("unexpected backends count; got %d; want 4", n)
-	}
-
-	// Both policies are first_available, so vmauth must always pick group0's first target.
-	for range 5 {
-		bu := up.getBackendURL()
-		if bu.url.Host != "g0-a" {
-			t.Fatalf("unexpected target; got %q; want g0-a", bu.url.Host)
-		}
-		bu.put()
-	}
-
-	// Mark group0's first target broken - the backend-level policy falls back to g0-b,
-	// while the top-level group selection still prefers group0, since it has a non-broken target.
-	bus.groups[0].bus[0].setBroken()
-	bu := up.getBackendURL()
-	if bu.url.Host != "g0-b" {
-		t.Fatalf("unexpected target; got %q; want g0-b", bu.url.Host)
-	}
-	bu.put()
-
-	// Mark group0 fully broken - the top-level selection falls back to group1.
-	bus.groups[0].bus[1].setBroken()
-	bu = up.getBackendURL()
-	if bu.url.Host != "g1-a" {
-		t.Fatalf("unexpected target; got %q; want g1-a", bu.url.Host)
-	}
-	bu.put()
-}
-
-func TestURLPrefixUnmarshalYAMLBackendGroup(t *testing.T) {
-	var up URLPrefix
-	data := []byte(`
-- http://plain-backend
-- url_prefix:
-  - http://group-a
-  - http://group-b
-  load_balancing_policy: least_loaded
-  discover_backend_ips: true
-  max_concurrent_requests: 7
-  tls_insecure_skip_verify: true
-`)
-	if err := yaml.Unmarshal(data, &up); err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	if len(up.busOriginal) != 2 {
-		t.Fatalf("unexpected number of busOriginal entries; got %d; want 2", len(up.busOriginal))
-	}
-
-	spec0 := up.busOriginal[0]
-	if len(spec0.urls) != 1 || spec0.urls[0].String() != "http://plain-backend" {
-		t.Fatalf("unexpected spec0 urls: %v", spec0.urls)
-	}
-	if spec0.loadBalancingPolicy != "" || spec0.maxConcurrentRequests != 0 || spec0.hasTLSOverride() {
-		t.Fatalf("unexpected overrides on a plain string spec: %+v", spec0)
-	}
-
-	spec1 := up.busOriginal[1]
-	if len(spec1.urls) != 2 || spec1.urls[0].String() != "http://group-a" || spec1.urls[1].String() != "http://group-b" {
-		t.Fatalf("unexpected spec1 urls: %v", spec1.urls)
-	}
-	if spec1.loadBalancingPolicy != "least_loaded" {
-		t.Fatalf("unexpected spec1.loadBalancingPolicy; got %q; want %q", spec1.loadBalancingPolicy, "least_loaded")
-	}
-	if spec1.discoverBackendIPs == nil || !*spec1.discoverBackendIPs {
-		t.Fatalf("expecting spec1.discoverBackendIPs=true")
-	}
-	if spec1.maxConcurrentRequests != 7 {
-		t.Fatalf("unexpected spec1.maxConcurrentRequests; got %d; want 7", spec1.maxConcurrentRequests)
-	}
-	if spec1.tlsInsecureSkipVerify == nil || !*spec1.tlsInsecureSkipVerify {
-		t.Fatalf("expecting spec1.tlsInsecureSkipVerify=true")
-	}
-}
-
-func TestURLPrefixUnmarshalYAMLBackendGroupInvalidPolicy(t *testing.T) {
-	var up URLPrefix
-	data := []byte(`
-- url_prefix: http://a
-  load_balancing_policy: bogus
-`)
-	if err := yaml.Unmarshal(data, &up); err == nil {
-		t.Fatalf("expecting non-nil error for invalid load_balancing_policy in a backend group")
-	}
-}
-
-func TestURLPrefixUnmarshalYAMLBackendGroupMissingURLPrefix(t *testing.T) {
-	var up URLPrefix
-	data := []byte(`
-- discover_backend_ips: true
-`)
-	if err := yaml.Unmarshal(data, &up); err == nil {
-		t.Fatalf("expecting non-nil error for missing url_prefix in a backend group mapping")
-	}
-}
-
-func TestSanitizeAndInitializeBackendGroupOverrides(t *testing.T) {
-	data := []byte(`
-users:
-- username: foo
-  password: bar
-  load_balancing_policy: first_available
-  url_prefix:
-  - http://primary-a
-  - url_prefix:
-    - http://standby-a
-    - http://standby-b
-    load_balancing_policy: least_loaded
-    discover_backend_ips: true
-    max_concurrent_requests: 5
-    tls_insecure_skip_verify: true
-`)
-	ac, err := parseAuthConfig(data)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	m, err := parseAuthConfigUsers(ac)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	var ui *UserInfo
-	for _, u := range m {
-		ui = u
-	}
-	if ui == nil {
-		t.Fatalf("expecting a single parsed user")
-	}
-
-	up := ui.URLPrefix
-	bus := up.bus.Load()
-	if len(bus.groups) != 2 {
-		t.Fatalf("unexpected number of groups; got %d; want 2", len(bus.groups))
-	}
-
-	g0 := bus.groups[0]
-	if g0.loadBalancingPolicy != "first_available" {
-		t.Fatalf("unexpected g0 loadBalancingPolicy; got %q; want %q (inherited)", g0.loadBalancingPolicy, "first_available")
-	}
-	if g0.rt != nil {
-		t.Fatalf("expecting g0.rt to be nil (no per-group tls override)")
-	}
-	if g0.concurrencyLimitCh != nil {
-		t.Fatalf("expecting g0.concurrencyLimitCh to be nil (no per-group max_concurrent_requests)")
-	}
-
-	g1 := bus.groups[1]
-	if g1.loadBalancingPolicy != "least_loaded" {
-		t.Fatalf("unexpected g1 loadBalancingPolicy; got %q; want %q (overridden)", g1.loadBalancingPolicy, "least_loaded")
-	}
-	if g1.rt == nil {
-		t.Fatalf("expecting g1.rt to be non-nil due to tls_insecure_skip_verify override")
-	}
-	if cap(g1.concurrencyLimitCh) != 5 {
-		t.Fatalf("unexpected g1 concurrency limit capacity; got %d; want 5", cap(g1.concurrencyLimitCh))
-	}
-	if len(g1.bus) != 2 {
-		t.Fatalf("unexpected g1 backend count; got %d; want 2", len(g1.bus))
-	}
-	if !up.hasAnyBackendDiscovery {
-		t.Fatalf("expecting hasAnyBackendDiscovery=true, since g1 overrides discover_backend_ips=true")
-	}
-
-	// g1 has no explicit `name`, so its metrics must fall back to its ordinal position (index 1).
-	wantCounter := ac.ms.GetOrCreateCounter(`vmauth_backend_group_concurrent_requests_limit_reached_total{username="foo",backend_group="1"}`)
-	if wantCounter != g1.concurrencyLimitReached {
-		t.Fatalf("g1's concurrency limit metric wasn't registered with the expected ordinal backend_group label")
-	}
-}
-
-func TestSanitizeAndInitializeBackendGroupName(t *testing.T) {
-	data := []byte(`
-users:
-- username: foo
-  password: bar
-  url_prefix:
-  - url_prefix: http://primary-a
-    name: primary
-    max_concurrent_requests: 3
-`)
-	ac, err := parseAuthConfig(data)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	m, err := parseAuthConfigUsers(ac)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	var ui *UserInfo
-	for _, u := range m {
-		ui = u
-	}
-	if ui == nil {
-		t.Fatalf("expecting a single parsed user")
-	}
-
-	g0 := ui.URLPrefix.bus.Load().groups[0]
-	wantCounter := ac.ms.GetOrCreateCounter(`vmauth_backend_group_concurrent_requests_limit_reached_total{username="foo",backend_group="primary"}`)
-	if wantCounter != g0.concurrencyLimitReached {
-		t.Fatalf("g0's concurrency limit metric wasn't registered with the explicit backend_group=\"primary\" label")
-	}
-}
-
-func TestBackendURLGroupConcurrencyLimit(t *testing.T) {
-	g := &backendURLGroup{
-		concurrencyLimitCh: make(chan struct{}, 1),
-	}
-	if !g.beginConcurrencyLimit() {
-		t.Fatalf("expecting first beginConcurrencyLimit to succeed")
-	}
-	if g.beginConcurrencyLimit() {
-		t.Fatalf("expecting second beginConcurrencyLimit to fail since the limit is 1")
-	}
-	if !g.isAtConcurrencyLimit() {
-		t.Fatalf("expecting isAtConcurrencyLimit to be true")
-	}
-	g.endConcurrencyLimit()
-	if g.isAtConcurrencyLimit() {
-		t.Fatalf("expecting isAtConcurrencyLimit to be false after endConcurrencyLimit")
-	}
-	if !g.beginConcurrencyLimit() {
-		t.Fatalf("expecting beginConcurrencyLimit to succeed again after endConcurrencyLimit")
-	}
-}
-
-func TestBackendURLGroupConcurrencyLimitDisabled(t *testing.T) {
-	g := &backendURLGroup{}
-	for range 10 {
-		if !g.beginConcurrencyLimit() {
-			t.Fatalf("expecting beginConcurrencyLimit to always succeed when no limit is configured")
-		}
-	}
-	// must not panic/block when no limit is configured
-	g.endConcurrencyLimit()
-}
-
-func TestGetFirstAvailableSkipsConcurrencyLimitedGroup(t *testing.T) {
-	g0 := newTestGroup(false)
-	g0.concurrencyLimitCh = make(chan struct{}, 1)
-	g0.concurrencyLimitCh <- struct{}{} // saturate g0
-	g1 := newTestGroup(false)
-	bus := &backendURLs{groups: []*backendURLGroup{g0, g1}}
-
-	if g := bus.getFirstAvailable(); g != g1 {
-		t.Fatalf("expecting g1 to be returned since g0 is at its concurrency limit")
-	}
-}
-
-func TestGetLeastLoadedSkipsConcurrencyLimitedGroup(t *testing.T) {
-	g0 := newTestGroup(false)
-	g0.concurrencyLimitCh = make(chan struct{}, 1)
-	g0.concurrencyLimitCh <- struct{}{} // saturate g0
-	g1 := newTestGroup(false)
-	var n atomic.Uint32
-	bus := &backendURLs{groups: []*backendURLGroup{g0, g1}, n: &n}
-
-	if g := bus.getLeastLoaded(); g != g1 {
-		t.Fatalf("expecting g1 to be returned since g0 is at its concurrency limit")
-	}
-}
-
 func getRegexs(paths []string) []*Regex {
 	var sps []*Regex
 	for _, path := range paths {
@@ -1559,12 +1092,14 @@ func mustParseURL(u string) *URLPrefix {
 }
 
 func mustParseURLs(us []string) *URLPrefix {
+	bus := newBackendURLs()
 	urls := make([]*url.URL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
 			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
 		}
+		bus.add(pu)
 		urls[i] = pu
 	}
 	up := &URLPrefix{}
@@ -1573,11 +1108,8 @@ func mustParseURLs(us []string) *URLPrefix {
 	} else {
 		up.vOriginal = us
 	}
-	up.busOriginal = []*backendGroupSpec{{urls: urls}}
-	up.backendGroupCounters = make([]atomic.Uint32, 1)
-	bus := newBackendURLs(&up.n)
-	bus.addGroup(urls, &up.backendGroupCounters[0])
 	up.bus.Store(bus)
+	up.busOriginal = urls
 	return up
 }
 

app/vmauth/jwt.go

@@ -163,7 +163,7 @@ func parseJWTUsers(ac *AuthConfig, oidcDP *oidcDiscovererPool) ([]*UserInfo, err
 			return nil, err
 		}
 
-		if err := ui.initURLs(ac.ms); err != nil {
+		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
 
@@ -186,7 +186,7 @@ func parseJWTUsers(ac *AuthConfig, oidcDP *oidcDiscovererPool) ([]*UserInfo, err
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
-		rt, err := newRoundTripper(ui.BackendSettings)
+		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
 		}
@@ -422,32 +422,30 @@ func parseJWTPlaceholdersForUserInfo(ui *UserInfo, isAllowed bool) error {
 }
 
 func validateJWTPlaceholdersForURL(up *URLPrefix, isAllowed bool) error {
-	for _, spec := range up.busOriginal {
-		for _, bu := range spec.urls {
-			ok := strings.Contains(bu.Path, placeholderPrefix)
-			if ok && !isAllowed {
-				return fmt.Errorf("placeholder: %q is only allowed at JWT token context", bu.Path)
+	for _, bu := range up.busOriginal {
+		ok := strings.Contains(bu.Path, placeholderPrefix)
+		if ok && !isAllowed {
+			return fmt.Errorf("placeholder: %q is only allowed at JWT token context", bu.Path)
+		}
+		if ok {
+			p := bu.Path
+			for _, ph := range allPlaceholders {
+				p = strings.ReplaceAll(p, ph, ``)
 			}
-			if ok {
-				p := bu.Path
-				for _, ph := range allPlaceholders {
-					p = strings.ReplaceAll(p, ph, ``)
-				}
-				if strings.Contains(p, placeholderPrefix) {
-					return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
-				}
+			if strings.Contains(p, placeholderPrefix) {
+				return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
 			}
-			for param, values := range bu.Query() {
-				for _, value := range values {
-					ok := strings.Contains(value, placeholderPrefix)
-					if ok && !isAllowed {
-						return fmt.Errorf("query param: %q with placeholder: %q is only allowed at JWT token context", param, value)
-					}
-					if ok {
-						// possible placeholder
-						if !slices.Contains(allPlaceholders, value) {
-							return fmt.Errorf("query param: %q has unsupported placeholder string: %q, supported values are: %s", param, value, strings.Join(allPlaceholders, ", "))
-						}
+		}
+		for param, values := range bu.Query() {
+			for _, value := range values {
+				ok := strings.Contains(value, placeholderPrefix)
+				if ok && !isAllowed {
+					return fmt.Errorf("query param: %q with placeholder: %q is only allowed at JWT token context", param, value)
+				}
+				if ok {
+					// possible placeholder
+					if !slices.Contains(allPlaceholders, value) {
+						return fmt.Errorf("query param: %q has unsupported placeholder string: %q, supported values are: %s", param, value, strings.Join(allPlaceholders, ", "))
 					}
 				}
 			}

app/vmauth/main.go

@@ -431,11 +431,6 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		if bu == nil {
 			break
 		}
-		if !bu.group.beginConcurrencyLimit() {
-			// The backend group hit its own max_concurrent_requests limit - skip it and try another backend.
-			bu.put()
-			continue
-		}
 		targetURL := bu.url
 		if tkn != nil {
 			vmac := tkn.VMAccess()
@@ -464,7 +459,6 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 			goto again
 		}
 
-		bu.group.endConcurrencyLimit()
 		bu.put()
 		if ok {
 			return
@@ -499,11 +493,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	bb, bbOK := req.Body.(*bufferedBody)
 	canRetry := !bbOK || bb.canRetry()
 
-	rt := bu.group.rt
-	if rt == nil {
-		rt = ui.rt
-	}
-	res, err := rt.RoundTrip(req)
+	res, err := ui.rt.RoundTrip(req)
 	if err == nil {
 		defer func() { _ = res.Body.Close() }()
 	}
@@ -721,25 +711,25 @@ var (
 	bufferRequestBodyDuration = metrics.NewSummary(`vmauth_buffer_request_body_duration_seconds`)
 )
 
-func newRoundTripper(bs BackendSettings) (http.RoundTripper, error) {
+func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
 	caFile := *backendTLSCAFile
-	if bs.TLSCAFile != "" {
-		caFile = bs.TLSCAFile
+	if caFileOpt != "" {
+		caFile = caFileOpt
 	}
 	certFile := *backendTLSCertFile
-	if bs.TLSCertFile != "" {
-		certFile = bs.TLSCertFile
+	if certFileOpt != "" {
+		certFile = certFileOpt
 	}
 	keyFile := *backendTLSKeyFile
-	if bs.TLSKeyFile != "" {
-		keyFile = bs.TLSKeyFile
+	if keyFileOpt != "" {
+		keyFile = keyFileOpt
 	}
 	serverName := *backendTLSServerName
-	if bs.TLSServerName != "" {
-		serverName = bs.TLSServerName
+	if serverNameOpt != "" {
+		serverName = serverNameOpt
 	}
 	insecureSkipVerify := *backendTLSInsecureSkipVerify
-	if p := bs.TLSInsecureSkipVerify; p != nil {
+	if p := insecureSkipVerifyP; p != nil {
 		insecureSkipVerify = *p
 	}
 	opts := &promauth.Options{

app/vmauth/main_test.go

@@ -590,65 +590,6 @@ X-Forwarded-For: 12.34.56.78, 42.2.3.84`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 }
 
-func TestRequestHandlerSkipsBackendGroupAtConcurrencyLimit(t *testing.T) {
-	tsA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "backend=A")
-	}))
-	defer tsA.Close()
-	tsB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "backend=B")
-	}))
-	defer tsB.Close()
-
-	cfgStr := fmt.Sprintf(`
-unauthorized_user:
-  load_balancing_policy: first_available
-  url_prefix:
-  - url_prefix: %s/foo
-    max_concurrent_requests: 1
-  - %s/foo
-`, tsA.URL, tsB.URL)
-
-	cfgOrigP := authConfigData.Load()
-	if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
-		t.Fatalf("cannot load config data: %s", err)
-	}
-	defer func() {
-		cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
-		if cfgOrigP != nil {
-			cfgOrig = *cfgOrigP
-		}
-		if _, err := reloadAuthConfigData(cfgOrig); err != nil {
-			t.Fatalf("cannot load the original config: %s", err)
-		}
-	}()
-
-	ac := authConfig.Load()
-	up := ac.UnauthorizedUser.URLPrefix
-	g0 := up.bus.Load().groups[0]
-	if cap(g0.concurrencyLimitCh) != 1 {
-		t.Fatalf("unexpected group0 concurrency limit; got %d; want 1", cap(g0.concurrencyLimitCh))
-	}
-	// Saturate group0's concurrency limit, simulating an in-flight request to backend A.
-	g0.concurrencyLimitCh <- struct{}{}
-	defer func() { <-g0.concurrencyLimitCh }()
-
-	r, err := http.NewRequest(http.MethodGet, "http://some-host.com/abc", nil)
-	if err != nil {
-		t.Fatalf("cannot initialize http request: %s", err)
-	}
-	r.RequestURI = r.URL.RequestURI()
-	w := &fakeResponseWriter{}
-	if !requestHandlerWithInternalRoutes(w, r) {
-		t.Fatalf("unexpected false is returned from requestHandler")
-	}
-
-	resp := w.getResponse()
-	if !strings.Contains(resp, "backend=B") {
-		t.Fatalf("expecting the request to be routed to backend B, since backend A's group is at its concurrency limit; got response:\n%s", resp)
-	}
-}
-
 func TestJWTRequestHandler(t *testing.T) {
 	// Generate RSA key pair for testing
 	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)

app/vmauth/target_url_test.go

@@ -87,7 +87,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		expectedRetryStatusCodes []int, expectedLoadBalancingPolicy string, expectedDropSrcPathPrefixParts int) {
 		t.Helper()
 
-		if err := ui.initURLs(nil); err != nil {
+		if err := ui.initURLs(); err != nil {
 			t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
 		}
 		u, err := url.Parse(requestURI)
@@ -172,10 +172,8 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 				mustNewHeader("'x: y'"),
 			},
 		},
-		RetryStatusCodes: []int{503, 501},
-		BackendSettings: BackendSettings{
-			LoadBalancingPolicy: "first_available",
-		},
+		RetryStatusCodes:       []int{503, 501},
+		LoadBalancingPolicy:    "first_available",
 		DropSrcPathPrefixParts: new(2),
 	}, "/a/b/c", "http://foo.bar/c", `bb: aaa`, `x: y`, []int{503, 501}, "first_available", 2)
 	f(&UserInfo{
@@ -404,12 +402,10 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
 				URLPrefix: mustParseURL("http://vminsert:8480"),
 			},
 		},
-		BackendSettings: BackendSettings{
-			DiscoverBackendIPs: &allowed,
-		},
-		URLPrefix: mustParseURL("http://non-exist-dns-addr"),
+		DiscoverBackendIPs: &allowed,
+		URLPrefix:          mustParseURL("http://non-exist-dns-addr"),
 	}
-	if err := ui.initURLs(nil); err != nil {
+	if err := ui.initURLs(); err != nil {
 		t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
 	}
 
@@ -459,12 +455,10 @@ func TestUserInfoGetBackendURL_SRVZeroBackends(t *testing.T) {
 				URLPrefix: mustParseURL("http://srv+vmselect"),
 			},
 		},
-		BackendSettings: BackendSettings{
-			DiscoverBackendIPs: &allowed,
-		},
-		URLPrefix: mustParseURL("http://non-exist-dns-addr"),
+		DiscoverBackendIPs: &allowed,
+		URLPrefix:          mustParseURL("http://non-exist-dns-addr"),
 	}
-	if err := ui.initURLs(nil); err != nil {
+	if err := ui.initURLs(); err != nil {
 		t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
 	}
 

app/vmselect/main.go

@@ -369,6 +369,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -388,6 +392,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}

app/vmselect/promql/binary_op.go

@@ -164,11 +164,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		left := bfa.left
 		right := bfa.right
 		op := bfa.be.Op
-		switch true {
-		case metricsql.IsBinaryOpCmp(op):
-			// Do not remove empty series for comparison operations,
-			// since this may lead to missing result.
-		default:
+		isCmpOp := metricsql.IsBinaryOpCmp(op)
+		if !isCmpOp {
+			// Do not remove empty series for comparison operations, since NaN can be an
+			// explicitly present sample value. In particular, `value != NaN` is true.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
@@ -191,6 +191,14 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		isBool := bfa.be.Bool
 		fillLeft := bfa.be.FillLeft
 		fillRight := bfa.be.FillRight
+		// A NaN produced by a vector comparison denotes a filtered-out sample rather
+		// than an explicitly present NaN value. Drop it when this result is used as
+		// the right-hand side of another comparison.
+		// Note that filtered-out samples surviving as NaN inside other wrappers such as
+		// transform functions or arithmetic operations aren't detected, since such NaN
+		// is indistinguishable from an explicitly present NaN value there.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
+		dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -203,11 +211,16 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 				b := rightValues[j]
 				leftIsNaN := math.IsNaN(a)
 				rightIsNaN := math.IsNaN(b)
-				// apply the fill value when either the left or right side is NaN, but not both.
+				// Both sides are NaN.
 				if leftIsNaN && rightIsNaN {
 					dstValues[j] = bf(a, b, isBool)
 					continue
 				}
+				if dropNaNRight && rightIsNaN && fillRight == nil {
+					dstValues[j] = nan
+					continue
+				}
+				// Apply the fill value when either the left or right side is NaN, but not both.
 				if leftIsNaN && fillLeft != nil {
 					a = fillLeft.N
 				}
@@ -223,6 +236,38 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
+// isVectorComparisonExpr returns whether e is a comparison operation
+// with at least one non-scalar operand.
+func isVectorComparisonExpr(e metricsql.Expr) bool {
+	if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
+		// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
+		// the shape of the result. Subqueries are left as is, since rollup
+		// functions skip NaN values on their own.
+		e = re.Expr
+	}
+	be, ok := e.(*metricsql.BinaryOpExpr)
+	if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
+		return false
+	}
+	return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
+}
+
+// isScalarLikeExpr returns whether e always evaluates to a scalar value.
+func isScalarLikeExpr(e metricsql.Expr) bool {
+	switch e := e.(type) {
+	case *metricsql.NumberExpr, *metricsql.DurationExpr:
+		return true
+	case *metricsql.BinaryOpExpr:
+		return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
+	case *metricsql.FuncExpr:
+		switch strings.ToLower(e.Name) {
+		case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
+			return true
+		}
+	}
+	return false
+}
+
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {

app/vmselect/promql/exec_test.go

@@ -2994,6 +2994,136 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != NaN`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != 1200`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (1 > 2)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
+		// A missing sample on the left side results in NaN for every comparison
+		// operation, so no special handling is needed there.
+		t.Parallel()
+		q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
+		// Prometheus drops the timestamps filtered out by the comparison on the right.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`-1 < 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `-1 < 2`

app/vmstorage/main.go

@@ -35,7 +35,10 @@ var (
 		"By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. "+
 		"This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
-	vmselectAddr                  = flag.String("vmselectAddr", "", "TCP address to accept connections from vmselect services")
+	vmselectAddr = flag.String("vmselectAddr", "", "TCP address to listen for incoming connections from vmselect. "+
+		"When set, the node will be able to accept cluster-native vmselect RPC requests as if it were vmstorage. "+
+		"The tenant ID assigned to this node's data is controlled by -accountID and -projectID flags. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy")
 	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
 		"This reduces CPU usage at the cost of higher network bandwidth usage")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")

app/vmui/packages/vmui/package.json

@@ -9,7 +9,7 @@
     "start": "vite",
     "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
+    "lint": "eslint --format stylish 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
     "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls.tsx

@@ -1,4 +1,4 @@
-import { FC, useEffect, useRef, useState } from "preact/compat";
+import { FC, useEffect, useRef } from "preact/compat";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { getAppModeEnable } from "../../../../utils/app-mode";
 import Button from "../../../Main/Button/Button";
@@ -9,27 +9,38 @@ import classNames from "classnames";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import useBoolean from "../../../../hooks/useBoolean";
+import { getMillisecondsFromDuration } from "../../../../utils/time";
+import { useSearchParams } from "react-router";
 
-interface AutoRefreshOption {
-  seconds: number
-  title: string
-}
-
-const delayOptions: AutoRefreshOption[] = [
-  { seconds: 0, title: "Off" },
-  { seconds: 1, title: "1s" },
-  { seconds: 2, title: "2s" },
-  { seconds: 5, title: "5s" },
-  { seconds: 10, title: "10s" },
-  { seconds: 30, title: "30s" },
-  { seconds: 60, title: "1m" },
-  { seconds: 300, title: "5m" },
-  { seconds: 900, title: "15m" },
-  { seconds: 1800, title: "30m" },
-  { seconds: 3600, title: "1h" },
-  { seconds: 7200, title: "2h" }
+const delayOptions = [
+  "Off",
+  "1s",
+  "2s",
+  "5s",
+  "10s",
+  "30s",
+  "1m",
+  "5m",
+  "15m",
+  "30m",
+  "1h",
+  "2h"
 ];
 
+const DEFAULT_OPTION = delayOptions[0];
+
+const MIN_REFRESH_MS = 1000;
+const MAX_REFRESH_MS = getMillisecondsFromDuration(delayOptions[delayOptions.length - 1]);
+const REFRESH_URL_PARAM = "refresh";
+
+const durationToMs = (dur: string | null) => {
+  return dur ? getMillisecondsFromDuration(dur) : 0;
+};
+
+const isValidDelay = (ms: number) => {
+  return ms >= MIN_REFRESH_MS && ms <= MAX_REFRESH_MS;
+};
+
 interface ExecutionControlsProps {
   tooltip: string;
   useAutorefresh?: boolean;
@@ -38,12 +49,14 @@ interface ExecutionControlsProps {
 
 export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAutorefresh, closeModal }) => {
   const { isMobile } = useDeviceDetect();
+  const [searchParams, setSearchParams] = useSearchParams();
 
   const dispatch = useTimeDispatch();
   const appModeEnable = getAppModeEnable();
-  const [autoRefresh, setAutoRefresh] = useState(false);
 
-  const [selectedDelay, setSelectedDelay] = useState<AutoRefreshOption>(delayOptions[0]);
+  const rawDelay = searchParams.get(REFRESH_URL_PARAM);
+  const msDelay = durationToMs(rawDelay);
+  const selectedDelay = isValidDelay(msDelay) ? rawDelay : DEFAULT_OPTION;
 
   const {
     value: openOptions,
@@ -52,11 +65,20 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   } = useBoolean(false);
   const optionsButtonRef = useRef<HTMLDivElement>(null);
 
-  const handleChange = (d: AutoRefreshOption) => {
-    if ((autoRefresh && !d.seconds) || (!autoRefresh && d.seconds)) {
-      setAutoRefresh(prev => !prev);
-    }
-    setSelectedDelay(d);
+  const handleChange = (dur: string) => () => {
+    setSearchParams(prev => {
+      const nextParams = new URLSearchParams(prev);
+      const ms = durationToMs(dur);
+
+      if (ms) {
+        nextParams.set(REFRESH_URL_PARAM, `${dur}`);
+      } else {
+        nextParams.delete(REFRESH_URL_PARAM);
+      }
+
+      return nextParams;
+    });
+
     handleCloseOptions();
   };
 
@@ -68,23 +90,19 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
   };
 
   useEffect(() => {
-    const delay = selectedDelay.seconds;
+    const ms = durationToMs(selectedDelay);
     let timer: number;
-    if (autoRefresh) {
+
+    if (useAutorefresh && isValidDelay(ms)) {
       timer = setInterval(() => {
         dispatch({ type: "RUN_QUERY" });
-      }, delay * 1000) as unknown as number;
-    } else {
-      setSelectedDelay(delayOptions[0]);
+      }, ms) as unknown as number;
     }
-    return () => {
-      timer && clearInterval(timer);
-    };
-  }, [selectedDelay, autoRefresh]);
 
-  const createHandlerChange = (d: AutoRefreshOption) => () => {
-    handleChange(d);
-  };
+    return () => {
+      clearInterval(timer);
+    };
+  }, [selectedDelay, useAutorefresh]);
 
   return (
     <>
@@ -106,7 +124,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 <span className="vm-mobile-option__icon"><RestartIcon/></span>
                 <div className="vm-mobile-option-text">
                   <span className="vm-mobile-option-text__label">Auto-refresh</span>
-                  <span className="vm-mobile-option-text__value">{selectedDelay.title}</span>
+                  <span className="vm-mobile-option-text__value">{selectedDelay}</span>
                 </div>
                 <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
               </div>
@@ -139,7 +157,7 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                       )}
                       onClick={toggleOpenOptions}
                     >
-                      {selectedDelay.title}
+                      {selectedDelay}
                     </Button>
                   </div>
                 </Tooltip>
@@ -187,12 +205,12 @@ export const ExecutionControls: FC<ExecutionControlsProps> = ({ tooltip, useAuto
                 className={classNames({
                   "vm-list-item": true,
                   "vm-list-item_mobile": isMobile,
-                  "vm-list-item_active": d.seconds === selectedDelay.seconds
+                  "vm-list-item_active": d === selectedDelay
                 })}
-                key={d.seconds}
-                onClick={createHandlerChange(d)}
+                key={d}
+                onClick={handleChange(d)}
               >
-                {d.title}
+                {d}
               </div>
             ))}
           </div>

app/vmui/packages/vmui/src/pages/Relabel/index.tsx

@@ -72,7 +72,7 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header-configs">
           <TextField
             type="textarea"
-            label="Relabel configs"
+            label="Config"
             value={config}
             autofocus
             onChange={handleChangeConfig}
@@ -82,8 +82,9 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header__labels">
           <TextField
             type="textarea"
-            label="Labels"
+            label="A time series"
             value={labels}
+            placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}"
             onChange={handleChangeLabels}
             onEnter={handleRunQuery}
           />
@@ -97,22 +98,22 @@ const Relabel: FC = () => {
             <Button
               variant="text"
               color="gray"
-              startIcon={<InfoIcon/>}
+              startIcon={<WikiIcon/>}
             >
               Relabeling cookbook
             </Button>
           </a>
           <a
             target="_blank"
-            href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
+            href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages"
             rel="help noreferrer"
           >
             <Button
               variant="text"
               color="gray"
-              startIcon={<WikiIcon/>}
+              startIcon={<InfoIcon/>}
             >
-              Documentation
+              Relabeling Stages
             </Button>
           </a>
           <Button

app/vmui/packages/vmui/src/utils/time.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect } from "vitest";
 import dayjs from "dayjs";
-import { getNanoTimestamp, parseSupportedDuration } from "./time";
+import { getMillisecondsFromDuration, getNanoTimestamp, parseSupportedDuration } from "./time";
 
 describe("Time utils", () => {
   describe("getNanoTimestamp", () => {
@@ -82,4 +82,19 @@ describe("Time utils", () => {
       expect(parseSupportedDuration("   ")).toBeUndefined();
     });
   });
+
+  describe("getMillisecondsFromDuration", () => {
+    it("should convert valid durations to milliseconds", () => {
+      expect(getMillisecondsFromDuration("1s")).toBe(1000);
+      expect(getMillisecondsFromDuration("1.5s")).toBe(1500);
+      expect(getMillisecondsFromDuration("1m30s")).toBe(90000);
+      expect(getMillisecondsFromDuration("1h 30m")).toBe(5400000);
+      expect(getMillisecondsFromDuration("500ms")).toBe(500);
+    });
+
+    it("should return zero when duration cannot be parsed", () => {
+      expect(getMillisecondsFromDuration("garbage")).toBe(0);
+      expect(getMillisecondsFromDuration("")).toBe(0);
+    });
+  });
 });

app/vmui/packages/vmui/src/utils/time.ts

@@ -100,6 +100,10 @@ export const getSecondsFromDuration = (dur: string) => {
   return dayjs.duration(durObject).asSeconds();
 };
 
+export const getMillisecondsFromDuration = (dur: string): number => {
+  return getSecondsFromDuration(dur) * 1000;
+};
+
 const instantQueryViews = [DisplayType.table, DisplayType.code];
 export const getStepFromDuration = (dur: number, histogram?: boolean, displayType?: DisplayType): string => {
   if (displayType && instantQueryViews.includes(displayType)) return roundStep(dur);
@@ -276,4 +280,3 @@ export const getNanoTimestamp = (dateStr: string): bigint => {
   // Return the full timestamp in nanoseconds as a BigInt
   return BigInt(baseMs) * 1000000n + BigInt(extraNano);
 };
-

dashboards/victoriametrics-cluster.json

@@ -5136,6 +5136,110 @@
           ],
           "title": "Major page faults rate  ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 305
+          },
+          "id": 230,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

dashboards/victoriametrics.json

@@ -5181,6 +5181,110 @@
           ],
           "title": "Major page faults rate",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 81
+          },
+          "id": 159,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
@@ -8742,4 +8846,4 @@
   "uid": "wNf0q_kZk",
   "version": 1,
   "weekStart": ""
-}
+}

dashboards/vm/victoriametrics-cluster.json

@@ -5137,6 +5137,110 @@
           ],
           "title": "Major page faults rate  ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 305
+          },
+          "id": 230,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

dashboards/vm/victoriametrics.json

@@ -5182,6 +5182,110 @@
           ],
           "title": "Major page faults rate",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 81
+          },
+          "id": 159,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",
@@ -8743,4 +8847,4 @@
   "uid": "wNf0q_kZk_vm",
   "version": 1,
   "weekStart": ""
-}
+}

dashboards/vm/vmagent.json

@@ -4581,6 +4581,110 @@
           ],
           "title": "Rows ignored for last 1h ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 189
+          },
+          "id": 171,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

dashboards/vmagent.json

@@ -4580,6 +4580,110 @@
           ],
           "title": "Rows ignored for last 1h ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Average duration of fsync system calls. Spikes or high latency indicates that storage I/O cannot keep up with the write rate, which may slow down data ingestion.\n\nIf this value is elevated:\n- Check disk-related metrics.\n- If using cloud storage, check IOPS and throughput limits.\n- If using NFS, check network performance, server resource usage, and any errors on the NFS server side.\n- Check for known bugs in the filesystem or kernel version being used.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 189
+          },
+          "id": 171,
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": true,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "rate(vm_filestream_fsync_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / rate(vm_filestream_fsync_calls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+              "format": "time_series",
+              "instant": false,
+              "legendFormat": "{{instance}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Fsync avg duration ($instance)",
+          "type": "timeseries"
         }
       ],
       "title": "Troubleshooting",

deployment/docker/README.md

@@ -163,6 +163,8 @@ The list of alerting rules is the following:
   alerting rules related to [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) component;
 * [alerts-vmanomaly.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmanomaly.yml):
   alerting rules related to [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/);
+* [alerts-vmbackupmanager.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml):
+  alerting rules related to [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) component;
 
 Please, also see [how to monitor VictoriaMetrics installations](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring).
 ## Troubleshooting

deployment/docker/compose-vm-cluster.yml

@@ -129,6 +129,7 @@ services:
       - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
       - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
       - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
+      - ./rules/alerts-vmbackupmanager.yml:/etc/alerts/alerts-vmbackupmanager.yml
     command:
       - "--datasource.url=http://vmauth:8427/select/0/prometheus"
       - "--datasource.basicAuth.username=foo"

deployment/docker/compose-vm-single.yml

@@ -70,6 +70,7 @@ services:
       - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
       - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
       - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
+      - ./rules/alerts-vmbackupmanager.yml:/etc/alerts/alerts-vmbackupmanager.yml
     command:
       - "--datasource.url=http://victoriametrics:8428/"
       - "--remoteRead.url=http://victoriametrics:8428/"

deployment/docker/rules/alerts-vmagent.yml

@@ -10,7 +10,7 @@ groups:
     concurrency: 2
     rules:
       - alert: PersistentQueueIsDroppingData
-        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path) > 0
+        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path,name) > 0
         for: 10m
         labels:
           severity: critical
@@ -201,4 +201,4 @@ groups:
         annotations:
           summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours."
           description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
-            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
+            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."

deployment/docker/rules/alerts-vmanomaly.yml

@@ -30,6 +30,35 @@ groups:
           summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
           description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5m"
 
+      - alert: SchedulerWorkerDown
+        expr: >
+          (vmanomaly_scheduler_alive{job=~".*vmanomaly.*"} == 0)
+          and on (job, instance, scheduler_alias, preset)
+          (sum(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"})
+          by (job, instance, scheduler_alias, preset) > 0)
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Scheduler {{ $labels.scheduler_alias }} is down in {{ $labels.job }} ({{ $labels.instance }})"
+          description: |
+            Periodic scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} has remained down for more than 2 minutes
+            after entering automatic recovery. The restart policy may be exhausted.
+            Check vmanomaly logs and the vmanomaly_scheduler_restarts_total metric for restart failures.
+
+      - alert: FrequentSchedulerRestarts
+        expr: >
+          sum(increase(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"}[15m]))
+          by (job, instance, scheduler_alias, preset) > 2
+        labels:
+          severity: warning
+        annotations:
+          summary: "Scheduler {{ $labels.scheduler_alias }} is restarting frequently in {{ $labels.job }} ({{ $labels.instance }})"
+          description: |
+            Scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} had more than two restart attempts
+            during the last 15 minutes. The service may be self-healing, but repeated restarts indicate an unstable
+            scheduler, configuration, datasource, or runtime dependency. Check vmanomaly logs for the original failure.
+
       # default value of 900 Should be changed to the scrape_interval for pull metrics. For push metrics this should be the lowest fit_every or infer_every in your vmanomaly config.
       - alert: NoSelfMonitoringMetrics
         expr: >

deployment/docker/rules/alerts-vmbackupmanager.yml

@@ -0,0 +1,110 @@
+# File contains default list of alerts for vmbackupmanager.
+# The alerts below are just recommendations and may require some updates
+# and threshold calibration according to every specific setup.
+groups:
+  - name: vmbackupmanager
+    rules:
+      - alert: NoLatestBackupWithinLastDay
+        expr: |
+          (
+            vm_backup_last_success_at{type="latest"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="latest"} > 0
+            and time() - vm_backup_last_success_at{type="latest"} > 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "latest backup hasn't completed successfully for more than 1 day on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed the latest backup for more than 1 day.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoHourlyBackupWithinLastDay
+        expr: |
+          (
+            vm_backup_last_success_at{type="hourly"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="hourly"} > 0
+            and time() - vm_backup_last_success_at{type="hourly"} > 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "hourly backup hasn't completed successfully for more than 1 day on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed an hourly backup for more than 1 day.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoDailyBackupWithinLast3Days
+        expr: |
+          (
+            vm_backup_last_success_at{type="daily"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 3 * 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="daily"} > 0
+            and time() - vm_backup_last_success_at{type="daily"} > 3 * 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "daily backup hasn't completed successfully for more than 3 days on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed a daily backup for more than 3 days.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoWeeklyBackupWithinLast14Days
+        expr: |
+          (
+            vm_backup_last_success_at{type="weekly"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 14 * 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="weekly"} > 0
+            and time() - vm_backup_last_success_at{type="weekly"} > 14 * 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "weekly backup hasn't completed successfully for more than 14 days on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed a weekly backup for more than 14 days.
+            Check error logs on vmbackupmanager for root cause.
+
+      - alert: NoMonthlyBackupWithinLast62Days
+        expr: |
+          (
+            vm_backup_last_success_at{type="monthly"} == 0
+            and on(job, instance)
+            vm_app_uptime_seconds > 62 * 24 * 3600
+          )
+          or
+          (
+            vm_backup_last_success_at{type="monthly"} > 0
+            and time() - vm_backup_last_success_at{type="monthly"} > 62 * 24 * 3600
+          )
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "monthly backup hasn't completed successfully for more than 62 days on \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          description: >
+            vmbackupmanager on "{{ $labels.job }}"("{{ $labels.instance }}") hasn't completed a monthly backup for more than 62 days.
+            Check error logs on vmbackupmanager for root cause.

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.29.7
+    image: victoriametrics/vmanomaly:v1.30.0
     depends_on:
       - "victoriametrics"
     ports:

deployment/docker/vmanomaly/vmanomaly-integration/vmanomaly_config.yml

@@ -1,16 +1,18 @@
 schedulers:
   periodic:
     infer_every: "1m"
-    fit_every: "1h"
-    fit_window: "2d" # 2d-14d based on the presence of weekly seasonality in your data
+    fit_every: "100w" # the online model keeps learning during inference
+    fit_window: "2w"
 
 models:
-  prophet:
-    class: "prophet"
-    args:
-      interval_width: 0.98
-      weekly_seasonality: False  # comment it if your data has weekly seasonality
-      yearly_seasonality: False
+  temporal_envelope:
+    class: "temporal_envelope"
+    queries: "all"
+    schedulers: "all"
+    seasonalities: ["hod_smooth", "dow_smooth"]
+    alpha: 0.005
+    loss_reactivity: 5
+    iqr_threshold: 2
 
 reader:
   datasource_url: "http://victoriametrics:8428/"
@@ -26,4 +28,4 @@ writer:
 monitoring:
   pull: # Enable /metrics endpoint.
     addr: "0.0.0.0"
-    port: 8490
+    port: 8490

docs/ai-tools/README.md

@@ -69,18 +69,21 @@ See more details at [VictoriaMetrics/mcp-victoriatraces](https://github.com/Vict
 REST API and documentation for AI-assisted anomaly detection, model management, and observability insights.
 
 Capabilities include:
-- Health Monitoring: Check `vmanomaly` server health and build information
-- Model Management: List, validate, and configure anomaly detection models (like `zscore_online`, `prophet`, and more)
-- Configuration Generation: Generate complete `vmanomaly` YAML configurations
-- Alert Rule Generation: Generate [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules) based on [anomaly score metrics](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) to simplify alerting setup
-- Documentation Search: Full-text search across embedded `vmanomaly` documentation with fuzzy matching
+- Check `vmanomaly` health, build information, compatibility, and self-monitoring metrics
+- Inspect model schemas and validate model or complete service configurations
+- Profile sampled query results for trends, seasonalities, changepoints, gaps, and intermittent behavior
+- Run asynchronous autotune tasks and turn their results into data-driven model recommendations
+- Generate complete `vmanomaly` YAML configurations and [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules)
+- Search embedded `vmanomaly` documentation with fuzzy matching
 
 See more details at [VictoriaMetrics/mcp-vmanomaly](https://github.com/VictoriaMetrics/mcp-vmanomaly).
+# vmanomaly UI Copilot
+The vmanomaly UI includes an [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) that can assist users with anomaly detection tasks, model configuration, and troubleshooting, changing the UI state based on user queries and providing actionable suggestions through automated data profiling and validation. The AI Copilot is powered by respective [MCP Server](#vmanomaly-mcp-server) and [Agent Skills](#agent-skills), enabling it to understand the context of the user's actions and provide relevant guidance.
 
 # Agent Skills
 
-[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate, 
-and troubleshoot VictoriaMetrics observability components, including metrics, logs, and traces.
+[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate,
+and troubleshoot VictoriaMetrics observability components, including metrics, logs, traces, and [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/).
 
 These skills provide predefined workflows and capabilities such as:
 * Query metrics, logs, traces and alerts
@@ -89,6 +92,9 @@ These skills provide predefined workflows and capabilities such as:
 * Cardinality optimization 
 * Unused metric detection
 * Stream aggregation configuration
+* Build validated `vmanomaly` configurations from measured time-series characteristics (e.g., seasonality, changepoints, trends)
+* Query and operate the `vmanomaly` API
+* Review existing anomaly detection configurations against real data and identify false-positive or model-data fit issues
 
 To install the available skills for AI agents, run:
 ```sh
@@ -108,4 +114,4 @@ AI code assistants like Claude Code, OpenAI Codex, Gemini CLI, Qwen Code, and Op
 helps to monitor cost usage, analytics, performance, compliance and improves troubleshooting experience. All major
 AI coding tools support OpenTelemetry and can be easily integrated into VictoriaMetrics Observability Stack.
 Please see more details in [Vibe coding tools observability with VictoriaMetrics Stack and OpenTelemetry
-](https://victoriametrics.com/blog/vibe-coding-observability/).
+](https://victoriametrics.com/blog/vibe-coding-observability/).

docs/ai-tools/_index.md

@@ -9,6 +9,7 @@ tags:
   - metrics
   - logs
   - traces
+  - anomaly detection
   - AI
   - AI integration
   - agent

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,31 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+{{% collapse name="2026" open=true %}}
+
+## v1.30.0
+Released: 2026-07-23
+
+- FEATURE: Added univariate and multivariate online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) models for complex time-series profiles. They combine robust causal trend, compact calendar and learned holiday behavior, changepoint adaptation, residual prediction intervals, future-horizon forecasts, and optional cross-series dependency detection in bounded state.
+
+- FEATURE: Added asynchronous [`/api/v1/autotune/tasks`](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow) endpoints and improved unsupervised tuning with model-capability-aware objectives, optional causal exact validation for online models, constrained alert-volume selection, fitted-state complexity tie-breaking, and frozen model-specific parameters.
+
+- FEATURE: Added `/api/v1/timeseries/characteristics` for bounded analysis of trend, calendar seasonality, changepoints, gaps, and intermittent or spiky behavior across sampled query results. This improves automated model selection and search-space suggestions in both agentic workflows and [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance)-backed [UI](https://docs.victoriametrics.com/anomaly-detection/ui/) experiments.
+
+- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v172) to [v1.8.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v180). Notable mentions are model and query configuration, query prettification, fullscreen charts, out-of-date results alert and improved [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) suggestion and cancellation behavior.
+
+- FEATURE: Made AI-assisted configuration more reliable with reusable [vmanomaly workflow skills](https://github.com/VictoriaMetrics/skills), data-aware model suggestions, protocol-safe cancellation and synchronized query, model, and anomaly-setting updates in [UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
+
+- IMPROVEMENT: Fitted history can be retained as a stronger prior through `history_strength` argument, requiring a reduced number of observations (say 2 weeks for weekly seasonality) to be effective if the history window is a good indicator of "normal" data patterns. Models supported: `mad_online`, `zscore_online`, `quantile_online`.
+
+- IMPROVEMENT: Improved bounded datasource operation with optional ad-hoc series limits, stale-series lookback caps, and propagation of configured per-query sampling periods to models and autotune trials.
+
+- IMPROVEMENT: Added `reader.fetch_timeout` and `reader.processing_timeout` for [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [`VLogsReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#victorialogs-reader), allowing datasource reads and post-fetch processing to be tuned *independently* while preserving `reader.timeout` as the backward-compatible default for both phases.
+
+- BUGFIX: Fixed [backtesting runs](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) across multiple fit cycles and for auto-tuned online wrappers, preventing duplicate or missing predictions while retaining causal model updates and restoration of compatible legacy auto-tuned state.
+
+- BUGFIX: Prevented service shutdown after a [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) worker dies by adding bounded restart attempts with backoff and exposing restart health through [startup metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics).
+
 ## v1.29.7
 Released: 2026-06-25
 
@@ -124,6 +149,10 @@ Released: 2026-01-12
 
 - BUGFIX: Restored expected behavior when `fit_every` equals `infer_every` in [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) - now full data range `fit_window` is fetched for model trainings instead of a  last point from that interval.
 
+{{% /collapse %}}
+
+{{% collapse name="2025" %}}
+
 ## v1.28.3
 Released: 2025-12-17
 
@@ -395,6 +424,10 @@ Released: 2025-01-20
 - BUGFIX: Now [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) properly handles the cases where the number of queries processed in parallel (up to `reader.queries` cardinality) exceeds the default limit of 10 HTTP(S) connections, preventing potential data loss from discarded queries. The pool limit will automatically adjust to match `reader.queries` cardinality.
 - BUGFIX: Corrected the construction of write endpoints for cluster VictoriaMetrics `url`s (`tenant_id` arg is set) in `monitoring.push` [section configurations](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters).
 
+{{% /collapse %}}
+
+{{% collapse name="2024" %}}
+
 ## v1.18.8
 Released: 2024-12-03
 
@@ -709,6 +742,10 @@ Released: 2024-01-15
 - IMPROVEMENT: Don't check /health endpoint, check the real /query_range or /import endpoints directly. Users kept getting problems with /health.
 - DEPRECATION: "health_path" param is deprecated and doesn't do anything in config ([reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), [monitoring.push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters)).
 
+{{% /collapse %}}
+
+{{% collapse name="2023" %}}
+
 
 ## v1.7.2
 Released: 2023-12-21
@@ -803,6 +840,12 @@ Released: 2023-01-23
 Released: 2023-01-06
 - BUGFIX: prophet model incorrectly predicted two points in case of only one
 
+{{% /collapse %}}
+
+{{% collapse name="2022" %}}
+
 ## v1.0.0-beta
 Released: 2022-12-08
 - First public release is available
+
+{{% /collapse %}}

docs/anomaly-detection/FAQ.md

@@ -118,34 +118,17 @@ To visualize and interact with both [self-monitoring metrics](https://docs.victo
 - {{% available_from "v1.26.0" anomaly %}} For rapid exploration of how different models, their configurations and included domain knowledge impacts the results of anomaly detection, use the built-in [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 
-## Is vmanomaly stateful?
-By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
-
-Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
-
-## Config hot-reloading
-
-`vmanomaly` supports [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to automatically reload configurations on config files changes. It can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) and allows for configuration updates without explicit service restarts.
-
-## Environment variables
-
-`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
-
-## Deploying vmanomaly
-
-`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
-
-## Migration
-
-For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
-
 ## Choosing the right model for vmanomaly
 
-> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on selecting the best model and its configuration for your use case. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
+Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect:
 
-Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet).
+- Use [Online MAD](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) for simple, mostly stationary data with no-to-slow trend, when robustness to outliers is important.
+- Use [Online Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) for simple, light-tailed data where standard-deviation units are meaningful.
+- Use [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} for complex data with trends, calendar patterns, holidays, or persistent shifts. It is the preferred *online* alternative to Prophet (which will be deprecated in the future releases).
+- Use multivariate Temporal Envelope when normal relationships between aligned metrics matter. This should replace [Isolation Forest](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate) used in previous versions of `vmanomaly`, which will be deprecated in future releases.
+- Use [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) when Prophet-specific decomposition outputs, or offline batch behavior are required. Consider using Temporal Envelope instead, as it is more efficient and provides better results in most cases.
 
-Also, there is an option to auto-tune the most important (hyper)parameters of selected model class {{% available_from "v1.12.0" anomaly %}}, find [the details here](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned).
+There is also an option to auto-tune the most important parameters of a selected model class {{% available_from "v1.12.0" anomaly %}}. {{% available_from "v1.30.0" anomaly %}} The asynchronous autotune API can first profile a bounded sample through `/api/v1/timeseries/characteristics`, then tune a shared concrete configuration through `/api/v1/autotune/tasks`. See the [autotune workflow](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow).
 
 Please refer to [respective blogpost on anomaly types and alerting heuristics](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/) for more details.
 
@@ -153,9 +136,6 @@ Still not 100% sure what to use? We are [here to help](https://docs.victoriametr
 
 ## Incorporating domain knowledge
 
-> [!TIP]
-> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on incorporating domain knowledge into your anomaly detection models. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
-
 Anomaly detection models can significantly improve when incorporating business-specific assumptions about the data and what constitutes an anomaly. `vmanomaly` supports various [business-side configuration parameters](https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args) across all built-in models to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** and **align model behavior with business needs**, for example:
 
 - **Setting `detection_direction`** - use [`detection_direction`](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) to specify whether anomalies occur **above or below expectations**:
@@ -211,39 +191,15 @@ models:
     provide_series: ['anomaly_score', 'y', 'yhat', 'yhat_lower', 'yhat_upper']
 ```
 
-## Alert generation in vmanomaly
-While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
+## Can AI help configure vmanomaly?
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
+Yes. The available tools serve different workflows:
 
-Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
+- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
+- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
+- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
 
-```yaml
-groups:
-- name: VMAnomalyAlerts
-  interval: 60s
-  rules:
-  - alert: HighAnomalyScore
-    expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
-    for: 5m  # adjust to your needs based on data frequency and alerting policies
-    labels:
-      severity: warning
-      query_alias: explore
-      model_alias: default
-      scheduler_alias: periodic
-      preset: ui
-    annotations:
-      summary: High anomaly score detected.
-      description: Anomaly score exceeded threshold ({{ $value }}) for more than
-        {{ $for }} for query {{ $labels.for }}.
-```
-
-> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
-
-> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
-
-## Preventing alert fatigue
-Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
+Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
 
 ## How to backtest particular configuration on historical data?
 
@@ -298,7 +254,7 @@ Configuration above will produce N intervals of full length (`fit_window`=14d +
 
 ## Forecasting
 
-`vmanomaly` can generate future forecasts (e.g. using [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}), which is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
+`vmanomaly` can generate future forecasts using [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} or [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}. This is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
 
 > However, please note that this mode should be used with care, as the model will produce `yhat_{h}` (and probably `yhat_lower_{h}`, and `yhat_upper_{h}`) time series **for each timeseries returned by input queries and for each forecasting horizon specified in `forecast_at` argument, which can lead to a significant increase in the number of active timeseries in VictoriaMetrics TSDB**.
 
@@ -309,12 +265,12 @@ Here's an example of how to produce forecasts using `vmanomaly` and combine it w
 schedulers:
   periodic_5m:  # this scheduler will be used to produce anomaly scores each 5 minutes using "regular" simple model
     class: 'periodic'
-    fit_every: '30d'
+    fit_every: '100w'
     fit_window: '3d'
     infer_every: '5m'
   periodic_forecast:  # this scheduler will be used to produce forecasts each 24h using "daily" model
     class: 'periodic'
-    fit_every: '7d'
+    fit_every: '1000w'
     fit_window: '730d' # to fit the model on 2 years of data to account for seasonality and holidays
     infer_every: '24h'
 # https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader
@@ -356,20 +312,18 @@ models:
     quantiles: [0.25, 0.5, 0.75]  # to produce median and upper quartiles
     iqr_threshold: 2.0
 
-  prophet_1d:
-    class: 'prophet'
+  envelope_1d:
+    class: 'temporal_envelope'
     queries: ['disk_usage_perc_1d']
     schedulers: ['periodic_forecast']
     clip_predictions: True
     detection_direction: 'above_expected'  # as we are interested in spikes in capacity planning
     forecast_at: ['3d', '7d']  # this will produce forecasts for 3 and 7 days ahead
     provide_series: ['yhat', 'yhat_upper']  # to write forecasts back to VictoriaMetrics, omitting `yhat_lower` as it is not needed in this example
-    # other model params, yearly_seasonality may stay
-
-    # https://facebook.github.io/prophet/docs/quick_start#python-api
-    args:
-      interval_width: 0.98  # see https://facebook.github.io/prophet/docs/uncertainty_intervals
-      country_holidays: 'US'
+    seasonalities: [dow_smooth]
+    holidays:
+      countries: [US]
+      group: true
 # https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer
 writer:
   class: 'vm'
@@ -408,6 +362,61 @@ groups:
       description: "Disk usage is forecasted to exceed 95% in the next 3 days for instance {{ $labels.instance }}. Forecasted value: {{ $value }}."
 ```
 
+## Alert generation in vmanomaly
+While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
+
+<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
+
+Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
+
+```yaml
+groups:
+- name: VMAnomalyAlerts
+  interval: 60s
+  rules:
+  - alert: HighAnomalyScore
+    expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
+    for: 5m  # adjust to your needs based on data frequency and alerting policies
+    labels:
+      severity: warning
+      query_alias: explore
+      model_alias: default
+      scheduler_alias: periodic
+      preset: ui
+    annotations:
+      summary: High anomaly score detected.
+      description: Anomaly score exceeded threshold ({{ $value }}) for more than
+        {{ $for }} for query {{ $labels.for }}.
+```
+
+> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
+
+> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
+
+## Preventing alert fatigue
+Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
+
+## Deploying vmanomaly
+
+`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
+
+## Environment variables
+
+`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
+
+## Is vmanomaly stateful?
+By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
+
+Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
+
+## Config hot-reloading
+
+`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
+
+## Migration
+
+For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
+
 ## Resource consumption of vmanomaly
 `vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), and the complexity of the employed [models](https://docs.victoriametrics.com/anomaly-detection/components/models/). Its resource usage is directly related to these factors, making it adaptable to various operational scales. Various optimizations are available to balance between RAM usage, processing speed, and model capacity. These options are described in the sections below.
 
@@ -423,7 +432,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.7
+    image: victoriametrics/vmanomaly:v1.30.0
     # ...
     restart: always
     volumes:
@@ -460,6 +469,8 @@ With the introduction of [online models](https://docs.victoriametrics.com/anomal
 - **Optimized resource utilization**: By spreading the computational load over time and reducing peak demands, online models make more efficient use of resources and inducing less data transfer from VictoriaMetrics TSDB, improving overall system performance.
 - **Faster convergence**: Online models can adapt {{% available_from "v1.23.0" anomaly %}} to changes in data patterns more quickly, which is particularly beneficial in dynamic environments where data characteristics may shift frequently. See `decay` argument description [here](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay).
 
+{{% available_from "v1.30.0" anomaly %}} Online models expose a **common** `min_n_samples_seen` warmup: they continue learning, but emit `anomaly_score: 0` until enough observations have been seen. Online MAD, Z-score, and Seasonal Quantile also support `history_strength`, which retains fitted history as a stronger prior while inference continues to adapt the state.
+
 > {{% available_from "v1.24.0" anomaly %}} Online models are best used in conjunction with [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the model state across service restarts. This allows the model to continue adapting to new data without losing previously learned patterns, thus avoiding the need for a full `fit` stage to start working again. {{% available_from "v1.28.1" anomaly %}} Additionally, setting [retention policies](https://docs.victoriametrics.com/anomaly-detection/components/settings/#retention) helps manage disk space or RAM used by periodical cleanup of old model instances.
 
 Here's an example of how we can switch from (offline) [Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score) to [Online Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score):
@@ -641,7 +652,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.7 && docker image tag victoriametrics/vmanomaly:v1.29.7 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.30.0 && docker image tag victoriametrics/vmanomaly:v1.30.0 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/Migration.md

@@ -45,7 +45,7 @@ There are 2 types of compatibility to consider when migrating in stateful mode:
 
 | Group start | Group end | Compatibility | Notes |
 |---------|--------- |------------|-------|
-| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1297) | Fully Compatible | - |
+| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300) | Fully Compatible | v1.30.0 adds new [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model state without changing the compatibility of existing model and data artifacts. |
 | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Partially compatible* | Dumped models of class [prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) and [seasonal quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) have problems with loading to [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) due to dropped `pytz` library. **Upgrading directly from v1.28.7 to [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) with a fix is suggested** |
 | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
 | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |

docs/anomaly-detection/QuickStart.md

@@ -132,7 +132,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.7
+docker pull victoriametrics/vmanomaly:v1.30.0
 ```
 
 2. Create the license file with your license key.
@@ -152,7 +152,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.7 \
+    victoriametrics/vmanomaly:v1.30.0 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -169,7 +169,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.7 \
+    victoriametrics/vmanomaly:v1.30.0 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -182,7 +182,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.7
+    image: victoriametrics/vmanomaly:v1.30.0
     # ...
     restart: always
     volumes:
@@ -245,7 +245,7 @@ Before deploying, check the correctness of your configuration validate config fi
 
 ### Example
 
-Here is an example of a config file that will run the [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) model on `vm_cache_entries` metric, with periodic scheduler that runs inference every minute and fits the model every day. The model will be trained on the last 2 weeks of data each time it is (re)fitted. The model will produce `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging purposes. The model will be timezone-aware and will use cyclical encoding for the hour of the day and day of the week seasonality.
+Here is an example of a config file that runs the online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model on a CPU metric. The scheduler runs inference every five minutes and performs a full refit only every 100 weeks; between refits the model updates causally from each inference batch. The initial fit uses four weeks of data. The model produces `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging, and its hour-of-day and day-of-week profiles follow the query timezone and daylight-saving-time changes.
 
 ```yaml
 settings:
@@ -260,38 +260,30 @@ settings:
     # scheduler: INFO
     # reader: INFO
     # writer: INFO
-    model.prophet: WARNING
+    model.online.temporal_envelope: WARNING
 
 schedulers:
-  1d_5m:
+  100w_5m:
     # https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler
     class: 'periodic'
     infer_every: '5m'
     scatter_infer_jobs: true
-    fit_every: '1d'
+    # Temporal Envelope learns online between full refits.
+    fit_every: '100w'
     fit_window: '4w'
 
 models:
-  # https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet
-  prophet_model:
-    class: 'prophet'
+  # https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope
+  temporal_envelope_model:
+    class: 'temporal_envelope'
+    queries: ['cpu_user']
+    schedulers: ['100w_5m']
     provide_series: ['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']  # for debugging
-    tz_aware: True  # set to True if your data is timezone-aware, to deal with DST changes correctly
-    tz_use_cyclical_encoding: True
-    tz_seasonalities: # intra-day + intra-week seasonality
-      - name: 'hod'  # intra-day seasonality, hour of the day
-        fourier_order: 4  # keep it 3-8 based on intraday pattern complexity
-        prior_scale: 10
-      - name: 'dow'  # intra-week seasonality, time of the week
-        fourier_order: 2  # keep it 2-4, as dependencies are learned separately for each weekday
-    compression:  # available since v1.28.1
-      window: "30m"               # downsample 5m data into 30m intervals before fitting
-      agg_method: "mean"          # use mean aggregation within each window
-      adjust_boundaries: true     # adjust confidence intervals after downsampling
-    # inner model args (key-value pairs) accepted by
-    # https://facebook.github.io/prophet/docs/quick_start#python-api
-    args:
-      interval_width: 0.98  # see https://facebook.github.io/prophet/docs/uncertainty_intervals
+    seasonalities: ['hod_smooth', 'dow_smooth']
+    alpha: 0.005  # trend reactivity; try 0.0025-0.02
+    loss_reactivity: 5  # try 1-5; lower values reduce the influence of spikes
+    iqr_threshold: 2  # try 1-4 to adjust data-driven interval width
+    min_n_samples_seen: 16
 
 reader:
   class: 'vm'  # use VictoriaMetrics as a data source
@@ -299,6 +291,7 @@ reader:
   datasource_url: "https://play.victoriametrics.com/" # [YOUR_DATASOURCE_URL]
   tenant_id: '0:0'
   sampling_period: "5m"
+  tz: 'UTC'  # set the IANA timezone that defines local calendar patterns, e.g. 'America/New_York'
   series_processing_batch_size: 8  # number of time series to process together while preparing data for fit or infer stages
   queries:
     # define your queries with MetricsQL - https://docs.victoriametrics.com/victoriametrics/metricsql/
@@ -317,7 +310,7 @@ writer:
 
 ### UI
 
-{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI can be used for prototyping and interactive experimenting to produce vmanomaly's and vmalert's configuration files. Please refer to the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for detailed instructions and examples. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the UI to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
+{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI supports prototyping and interactive generation of `vmanomaly` and `vmalert` configuration files. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for instructions and examples. For optional AI-assisted workflows, use the [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance), connect the [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server), or follow the published [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills).
 
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 > [!TIP]
@@ -409,7 +402,7 @@ For optimal service behavior, consider the following tweaks when configuring `vm
 
 - Set up **state restoration** {{% available_from "v1.24.0" anomaly %}} to resume from the last known state for long-term stability. This is controlled by the `settings.restore_state` boolean [arg](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration).
 
-- Set up **config hot-reloading** {{% available_from "v1.25.0" anomaly %}} to automatically reload configurations on config files changes. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) and allows for configuration updates without explicit service restarts.
+- Set up **configuration hot reload** {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
 
 **Schedulers**:
 - Configure the **inference frequency** in the [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) section of the configuration file.

docs/anomaly-detection/README.md

@@ -27,7 +27,7 @@ Key functions:
 
 The diagram below illustrates how `vmanomaly` fits into an observability setup, such as detecting anomalies in metrics collected by `node_exporter`:
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
+<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
 
 ## How does it work?
 
@@ -40,7 +40,7 @@ VictoriaMetrics Anomaly Detection **continuously re-fit and apply machine learni
   - **Confidence intervals** (`[yhat_lower, yhat_upper]`)  
   These outputs integrate seamlessly into downstream applications, making it easier to **visually inspect anomalies**, e.g. in respective [Grafana dashboards](https://docs.victoriametrics.com/anomaly-detection/presets/#grafana-dashboard).
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/components/vmanomaly-components.webp" alt="node_exporter_example_diagram" style="width:80%"/>
+{{% content "components/vmanomaly-components-diagram.md" %}}
 
 ## Key benefits
 
@@ -58,7 +58,7 @@ Get started with VictoriaMetrics Anomaly Detection by following our guides and i
 
 - **Quickstart**: Learn how to quickly set up `vmanomaly` by following the [Quickstart Guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/).
 - **UI**: Explore anomaly detection configurations through the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
-- **MCP**: Allow AI to assist you in generating service and alerting configurations, answering questions, planning migration with the [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly). Find the setup guide how to setup and use it [here](https://github.com/VictoriaMetrics/mcp-vmanomaly?tab=readme-ov-file#installation).
+- **AI-assisted setup (optional)**: Use the [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance), [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server), or [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) to inspect time-series characteristics, select and autotune models, and generate validated configurations.
 - **Integration**: Integrate anomaly detection into your existing observability stack. Find detailed steps [here](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
 - **Anomaly Detection Presets**: Enable anomaly detection on predefined sets of metrics. Learn more [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
 

docs/anomaly-detection/Scaling-vmanomaly.md

@@ -78,9 +78,7 @@ These [sub-configurations](#sub-configuration) can be assigned to a specific sha
 
 Additionally, a replication factor `R ≥ 1` ensures [high availability](#high-availability) by enforcing redundancy across shards.
 
-<p></p>
-
-![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
+{{% content "vmanomaly-sharding-ha-diagram.md" %}}
 
 > Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
 
@@ -130,9 +128,7 @@ Similar to other VictoriaMetrics ecosystem components, like [VMAgent](https://do
 
 When `VMANOMALY_REPLICATION_FACTOR` > 1, each [sub-config](#sub-configuration) `n` from `{0, N-1}` is assigned to exactly `R` nodes. This ensures redundancy, preventing single-node failures from causing data loss.
 
-<p></p>
-
-![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
+{{% content "vmanomaly-sharding-ha-diagram.md" %}}
 
 > Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
 

docs/anomaly-detection/Self-monitoring.md

@@ -135,6 +135,8 @@ These alerting rules complements the [dashboard](#grafana-dashboard) to monitor
 `vmanomaly-health` alerting group:
 - **`TooManyRestarts`**: Triggers if an instance restarts more than twice within 15 minutes, suggesting the process might be crashlooping and needs investigation.
 - **`ServiceDown`**: Alerts if an instance is down for more than 5 minutes, indicating a service outage.
+- **`SchedulerWorkerDown`**: {{% available_from "v1.30.0" anomaly %}} Alerts when an individual periodic scheduler remains down for more than 2 minutes after automatic recovery attempts. This catches partial service failures that a process-level `ServiceDown` alert cannot detect.
+- **`FrequentSchedulerRestarts`**: {{% available_from "v1.30.0" anomaly %}} Warns when an individual scheduler has more than two restart attempts within 15 minutes. A single successful self-heal does not alert, while repeated attempts indicate a flapping scheduler or dependency.
 - **`ProcessNearFDLimits`**: Alerts when the number of available file descriptors falls below 100, which could lead to severe degradation if the limit is exhausted.
 - **`TooHighCPUUsage`**: Alerts when CPU usage exceeds 90% for a continuous 5-minute period, indicating possible resource exhaustion and the need to adjust resource allocation or load.
 - **`TooHighMemoryUsage`**: Alerts when RAM usage exceeds 85% for a continuous 5-minute period and the need to adjust resource allocation or load.

docs/anomaly-detection/UI.md

@@ -20,13 +20,13 @@ aliases:
 
 ## Introduction
 
-{{% available_from "v1.26.0" anomaly %}} `vmanomaly` is shipped with a built-in [vmui-like](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) [UI](https://en.wikipedia.org/wiki/Graphical_user_interface) that provides an intuitive interface for rapid exploration of how anomaly detection models, their configurations and included domain knowledge impacts the results of anomaly detection, before such configurations are deployed in production.
+{{% available_from "v1.26.0" anomaly %}} `vmanomaly` includes a built-in [vmui-like](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) [UI](https://en.wikipedia.org/wiki/Graphical_user_interface) for exploring queries, comparing anomaly detection models, and tuning model or domain settings before production deployment.
 
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 
 ## Accessing the UI
 
-The UI is available at `http://<vmanomaly-host>:8490` by default, however, the port can be changed in `server` [section](https://docs.victoriametrics.com/anomaly-detection/components/server/) of the [configuration file](https://docs.victoriametrics.com/anomaly-detection/components/) using the `port` parameter:
+The UI is available at `http://<vmanomaly-host>:8490` by default. Change the port with `server.port` in the [configuration file](https://docs.victoriametrics.com/anomaly-detection/components/):
 
 ```yaml
 server:
@@ -39,7 +39,7 @@ For impactful parameters please refer to [optimize resource usage](#optimize-res
 
 ## Playgrounds
 
-To start exploring the UI, you can use embedded demo with preconfigured queries and models down below on public playgrounds (VictoriaMetrics, VictoriaLogs and VictoriaTraces):
+Try the UI with preconfigured queries and models in the public VictoriaMetrics, VictoriaLogs, and VictoriaTraces playgrounds:
 
 {{% collapse name="Playground on VictoriaMetrics Datasource" %}}
 
@@ -151,10 +151,11 @@ users:
 Then, on [settings panel](#settings-panel) of the UI, set the URLs accordingly, also check the option to forward auth headers to the datasource:
 
 ![vmanomaly-ui-sections-settings](vmanomaly-ui-sections-settings.webp)
+{class="w-50 mx-auto"}
 
 ### Pre-configured Datasource
 
-{{% available_from "v1.28.2" anomaly %}} It is possible to disable the datasource selectors from UI (e.g. at purpose to serve internal teams) by using pre-configured one with respective environment variables at `vmanomaly` startup:
+{{% available_from "v1.28.2" anomaly %}} For a shared deployment with a fixed datasource, set these environment variables at `vmanomaly` startup. The UI then hides its datasource selectors:
 
 - `VMANOMALY_UI_DATASOURCE_URL` - to set static datasource URL
 - `VMANOMALY_UI_DATASOURCE_TYPE` - to set datasource type, supported options are `vm` for VictoriaMetrics, `vmlogs` for both VictoriaLogs and VictoriaTraces.
@@ -165,10 +166,9 @@ export VMANOMALY_UI_DATASOURCE_URL=https://play.victoriametrics.com/select/0:0/p
 export VMANOMALY_UI_DATASOURCE_TYPE=vm
 ```
 
-After that, start `vmanomaly` instance as usual, and the datasource selectors will be hidden from UI, while the pre-configured datasource will be used for all queries:
-![vmanomaly-ui-preconfigured-datasource-1](vmanomaly-ui-preconfigured-datasource-1.webp)
+Start `vmanomaly` as usual. All UI queries will use the configured datasource:
 
-![vmanomaly-ui-preconfigured-datasource-2](vmanomaly-ui-preconfigured-datasource-2.webp)
+![vmanomaly-ui-preconfigured-datasource-1](vmanomaly-ui-preconfigured-datasource-1.webp)
 
 ## Preset
 
@@ -193,9 +193,10 @@ The best applications of this mode are:
 
 ### What you can do with Copilot
 
-- **Ask questions** about any model (e.g. [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
+- **Ask questions** about any model (e.g. [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope), [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
 - **Improve detection quality** - describe what's wrong ("too many false positives", "missing spikes") and Copilot reads the config, searches the docs, and proposes a validated configuration change to fix the issue.
 - **Get config suggestions inline** - suggestions appear as interactive cards with an explanation and a YAML diff; click **Apply** to write the change directly to your current settings, or **Decline** to keep the conversation going.
+- {{% available_from "v1.30.0" anomaly %}} **Profile and tune the real query** - with [mcp-vmanomaly](#mcp-tools-server) connected, Copilot can inspect bounded time-series characteristics, recommend an online model, start an asynchronous autotune task, and apply its validated query and model suggestions.
 
 ### How it works
 
@@ -212,7 +213,7 @@ AI Assistant is disabled by default; enable it with `VMANOMALY_COPILOT_ENABLED=t
 Supported providers and model formats:
 
 - **Anthropic** - set `ANTHROPIC_API_KEY`; model format: `anthropic:<model>`
-  - Examples: `claude-haiku-4-5`, `claude-sonnet-4-6`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
+  - Examples: `claude-haiku-4-5`, `claude-sonnet-5`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
 - **OpenAI** - set `OPENAI_API_KEY`; model format: `openai:<model>` or `openai-responses:<model>`
   - Examples: `gpt-5-mini`, `gpt-5.2`; see [full list](https://platform.openai.com/docs/models)
   - {{% available_from "v1.29.1" anomaly %}} OpenAI-compatible non-OpenAI providers are supported through `OPENAI_BASE_URL` + `OPENAI_API_KEY`
@@ -315,14 +316,14 @@ docker run -it --rm \
   -e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
   -p 8080:8080 \
   -p 8490:8490 \
-  victoriametrics/vmanomaly:v1.29.7 \
+  victoriametrics/vmanomaly:v1.30.0 \
   vmanomaly_config.yaml
 ```
 
 
 ## UI Navigation
 
-The vmanomaly UI provides a user-friendly interface for exploring and configuring anomaly detection models. The main components of the UI include:
+The UI has four main areas:
 
 - [**Query Explorer**](#query-explorer): A vmui-like interface for typing and executing MetricsQL/LogsQL queries to visualize data.
 - [**Model Panel**](#model-panel): A form for editing anomaly detection model hyperparameters and applying domain knowledge settings.
@@ -331,7 +332,7 @@ The vmanomaly UI provides a user-friendly interface for exploring and configurin
 
 ### Query Explorer
 
-The Query Explorer provides a vmui-like interface for typing and executing MetricsQL/LogsQL queries to visualize data. 
+Use Query Explorer to run MetricsQL or LogsQL queries and visualize input data.
 
 ![vmanomaly-ui-sections-explore](vmanomaly-ui-sections-explore.webp)
 
@@ -345,23 +346,23 @@ Users can:
 
 ### Visualization Panel
 
-The Visualization Panel has 2 modes of displaying data - either raw queried data or data with detected anomalies, depending on the action taken in the Model Panel.
+The Visualization Panel displays either raw query results or model output, depending on the selected action.
 
-**Visualizations of the queried data** ("Execute Query" button)
+After selecting **Execute Query**:
 
 ![vmanomaly-ui-sections-plot-area-query-mode](vmanomaly-ui-sections-plot-area-query-mode.webp)
 
-> All the metrics are shown in a single plot, similar to vmui, with zooming and panning capabilities.
+All returned series appear in one vmui-like plot with zooming and panning.
 
-**Initial data with detected anomalies** ("Detect Anomalies" button)
+After selecting **Detect Anomalies**:
 
 ![vmanomaly-ui-sections-plot-area-detect-mode](vmanomaly-ui-sections-plot-area-detect-mode.webp)
 
-> The plot shows the queried data, **grouped by individual series**, iterated over legend, with the actual values (`y`) compared to the expected values (model predictions, `y_hat`), confidence intervals (`y_hat_lower`, `y_hat_upper`), and detected anomalies. The anomalies are marked with red circles, and hovering over them provides additional information such as the anomaly score and associated labels.
+The plot groups model output by input series and compares actual values (`y`) with predictions (`yhat`), confidence intervals (`yhat_lower`, `yhat_upper`), and detected anomalies. Hover over an anomaly marker to inspect its score and labels.
 
-Also, timeseries (such as `y`, `y_hat`, etc.) can be toggled on/off by clicking on the legend items.
+Toggle individual output series from the legend.
 
-{{% available_from "v1.29.2" anomaly %}} Seeing model [business-boundaries](https://docs.victoriametrics.com/anomaly-detection/faq/#incorporating-domain-knowledge), such as [detection direction](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) and minimal deviation from expected ([absolute](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-deviation-from-expected) and [relative](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-relative-deviation-from-expected) combined) can be turned on with "business boundaries" toggle. Showing/hiding individual bands can be done by clicking on the respective legend items, while showing/hiding all business boundaries at once can be done with "business boundaries" toggle.
+{{% available_from "v1.29.2" anomaly %}} Enable **Business Boundaries** to overlay the configured [detection direction](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) and combined [absolute](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-deviation-from-expected) and [relative](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-relative-deviation-from-expected) minimum-deviation bands. Toggle individual bands from the legend or all bands with the main control.
 
 [Back to UI navigation](#ui-navigation)
 
@@ -371,18 +372,18 @@ Also, timeseries (such as `y`, `y_hat`, etc.) can be toggled on/off by clicking
 
 The Model Panel provides:
 
-Parameters, such as "Fit Every", "Fit Window" and {{% available_from "v1.28.0" anomaly %}} "Infer Every" to imitate [production scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler), as well as overriding default [anomaly detection threshold](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) (1.0).
+- Scheduling controls such as **Fit Every**, **Fit Window**, and {{% available_from "v1.28.0" anomaly %}} **Infer Every** for imitating [production scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler).
+- An override for the default [anomaly detection threshold](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) of `1.0`.
+- Actions for running or canceling detection, downloading results, and exporting model configurations or example alerting rules.
 
-> {{% available_from "v1.28.0" anomaly %}} "Exact" mode checkbox is used in combination with "Infer Every" control for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) such as `mad_online` or `quantile_online`, to provide unbiased estimates of how production scheduler would perform anomaly detection on incoming data streams. In "exact" mode, the model is updated exactly at every "infer every" micro-batch interval, at a cost of increased computation time.
+> {{% available_from "v1.28.0" anomaly %}} For [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models), combine **Exact** mode with **Infer Every** to reproduce causal production micro-batches. This gives a more representative backtest at the cost of additional computation.
 
-Controls for running/canceling anomaly detection on the queried data, downloading the results as CSV/JSON, accessing and downloading the model configuration or example alerting rules in YAML format.
+A form-based menu configures model hyperparameters and domain knowledge:
 
-A form-based menu for finetuning model hyperparameters and applying domain knowledge settings:
-
-- Model type selection (e.g., rolling quantile, Prophet, etc.)
+- Model selection, for example Temporal Envelope or Online MAD.
 ![vmanomaly-ui-model-config-menu](vmanomaly-ui-model-config-menu.webp)
-- Wizard with **model-agnostic parameters** (e.g., detection direction, data range, scale, clipping, minimum deviation from expected, etc.) and **model-specific hyperparameters** for chosen model type (e.g., quantile and window steps for [rolling quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-quantile) model). {{% available_from "v1.27.0" anomaly %}} autocomplete of example parameters by hitting Tab key is supported.
-![vmanomaly-ui-model-config-wizard](vmanomaly-ui-model-config-wizard.webp)
+- A wizard with **model-agnostic settings** such as detection direction, data range, clipping, and minimum deviation, plus the selected model's hyperparameters. {{% available_from "v1.27.0" anomaly %}} Press **Tab** to autocomplete suggested values.
+  <a class="content-image d-flex justify-content-center" data-bs-target="#image-modal" data-bs-toggle="modal" href="/anomaly-detection/vmanomaly-ui-model-config-wizard.webp"><img alt="vmanomaly-ui-model-config-wizard" class="w-75 mx-auto" src="/anomaly-detection/vmanomaly-ui-model-config-wizard.webp" style="min-width: 0;" /></a>
 
 [Back to UI navigation](#ui-navigation)
 
@@ -397,6 +398,7 @@ The vmui-like "Settings" panel allows users to configure global settings and pre
 - {{% available_from "v1.27.0" anomaly %}} Auth Headers forwarding to datasource (VictoriaMetrics, VictoriaLogs). 
 
 ![vmanomaly-ui-sections-settings](vmanomaly-ui-sections-settings.webp)
+{class="w-50 mx-auto"}
 
 [Back to navigation](#ui-navigation)
 
@@ -585,6 +587,7 @@ Set the "Fit Every" and "Fit Window" parameters to control how often and over wh
 Tune the model hyperparameters and apply domain knowledge settings using the form-based menu in the Model Panel. See (i) tooltips for parameter descriptions and [model documentation](https://docs.victoriametrics.com/anomaly-detection/components/models/) link for recommended values and guidelines.
 
 ![vmanomaly-ui-model-config-wizard](vmanomaly-ui-model-config-wizard.webp)
+{class="w-75 mx-auto"}
 
 For example, for a **MAD online** [model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad), that should be run on a query, returning per-mode CPU utilization (as fractions of 1, data range `[0, 1]`), where you are interested in capturing **spikes of at least 6% deviations** from expected behavior:
 
@@ -640,6 +643,31 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
+{{% collapse name="Release history" %}}
+
+### v1.8.0
+Released: 2026-07-23
+
+vmanomaly version: [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300)
+
+- FEATURE: UX Improvements, refreshed interface with clearer model and query forms, inline configuration previews, and fullscreen chart controls.
+
+- FEATURE: Server (production-running) models and queries can be accessed and selected from the UI, with a new "Queries" button, while model selection now includes a drop-down for server-configured scheduled models in model wizard.
+
+- IMPROVEMENT: Boosted [AI Copilot](#ai-assistance) stability and suggestions quality, aligned with MCP/skills toolset, and proper handling of canceled or incomplete tool calls.
+
+- IMPROVEMENT: Added query prettification and stable value formatting, including platform-aware keyboard-shortcut hints.
+
+- IMPROVEMENT: Results now visibly switch to an out-of-date state after the query, time range, or model configuration changes. The action updates to rerun detection and remains visible in both expanded and collapsed model views.
+
+- IMPROVEMENT: Chart range navigation commits one query on interaction completion instead of issuing many intermediate requests.
+
+- IMPROVEMENT: Compact consecutive identical [AI Copilot](#ai-assistance) tool calls, keep query/model/anomaly suggestions synchronized, and recover cleanly from canceled or incomplete tool calls.
+
+- BUGFIX: Fixed exact UI backtesting across multiple fit cycles and for auto-tuned online wrappers, preventing *duplicate or missing predictions* while retaining causal model updates and restoration of compatible legacy auto-tuned state.
+
+- BUGFIX: Kept automatic trailing-slash redirects for configured path prefixes and `/vmui` relative to the public origin, preventing internal backend hostnames from leaking through reverse proxies such as `vmauth`.
+
 ### v1.7.2
 Released: 2026-06-25
 
@@ -809,3 +837,5 @@ Released: 2025-10-02
 vmanomaly version: [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1260)
 
 Initial public release of the vmanomaly UI.
+
+{{% /collapse %}}

docs/anomaly-detection/components/README.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-This chapter describes different components, that correspond to respective sections of a config to launch VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/) service:
+This chapter describes the configuration sections used to run VictoriaMetrics Anomaly Detection, or [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/):
 
 - [Model(s) section](https://docs.victoriametrics.com/anomaly-detection/components/models/) - Required
 - [Reader section](https://docs.victoriametrics.com/anomaly-detection/components/reader/) - Required
@@ -16,9 +16,9 @@ This chapter describes different components, that correspond to respective secti
 - [Settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/) - Optional
 - [Server section](https://docs.victoriametrics.com/anomaly-detection/components/server/) - Optional
 
-> Once the service starts, automated config validation is performed {{% available_from "v1.7.2" anomaly %}}. Please see container logs for errors that need to be fixed to create fully valid config, visiting sections above for examples and documentation.
+> The service validates its configuration at startup{{% available_from "v1.7.2" anomaly %}}. Check the container logs for validation errors and use the sections above for field descriptions and examples.
 
-> Components' class {{% available_from "v1.13.0" anomaly %}} can be referenced by a short alias instead of a full class path - i.e. `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, `scheduler.periodic.PeriodicScheduler` becomes `periodic`, etc. Please see according sections for the details.
+> Component classes{{% available_from "v1.13.0" anomaly %}} can be referenced by short aliases instead of full import paths. For example, `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, and `scheduler.periodic.PeriodicScheduler` becomes `periodic`.
 
 > `preset` modes are available {{% available_from "v1.13.0" anomaly %}} for `vmanomaly`. Please find the guide [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
 
@@ -28,11 +28,11 @@ Below, you will find an example illustrating how the components of `vmanomaly` i
 
 > [Reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [Writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) also support [multitenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy), so you can read/write from/to different locations - see `tenant_id` param description.
 
-![vmanomaly-components](vmanomaly-components.webp)
+{{% content "vmanomaly-components-diagram.md" %}}
 
 ## Example config
 
-Here's a minimalistic full config example, demonstrating many-to-many configuration (actual for [latest version](https://docs.victoriametrics.com/anomaly-detection/changelog/)):
+The following minimal configuration demonstrates current many-to-many model, query, and scheduler mapping:
 
 ```yaml
 settings:
@@ -52,7 +52,7 @@ schedulers:
     scatter_infer_jobs: true  # distribute infer jobs evenly across the infer interval to reduce synchronized bursts
     fit_every: "365d"  # how often to re-fit the models, for online models used effectively once, then they are updated with new data and won't require re-fit
     fit_window: "3d"  # how much historical data to use for fit stage
-    start_from: "00:00"  # start from specified time, i.e. 00:00 given timezone and do daily fits as `fit_every` is 1 day
+    start_from: "00:00"  # align the annual fit schedule to midnight in the configured timezone
     tz: "Europe/Kyiv"  # timezone to use for start_from
   periodic_offline_1w:
     class: 'periodic'
@@ -146,14 +146,14 @@ server:
 
 > This feature is better used in conjunction with [stateful service](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the state of the models and schedulers between restarts and reuse what can be reused, thus avoiding unnecessary re-training of models, re-initialization of schedulers and re-reading of data.
 
-{{% available_from "v1.25.0" anomaly %}} Service supports hot reload of configuration files, which allows for automatic reloading of configurations on config files change without the need of explicit service restart. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). `vmanomaly_config_reload_enabled` flag in [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) will be set to 1 (if enabled) or 0 (if disabled).
+{{% available_from "v1.25.0" anomaly %}} The service supports hot reload of configuration files, applying changes without an explicit restart. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). The `vmanomaly_config_reload_enabled` [self-monitoring metric](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) is `1` when hot reload is enabled and `0` otherwise.
 
 > [!NOTE]
 > {{% deprecated_from "v1.29.5" anomaly %}} File system event-based hot reload has been deprecated in favor of content-based polling with configurable `-configCheckInterval` due to reliability issues with Kubernetes ConfigMap symlink rotations and other filesystems where event delivery can be inconsistent. If you were using file system event-based hot reload, please switch to content-based polling by enabling `--watch` flag and configuring `-configCheckInterval` as needed.
 
 ### How it works
 
-It works by checking watched `.yml|.yaml` file contents in the specified files or directories on the configured interval `-configCheckInterval` (default is `30s`) {{% available_from "v1.29.5" anomaly %}}. When a content change is detected, the service will attempt to reload the configuration files after the existing debounce window, rebuild the [global config](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration) and reinitialize the components. If the reload is successful, the `vmanomaly_config_reloads_total` metric will be incremented for `status="success"` label, otherwise it will be incremented with `status="failure"` label and a respective error message on config validation failure(s) will be logged.
+The service checks watched `.yml` and `.yaml` files at the `-configCheckInterval` interval (default `30s`){{% available_from "v1.29.5" anomaly %}}. When it detects a content change, it waits for the debounce window, rebuilds the [global configuration](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration), and reinitializes the components. The `vmanomaly_config_reloads_total` metric is incremented with `status="success"` or `status="failure"`; validation failures are also logged.
 
 > If the reload fails, the service will log an error message indicating the reason for the failure, and the **previous configuration will remain active until a successful reload occurs** to preserve the service's stability. This means that if there are errors in the new configuration, the service will continue to operate with the last valid configuration until the issues are resolved.
 

docs/anomaly-detection/components/autotune.svg

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 968 420" role="img" aria-labelledby="title description">
+  <title id="title">AutoTunedModel tuning and inference lifecycle</title>
+  <desc id="description">The tuning process tests and scores model candidates across n time-series splits until the trial count or timeout is reached. Each fold fits a candidate on training data and predicts anomalies on its validation segment. The best model is then used on inference data until the next fit call.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M0 0 10 5 0 10Z" fill="#202124"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #202124; font-size: 18px; }
+      .line { fill: none; stroke: #202124; stroke-width: 2.5; }
+      .arrow { fill: none; stroke: #202124; stroke-width: 2.5; marker-end: url(#arrow); }
+      .dash { fill: none; stroke: #202124; stroke-width: 2.5; stroke-dasharray: 8 9; }
+      .box { fill: #fff; stroke: #202124; stroke-width: 1.5; }
+    </style>
+  </defs>
+
+  <rect width="968" height="420" fill="#fff"/>
+  <rect x="112" y="10" width="242" height="83" class="box"/>
+  <text x="233" y="38" text-anchor="middle">
+    <tspan x="233" dy="0">test and score candidates</tspan>
+    <tspan x="233" dy="21">until `n_trials` or `timeout`</tspan>
+    <tspan x="233" dy="21">is reached</tspan>
+  </text>
+  <path d="M354 49 V67" class="arrow"/>
+  <text x="355" y="90" text-anchor="middle">Tuning</text>
+  <text x="355" y="112" text-anchor="middle">process</text>
+
+  <path d="M162 157 V136 H549 V157" class="line"/>
+  <path d="M355 112 V136" class="line"/>
+  <path d="M399 93 H653 V152" class="line"/>
+
+  <path d="M113 161 H75 V350 H113" class="line"/>
+  <path d="M75 236 H39" class="line"/>
+  <text x="28" y="250" transform="rotate(-90 28 250)" text-anchor="middle">n splits</text>
+
+  <text x="102" y="227">Fold 1</text>
+  <text x="102" y="255">Fold 2</text>
+  <text x="102" y="283">Fold 3</text>
+  <text x="120" y="327">...</text>
+
+  <text x="260" y="179" text-anchor="middle">Fit</text>
+  <text x="260" y="202" text-anchor="middle">candidate</text>
+  <text x="413" y="179" text-anchor="middle">Predict</text>
+  <text x="413" y="202" text-anchor="middle">anomalies</text>
+
+  <path d="M173 221 H354 V207" class="line"/>
+  <path d="M232 249 H412 V235" class="line"/>
+  <path d="M293 280 H472 V265" class="line"/>
+  <path d="M354 221 H412" class="dash"/>
+  <path d="M412 249 H474" class="dash"/>
+  <path d="M472 280 H513" class="dash"/>
+
+  <path d="M548 144 V415" class="dash" style="stroke-width:1.5;stroke-dasharray:4 6"/>
+  <rect x="577" y="152" width="136" height="61" rx="12" class="box"/>
+  <text x="645" y="187" text-anchor="middle">best model</text>
+  <path d="M653 213 V348" class="arrow"/>
+  <text x="815" y="168" text-anchor="middle">
+    <tspan x="815" dy="0">used to predict</tspan>
+    <tspan x="815" dy="21">on inference data</tspan>
+    <tspan x="815" dy="21">until the next `fit` call</tspan>
+  </text>
+
+  <path d="M40 350 H895" class="arrow"/>
+  <text x="455" y="402" text-anchor="middle">Training data</text>
+  <text x="623" y="402" text-anchor="middle">Inference data</text>
+  <text x="856" y="402">Time axis, t</text>
+</svg>

docs/anomaly-detection/components/model-lifecycle-multivariate.svg

@@ -0,0 +1,139 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
+  <title id="title">Multivariate models lifecycle</title>
+  <desc id="description">MetricsQL queries retrieve an aligned set of series from VictoriaMetrics. Reader data fits one shared multivariate model. The exact same series set produces one anomaly score series with the intersected label set; inference is skipped when the fit and inference series sets differ. Writer stores produced scores in VictoriaMetrics.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
+      .title { font-size: 50px; font-weight: 400; }
+      .node { font-size: 34px; font-weight: 400; }
+      .label { font-size: 29px; }
+      .blue-label { font-size: 26px; }
+      .small { font-size: 25px; }
+      .box { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .group { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
+      .line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
+      .blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
+      .blue { fill: #1478c9; }
+    </style>
+  </defs>
+
+  <rect width="1920" height="1295" fill="#fff"/>
+  <text x="20" y="70" class="title">Multivariate Models Lifecycle</text>
+
+  <!-- VictoriaMetrics and query configuration -->
+  <rect x="535" y="175" width="420" height="215" class="box"/>
+  <text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
+  <text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
+  <rect x="270" y="270" width="150" height="155" class="box"/>
+  <text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
+  <text x="285" y="360" class="label">MetricsQL</text>
+  <text x="285" y="395" class="label">queries</text>
+  <path d="M420 346 H535" class="line"/>
+
+  <!-- Reader and datasource exchange -->
+  <rect x="580" y="500" width="310" height="100" class="box"/>
+  <text x="735" y="562" class="node" text-anchor="middle">Reader</text>
+  <path d="M580 545 H455 V365 H535" class="line"/>
+  <text x="465" y="478" class="label">1. Request data</text>
+  <path d="M955 270 H1040 V550 H890" class="line"/>
+  <text x="810" y="478" class="label">2. Get metrics</text>
+
+  <!-- Historical fit data returned by the configured queries -->
+  <g aria-label="Historical fit data">
+    <rect x="1090" y="85" width="430" height="505" class="group"/>
+    <rect x="1120" y="135" width="340" height="170" class="group"/>
+    <text x="1140" y="180" class="node">Query 1</text>
+    <rect x="1135" y="195" width="300" height="42" class="box"/>
+    <text x="1150" y="228" class="label">Metric 1.1</text>
+    <text x="1150" y="262" class="label">...</text>
+    <rect x="1135" y="265" width="300" height="42" class="box"/>
+    <text x="1150" y="298" class="label">Metric 1.M₁</text>
+    <text x="1305" y="355" class="node" text-anchor="middle">...</text>
+    <rect x="1120" y="390" width="340" height="170" class="group"/>
+    <text x="1140" y="435" class="node">Query N</text>
+    <rect x="1135" y="450" width="300" height="42" class="box"/>
+    <text x="1150" y="483" class="label">Metric N.1</text>
+    <text x="1150" y="517" class="label">...</text>
+    <rect x="1135" y="520" width="300" height="42" class="box"/>
+    <text x="1150" y="553" class="label">Metric N.Mₙ</text>
+  </g>
+
+  <!-- One shared model is fitted on the complete aligned set -->
+  <rect x="1215" y="795" width="310" height="115" class="box"/>
+  <text x="1370" y="865" class="node" text-anchor="middle">Model</text>
+  <rect x="1365" y="635" width="155" height="130" class="box"/>
+  <text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1382" y="723" class="label">Model</text>
+  <text x="1382" y="757" class="label">config</text>
+  <path d="M1305 590 V795" class="line"/>
+  <text x="1055" y="672" class="label">
+    <tspan x="1055" dy="0">3. Fit one model</tspan>
+    <tspan x="1055" dy="38">on all historical series</tspan>
+  </text>
+  <path d="M1443 765 V795" class="line"/>
+
+  <!-- Inference data must contain the same set of series -->
+  <g aria-label="Inference data">
+    <rect x="25" y="615" width="430" height="500" class="group"/>
+    <rect x="55" y="645" width="340" height="170" class="group"/>
+    <text x="75" y="690" class="node">Query 1</text>
+    <rect x="70" y="705" width="300" height="42" class="box"/>
+    <text x="85" y="738" class="label">Metric 1.1</text>
+    <text x="85" y="772" class="label">...</text>
+    <rect x="70" y="775" width="300" height="42" class="box"/>
+    <text x="85" y="808" class="label">Metric 1.M₁</text>
+    <text x="225" y="870" class="node" text-anchor="middle">...</text>
+    <rect x="55" y="900" width="340" height="175" class="group"/>
+    <text x="75" y="945" class="node">Query N</text>
+    <rect x="70" y="960" width="300" height="42" class="box"/>
+    <text x="85" y="993" class="label">Metric N.1</text>
+    <text x="85" y="1027" class="label">...</text>
+    <rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
+    <text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
+  </g>
+  <path d="M580 575 H500 V650 H455" class="line"/>
+  <text x="465" y="680" class="label">4. Provide inference data</text>
+
+  <!-- Single multivariate model registry -->
+  <g aria-label="Multivariate model registry">
+    <rect x="580" y="750" width="420" height="480" class="group"/>
+    <text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
+    <rect x="610" y="835" width="360" height="220" class="model-group"/>
+    <rect x="640" y="885" width="300" height="115" class="box"/>
+    <text x="790" y="955" class="node" text-anchor="middle">Model (single)</text>
+  </g>
+  <path d="M455 1050 H580" class="line"/>
+  <path d="M1215 852 H1000" class="line"/>
+
+  <!-- Joint output and mismatched-series skip path -->
+  <rect x="1600" y="1015" width="285" height="105" class="box"/>
+  <text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
+  <rect x="1740" y="805" width="145" height="145" class="box"/>
+  <text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1758" y="900" class="label">Writer</text>
+  <text x="1758" y="935" class="label">config</text>
+  <path d="M1813 950 V1015" class="line"/>
+  <path d="M1000 1040 H1600" class="line"/>
+  <text x="1295" y="965" class="label" text-anchor="middle">
+    <tspan x="1295" dy="0">5.a Produce one anomaly-score series</tspan>
+    <tspan x="1295" dy="38">with the intersected label set</tspan>
+  </text>
+  <path d="M1000 1110 H1600" class="blue-line"/>
+  <text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
+    <tspan x="1320" dy="0">5.b Skip inference when the fit and inference</tspan>
+    <tspan x="1320" dy="32">series sets differ;</tspan>
+    <tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
+  </text>
+
+  <!-- Persist the joint anomaly score -->
+  <path d="M1743 1015 V80 H745 V175" class="line"/>
+  <text x="1170" y="55" class="label" text-anchor="middle">6. Write one anomaly-score series (label set = intersection)</text>
+</svg>

docs/anomaly-detection/components/model-lifecycle-univariate.svg

@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
+  <title id="title">Univariate models lifecycle</title>
+  <desc id="description">MetricsQL queries retrieve multiple series from VictoriaMetrics. Reader data fits one model per series in the model registry. Known series produce individual anomaly scores for Writer; an unseen series is skipped until a fitted model exists. Writer stores produced scores in VictoriaMetrics.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
+      .title { font-size: 50px; font-weight: 400; }
+      .node { font-size: 34px; font-weight: 400; }
+      .label { font-size: 29px; }
+      .blue-label { font-size: 26px; }
+      .small { font-size: 25px; }
+      .box { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .group { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
+      .line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
+      .blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
+      .blue { fill: #1478c9; }
+    </style>
+  </defs>
+
+  <rect width="1920" height="1295" fill="#fff"/>
+  <text x="20" y="70" class="title">Univariate Models Lifecycle</text>
+
+  <!-- VictoriaMetrics and query configuration -->
+  <rect x="535" y="175" width="420" height="215" class="box"/>
+  <text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
+  <text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
+  <rect x="270" y="270" width="150" height="155" class="box"/>
+  <text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
+  <text x="285" y="360" class="label">MetricsQL</text>
+  <text x="285" y="395" class="label">queries</text>
+  <path d="M420 346 H535" class="line"/>
+
+  <!-- Reader and datasource exchange -->
+  <rect x="580" y="500" width="310" height="100" class="box"/>
+  <text x="735" y="562" class="node" text-anchor="middle">Reader</text>
+  <path d="M580 545 H455 V365 H535" class="line"/>
+  <text x="465" y="478" class="label">1. Request data</text>
+  <path d="M955 270 H1040 V550 H890" class="line"/>
+  <text x="810" y="478" class="label">2. Get metrics</text>
+
+  <!-- Historical fit data returned by the configured queries -->
+  <g aria-label="Historical fit data">
+    <rect x="1090" y="85" width="430" height="505" class="group"/>
+    <rect x="1120" y="135" width="340" height="170" class="group"/>
+    <text x="1140" y="180" class="node">Query 1</text>
+    <rect x="1135" y="195" width="300" height="42" class="box"/>
+    <text x="1150" y="228" class="label">Metric 1.1</text>
+    <text x="1150" y="262" class="label">...</text>
+    <rect x="1135" y="265" width="300" height="42" class="box"/>
+    <text x="1150" y="298" class="label">Metric 1.M₁</text>
+    <text x="1305" y="355" class="node" text-anchor="middle">...</text>
+    <rect x="1120" y="390" width="340" height="170" class="group"/>
+    <text x="1140" y="435" class="node">Query N</text>
+    <rect x="1135" y="450" width="300" height="42" class="box"/>
+    <text x="1150" y="483" class="label">Metric N.1</text>
+    <text x="1150" y="517" class="label">...</text>
+    <rect x="1135" y="520" width="300" height="42" class="box"/>
+    <text x="1150" y="553" class="label">Metric N.Mₙ</text>
+  </g>
+
+  <!-- Model fitting -->
+  <rect x="1215" y="795" width="310" height="115" class="box"/>
+  <text x="1370" y="865" class="node" text-anchor="middle">Model</text>
+  <rect x="1365" y="635" width="155" height="130" class="box"/>
+  <text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1382" y="723" class="label">Model</text>
+  <text x="1382" y="757" class="label">config</text>
+  <path d="M1305 590 V795" class="line"/>
+  <text x="1055" y="672" class="label">
+    <tspan x="1055" dy="0">3. Fit models</tspan>
+    <tspan x="1055" dy="38">on historical data</tspan>
+  </text>
+  <path d="M1443 765 V795" class="line"/>
+
+  <!-- Inference data, including a new unseen series -->
+  <g aria-label="Inference data">
+    <rect x="25" y="615" width="430" height="500" class="group"/>
+    <rect x="55" y="645" width="340" height="170" class="group"/>
+    <text x="75" y="690" class="node">Query 1 (has fit model)</text>
+    <rect x="70" y="705" width="300" height="42" class="box"/>
+    <text x="85" y="738" class="label">Metric 1.1</text>
+    <text x="85" y="772" class="label">...</text>
+    <rect x="70" y="775" width="300" height="42" class="box"/>
+    <text x="85" y="808" class="label">Metric 1.M₁</text>
+    <text x="225" y="870" class="node" text-anchor="middle">...</text>
+    <rect x="55" y="900" width="340" height="175" class="group"/>
+    <text x="75" y="945" class="node">Query N</text>
+    <rect x="70" y="960" width="300" height="42" class="box"/>
+    <text x="85" y="993" class="label">Metric N.1</text>
+    <text x="85" y="1027" class="label">...</text>
+    <rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
+    <text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
+  </g>
+  <path d="M580 575 H500 V650 H455" class="line"/>
+  <text x="465" y="680" class="label">4. Provide inference data</text>
+
+  <!-- One fitted model per known series -->
+  <g aria-label="Univariate model registry">
+    <rect x="580" y="750" width="420" height="480" class="group"/>
+    <text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
+    <rect x="610" y="830" width="360" height="170" class="model-group"/>
+    <rect x="630" y="865" width="300" height="42" class="box"/>
+    <text x="645" y="898" class="label">Model 1.1</text>
+    <text x="645" y="932" class="label">...</text>
+    <rect x="630" y="935" width="300" height="42" class="box"/>
+    <text x="645" y="968" class="label">Model 1.M₁</text>
+    <text x="790" y="1045" class="node" text-anchor="middle">...</text>
+    <rect x="610" y="1070" width="360" height="135" class="model-group"/>
+    <rect x="630" y="1090" width="300" height="42" class="box"/>
+    <text x="645" y="1123" class="label">Model N.1</text>
+    <rect x="630" y="1145" width="300" height="42" class="box"/>
+    <text x="645" y="1178" class="label">Model N.Mₙ</text>
+  </g>
+  <path d="M455 1050 H580" class="line"/>
+  <path d="M1215 852 H1000" class="line"/>
+
+  <!-- Known-series output and unseen-series skip path -->
+  <rect x="1600" y="1015" width="285" height="105" class="box"/>
+  <text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
+  <rect x="1740" y="805" width="145" height="145" class="box"/>
+  <text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1758" y="900" class="label">Writer</text>
+  <text x="1758" y="935" class="label">config</text>
+  <path d="M1813 950 V1015" class="line"/>
+  <path d="M1000 1040 H1600" class="line"/>
+  <text x="1275" y="975" class="label" text-anchor="middle">
+    <tspan x="1275" dy="0">5.a Produce anomaly scores</tspan>
+    <tspan x="1275" dy="38">for known series</tspan>
+  </text>
+  <path d="M1000 1110 H1600" class="blue-line"/>
+  <text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
+    <tspan x="1320" dy="0">5.b Skip inference until a fitted model exists</tspan>
+    <tspan x="1320" dy="32">for Metric N.Mₖ;</tspan>
+    <tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
+  </text>
+
+  <!-- Persist anomaly scores -->
+  <path d="M1743 1015 V80 H745 V175" class="line"/>
+  <text x="1130" y="55" class="label" text-anchor="middle">6. Write back produced anomaly scores</text>
+</svg>

docs/anomaly-detection/components/monitoring.md

@@ -24,6 +24,10 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
 - Adding `preset` and `scheduler_alias` keys to [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics) and [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics) metrics for consistency in multi-[scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) setups.
 - Renaming [Counters](https://prometheus.io/docs/concepts/metric_types/#counter) `vmanomaly_reader_response_count` to `vmanomaly_reader_responses` and `vmanomaly_writer_response_count` to `vmanomaly_writer_responses`.
 
+<div class="collapse-group">
+
+{{% collapse name="Pull model config parameters" %}}
+
 ## Pull Model Config parameters
 
 <table class="params">
@@ -60,6 +64,10 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
     </tbody>
 </table>
 
+{{% /collapse %}}
+
+{{% collapse name="Push config parameters" %}}
+
 ## Push Config parameters
 
 By default, metrics are pushed only after the completion of specific stages, e.g., `fit`, `infer`, or `fit_infer` (for each [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) if using a multi-scheduler configuration).
@@ -227,6 +235,10 @@ Path to a file with the client certificate key, i.e. `client.key`{{% available_f
     </tbody>
 </table>
 
+{{% /collapse %}}
+
+</div>
+
 ## Monitoring section config example
 
 ``` yaml
@@ -260,6 +272,10 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
 - [Model metrics](#models-behaviour-metrics)
 - [Writer metrics](#writer-behaviour-metrics)
 
+<div class="collapse-group">
+
+{{% collapse name="Startup metrics" %}}
+
 ### Startup metrics
 
 <table class="params">
@@ -304,7 +320,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
 <span style="white-space: nowrap;">`vmanomaly_available_memory_bytes`</span>
             </td>
             <td>Gauge</td>
-            <td>Virtual memory size in bytes, available to the process{{% available_from "v1.18.4" anomaly %}}.</td>
+            <td>Effective memory capacity available to the process in bytes{{% available_from "v1.18.4" anomaly %}}. The value honors cgroup limits when available, then process address-space limits, and otherwise reports host physical memory. It does not represent currently unused memory.</td>
         </tr>
         <tr>
             <td>
@@ -312,7 +328,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
 <span style="white-space: nowrap;">`vmanomaly_cpu_cores_available`</span>
             </td>
             <td>Gauge</td>
-            <td>Number of (logical) CPU cores available to the process{{% available_from "v1.18.4" anomaly %}}.</td>
+            <td>Effective CPU capacity available to the process{{% available_from "v1.18.4" anomaly %}}, constrained by host logical CPUs, process affinity, and cgroup quota. The value can be fractional when a fractional CPU quota is configured.</td>
         </tr>
         <tr>
             <td>
@@ -320,7 +336,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
 <span style="white-space: nowrap;">`vmanomaly_config_entities`</span>
             </td>
             <td>Gauge</td>
-            <td>Number of [sub-configs](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#sub-configuration) **available** (`{scope="total"}`) and **used** for particular [shard](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability) (`{scope="shard"}`) {{% available_from "v1.21.0" anomaly %}}</td>
+            <td>Number of [sub-configs](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#sub-configuration) **available** (`scope="total"`) and **used** by the current [shard](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability) (`scope="shard"`){{% available_from "v1.21.0" anomaly %}}, labeled by `preset` and `scope`.</td>
         </tr>
         <tr>
             <td>
@@ -352,11 +368,43 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
             <td>Gauge</td>
             <td>Timestamp of the last successful config [hot-reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) in seconds since epoch {{% available_from "v1.25.1" anomaly %}}</td>
         </tr>
+        <tr>
+            <td>
+<span style="white-space: nowrap;">`vmanomaly_scheduler_alive`</span>
+            </td>
+            <td>Gauge</td>
+            <td>Whether the scheduler worker thread identified by `scheduler_alias` and `preset` is alive (`1`) or not (`0`) {{% available_from "v1.30.0" anomaly %}}.</td>
+        </tr>
+        <tr>
+            <td>
+<span style="white-space: nowrap;">`vmanomaly_scheduler_restarts_total`</span>
+            </td>
+            <td>Counter</td>
+            <td>Number of bounded scheduler restart attempts {{% available_from "v1.30.0" anomaly %}}, labeled by `scheduler_alias`, `preset`, and `status` (`success` or `failure`).</td>
+        </tr>
+        <tr>
+            <td>
+<span style="white-space: nowrap;">`vm_license_expires_at`</span>
+            </td>
+            <td>Gauge</td>
+            <td>License expiration time as a Unix timestamp in seconds. See the [licensing section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#licensing) for example alerts.</td>
+        </tr>
+        <tr>
+            <td>
+<span style="white-space: nowrap;">`vm_license_expires_in_seconds`</span>
+            </td>
+            <td>Gauge</td>
+            <td>Time remaining until license expiration in seconds. See the [licensing section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#licensing) for warning and critical alert examples.</td>
+        </tr>
     </tbody>
 </table>
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Reader behaviour metrics" %}}
+
 ### Reader behaviour metrics
 Label names [description](#labelnames)
 
@@ -425,7 +473,7 @@ Label names [description](#labelnames)
 
 `Histogram` (was `Summary`{{% deprecated_from "v1.17.0" anomaly %}})
             </td>
-            <td>The total time (in seconds) taken for data parsing at each `step` (json, dataframe) for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
+            <td>The total time (in seconds) taken for data parsing at each `step` (`json` or `df`) for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
 
 `step`, `url`, `query_key`, `scheduler_alias`, `preset`
@@ -481,6 +529,10 @@ Label names [description](#labelnames)
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Models behaviour metrics" %}}
+
 ### Models behaviour metrics
 Label names [description](#labelnames)
 
@@ -521,7 +573,7 @@ Label names [description](#labelnames)
             <td>
 
 <span style="white-space: nowrap;">`Histogram`</span> (was `Summary`{{% deprecated_from "v1.17.0" anomaly %}}) </td>
-            <td>The total time (in seconds) taken by model invocations during the `stage` (`fit`, `infer`, `fit_infer`), based on the results of the `query_key` query, for models of class `model_alias`, within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
+            <td>The model-service stage duration in seconds for `fit`, `infer`, or combined `fit_infer` execution, based on the results of the `query_key` query for `model_alias`. Reader and writer I/O durations are reported by their respective metrics.</td>
             <td>
 
 `stage`, `query_key`, `model_alias`, `scheduler_alias`, `preset`
@@ -536,7 +588,7 @@ Label names [description](#labelnames)
 
 `Counter`
             </td>
-            <td>The number of datapoints accepted (excluding NaN or Inf values) by models of class `model_alias` from the results of the `query_key` query during the `stage` (`infer`, `fit_infer`), within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
+            <td>The number of valid datapoints accepted by `model_alias`, excluding NaN and Inf values, during `fit`, `infer`, or combined `fit_infer` execution for the `query_key` query.</td>
             <td>
 
 `stage`, `query_key`, `model_alias`, `scheduler_alias`, `preset`
@@ -607,6 +659,10 @@ Label names [description](#labelnames)
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Writer behaviour metrics" %}}
+
 ### Writer behaviour metrics
 Label names [description](#labelnames)
 
@@ -641,7 +697,7 @@ Label names [description](#labelnames)
         <tr>
             <td>
 
-<span style="white-space: nowrap;">`vmanomaly_writer_responses`</span> (named `vmanomaly_reader_response_count`{{% deprecated_from "v1.17.0" anomaly %}})
+<span style="white-space: nowrap;">`vmanomaly_writer_responses`</span> (named `vmanomaly_writer_response_count`{{% deprecated_from "v1.17.0" anomaly %}})
             </td>
             <td>
 
@@ -718,320 +774,261 @@ Label names [description](#labelnames)
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+</div>
+
 ### Labelnames
 
-* `stage` - stage of model - 'fit', 'infer' or 'fit_infer' for models that do it simultaneously, see [model types](https://docs.victoriametrics.com/anomaly-detection/components/models/#model-types).
+* `stage` - model execution stage: `fit`, `infer`, or `fit_infer` for a combined fit/inference scheduler run. See [model types](https://docs.victoriametrics.com/anomaly-detection/components/models/#model-types).
 * `query_key` - query alias from [`reader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/) config section.
 * `model_alias` - model alias from [`models`](https://docs.victoriametrics.com/anomaly-detection/components/models/) config section{{% available_from "v1.10.0" anomaly %}}.
 * `scheduler_alias` - scheduler alias from [`schedulers`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) config section{{% available_from "v1.11.0" anomaly %}}.
 * `preset` - preset alias for [`preset`](https://docs.victoriametrics.com/anomaly-detection/presets/) mode of `vmanomaly`{{% available_from "v1.12.0" anomaly %}}.
 * `url` - writer or reader url endpoint.
-* `code` - response status code or `connection_error`, `timeout`.
-* `step` - json or dataframe reading step.
+* `code` - HTTP response status code or `connection_error`, `timeout`, `ssl_error`, or `io_error`.
+* `step` - reader parsing step: `json` or `df`.
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
 
 ## Logs generated by vmanomaly
 
-The `vmanomaly` service logs operations, errors, and performance for its components (service, reader, writer), alongside [self-monitoring metrics](#metrics-generated-by-vmanomaly) updates. Below is a description of key logs {{% available_from "v1.17.1" anomaly %}} for each component and the related metrics affected.
+The `vmanomaly` service logs important lifecycle, I/O, model, and recovery events alongside
+[self-monitoring metrics](#metrics-generated-by-vmanomaly). The fragments below are stable prefixes for
+recognizing log families, rather than byte-for-byte message contracts; entity values and exception details follow
+the prefix.
 
-`{{X}}` indicates a placeholder in the log message templates described below, which will be replaced with the appropriate entity during logging.
+By default, `vmanomaly` uses the `INFO` level. Use the global `--loggerLevel` command-line argument or
+`settings.logger_levels`{{% available_from "v1.30.0" anomaly %}} for prefix-based component overrides:
 
+```yaml
+settings:
+  logger_levels:
+    reader: DEBUG       # also applies to reader.vm, reader.vlogs, and other child loggers
+    writer.vm: ERROR
+    copilot: WARNING
+```
 
-> By default, `vmanomaly` uses the `INFO` logging level. You can change this by specifying the `--loggerLevel` argument. See command-line arguments [here](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments).
+More-specific prefixes override their parent. Changes limited to `settings.logger_levels` can be
+[hot-reloaded](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) without restarting
+services. See [`settings.logger_levels`](https://docs.victoriametrics.com/anomaly-detection/components/settings/#logger-levels)
+and the [command-line arguments](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments).
 
 - [Startup logs](#startup-logs)
-- [Reader  logs](#reader-logs)
+- [Reader logs](#reader-logs)
 - [Service logs](#service-logs)
-- [Writer  logs](#writer-logs)
+- [Writer logs](#writer-logs)
+- [Scheduler supervision logs](#scheduler-supervision-logs)
+- [Hot-reload logs](#hot-reload-logs)
+- [Persisted-state logs](#persisted-state-logs)
+- [Query server and task logs](#query-server-and-task-logs)
+- [AI Copilot logs](#ai-copilot-logs)
 
+<div class="collapse-group">
+
+
+{{% collapse name="Startup logs" %}}
 
 ### Startup logs
 
-The `vmanomaly` service logs important information during the startup process. This includes checking for the license, validating configurations, and setting up schedulers, readers, and writers. Below are key logs that are generated during startup, which can help troubleshoot issues with the service's initial configuration or license validation.
+Startup logs summarize the version, license, effective storage mode, state restoration, process-pool mode,
+server addresses, hot-reload state, and active schedulers. The most useful prefixes are:
 
----
-
-**License check**. If no license key or file is provided, the service will fail to start and log an error message. If a license file is provided but cannot be read, the service logs a failure. Log messages:
-
-```text
-Please provide a license code using --license or --licenseFile arg, or as VM_LICENSE_FILE env. See https://victoriametrics.com/products/enterprise/trial/ to obtain a trial license.
-```
-
-```text
-failed to read file {{args.license_file}}: {{error_message}}
-```
-
----
-
-**Config validation**. If the service's configuration fails to load or does not meet validation requirements, an error message is logged and the service will exit. If the configuration is loaded successfully, a message confirming the successful load is logged. Log messages:
-
-```text
-Config validation failed, please fix these errors: {{error_details}}
-```
-
-```text
-Config has been loaded successfully.
-```
-
----
-
-**Model and data directory setup**. The service checks the environment variables `VMANOMALY_MODEL_DUMPS_DIR` and `VMANOMALY_DATA_DUMPS_DIR` to determine where to store models and data. If these variables are not set, models and data will be stored in memory. Please find the [on-disk mode details here](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode). Log messages:
-
-```text
-Using ENV MODEL_DUMP_DIR=`{{model_dump_dir}}` to store anomaly detection models.
-```
-```text
-ENV MODEL_DUMP_DIR is not set. Models will be kept in RAM between consecutive `fit` calls.
-```
-```text
-Using ENV DATA_DUMP_DIR=`{{data_dump_dir}}` to store anomaly detection data.
-```
-```text
-ENV DATA_DUMP_DIR is not set. Models' training data will be stored in RAM.
-```
-
----
-
-**Scheduler and service initialization**. After configuration is successfully loaded, the service initializes [schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) and services for each defined `scheduler_alias`. If there are issues with a specific scheduler (e.g., no models or queries found to attach to a scheduler), a warning is logged. When schedulers are initialized, the service logs a list of active schedulers. Log messages:
-
-```text
-Scheduler {{scheduler_alias}} wrapped and initialized with {{N}} model spec(s).
-```
-```text
-No model spec(s) found for scheduler `{{scheduler_alias}}`, skipping setting it up.
-```
-```text
-Active schedulers: {{list_of_schedulers}}.
-```
+- **License check**: `Please provide a license code`, `failed to read file`, and `Licensed to`.
+- **Config validation**: `Config validation failed`, `Config read failed`, and the fatal
+  `Config validation failed, shutting down`. Successful startup ends with `Config has been loaded successfully`.
+- **Model and data directory setup**: `Using ENV VMANOMALY_MODEL_DUMPS_DIR`,
+  `Using ENV VMANOMALY_DATA_DUMPS_DIR`, or their `is not set` in-memory variants. See
+  [on-disk mode](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
+- **Scheduler and service initialization**: `Version:`, `Using process pool executor`, `Listening on`,
+  `Serving /metrics`, `Hot reload enabled`, and `Active schedulers`. `Process pool health check failed, falling
+  back to sequential mode` reports a safe runtime fallback. Per-scheduler wrapping and omitted empty schedulers are
+  `DEBUG` diagnostics.
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
----
+{{% /collapse %}}
+
+{{% collapse name="Reader logs" %}}
 
 ### Reader logs
 
-The `reader` component logs events during the process of querying VictoriaMetrics and retrieving the data necessary for anomaly detection. This includes making HTTP requests, handling SSL, parsing responses, and processing data into formats like DataFrames. The logs help to troubleshoot issues such as connection problems, timeout errors, or misconfigured queries.
+Reader logs cover endpoint checks, request splitting, network failures, response parsing, and coordination between
+queries used by the same model.
 
----
+**Starting a healthcheck request**. The reader probes each configured tenant and discovers
+`search.maxPointsPerTimeseries`. `Max points per timeseries set as` is a `DEBUG` diagnostic. A warning beginning
+`Could not get constraints` means the reader uses its built-in limit. Endpoint initialization errors identify SSL,
+connection, or timeout failures.
 
-**Starting a healthcheck request**. When the `reader` component initializes, it checks whether the VictoriaMetrics endpoint is accessible by sending a request for `_vmanomaly_healthcheck`. Log messages:
+**No data found (False)**. A fit/read range with no results uses this form, showing both local and Unix times:
 
 ```text
-[Scheduler {{scheduler_alias}}] Max points per timeseries set as: {{vm_max_datapoints_per_ts}}
-```
-```text
-[Scheduler {{scheduler_alias}}] Reader endpoint SSL error {{url}}: {{error_message}}
-```
-```text
-[Scheduler {{scheduler_alias}}] Reader endpoint inaccessible {{url}}: {{error_message}}
-```
-```text
-[Scheduler {{scheduler_alias}}] Reader endpoint timeout {{url}}: {{error_message}}
+[Scheduler `SCHEDULER`] No data for query_key `QUERY` between LOCAL_START and LOCAL_END timezone TZ (START_EPOCH to END_EPOCH)
 ```
 
----
+Check the query, tenant, offsets, and selected range.
 
-
-**No data found (False)**. Based on [`query_from_last_seen_timestamp`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) VmReader flag. A `warning` log is generated when no data is found in the requested range. This could indicate that the query was misconfigured or that no new data exists for the time period requested. Log message format:
+**No unseen data found (True)**. An inference read whose timestamps were already processed uses:
 
 ```text
-[Scheduler {{scheduler_alias}}] No data between {{start_s}} and {{end_s}} for query "{{query_key}}"
+[Scheduler `SCHEDULER`] No unseen data for query_key `QUERY` between LOCAL_START and LOCAL_END, timezone TZ (START_EPOCH to END_EPOCH)
 ```
 
----
+This can be expected for overlapping scheduler windows, UI range navigation, or retries. Investigate when it
+persists while the datasource continues receiving newer samples.
 
-**No unseen data found (True)**. Based on [`query_from_last_seen_timestamp`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) VmReader flag. A `warning` log is generated when no new data is returned (i.e., all data has already been seen in a previous inference step(s)). This helps in identifying situations where data for inference has already been processed. Based on VmReader's `adjust` flag. Log messages:
+**Connection or timeout errors**. `Error querying URL for QUERY with PARAMS` includes the effective endpoint,
+query alias, request parameters, and nested SSL, connection, timeout, or I/O reason. The corresponding
+`vmanomaly_reader_responses` code is `ssl_error`, `connection_error`, `timeout`, or `io_error`.
 
-```text
-[Scheduler {{scheduler_alias}}] No unseen data between {{start_s}} and {{end_s}} for query "{{query_key}}"
-```
+At `DEBUG`, reader request lines start with `[Scheduler ...] GET` or `OPTIONS` and show the effective URL;
+token query parameters are redacted. `Cancellation requested for query` records cooperative cancellation.
+`Failed queries detected`, `Timeout waiting for queries`, and `Auto-marking pending queries as failed` identify
+coordination failures for related query sets.
 
----
+**Max datapoints warning**. `Query "QUERY" from START to END with step ... may exceed max datapoints per
+timeseries (LIMIT)` means the range will be split{{% available_from "v1.14.1" anomaly %}}. The message reports the
+effective limit and suggests reducing the range, increasing the step, or raising
+`search.maxPointsPerTimeseries`. A `DEBUG` message reports the resulting interval count.
 
-**Connection or timeout errors**. When the reader fails to retrieve data due to connection or timeout errors, a `warning` log is generated. These errors could result from network issues, incorrect query endpoints, or VictoriaMetrics being temporarily unavailable. Log message format:
+**Multi-tenancy warnings**. Messages starting with `The label vm_account_id was not found` indicate that a
+multitenant query lost routing labels. Preserve `vm_account_id` and `vm_project_id` through query aggregation; see
+[multitenancy support](https://docs.victoriametrics.com/anomaly-detection/components/writer/#multitenancy-support).
 
-```text
-[Scheduler {{scheduler_alias}}] Error querying {{query_key}} for {{url}}: {{error_message}}
-```
-
----
-
-**Max datapoints warning**. If the requested query range (defined by `fit_every` or `infer_every` [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#parameters-1) args) exceeds the maximum number of datapoints allowed by VictoriaMetrics, a `warning` log is generated, and the request is split into multiple intervals{{% available_from "v1.14.1" anomaly %}}. This ensures that the request does not violate VictoriaMetrics’ constraints. Log messages:
-
-```text
-[Scheduler {{scheduler_alias}}] Query "{{query_key}}" from {{start_s}} to {{end_s}} with step {{step}} may exceed max datapoints per timeseries and will be split...
-```
-
----
-
-**Multi-tenancy warnings**. If the reader detects any issues related to missing or misconfigured multi-tenancy labels (a `warning` log{{% available_from "v1.16.2" anomaly %}} is generated to indicate the issue. See additional details [here](https://docs.victoriametrics.com/anomaly-detection/components/writer/#multitenancy-support). Log message format:
-
-```text
-The label vm_account_id was not found in the label set of {{query_key}}, but tenant_id='multitenant' is set in reader configuration...
-```
-
----
-
-**Metrics updated in read operations**. During successful query execution process, the following reader [self-monitoring metrics](#reader-behaviour-metrics) are updated:
-
-- `vmanomaly_reader_request_duration_seconds`: Records the time (in seconds) taken to complete the query request.
-  
-- `vmanomaly_reader_responses`: Tracks the number of response codes received from VictoriaMetrics.
-
-- `vmanomaly_reader_received_bytes`: Counts the number of bytes received in the response.
-
-- `vmanomaly_reader_response_parsing_seconds`: Records the time spent parsing the response into different formats (e.g., JSON or DataFrame).
-
-- `vmanomaly_reader_timeseries_received`: Tracks how many timeseries were retrieved in the query result.
-
-- `vmanomaly_reader_datapoints_received`: Counts the number of datapoints retrieved in the query result.
-
----
-
-**Metrics skipped in case of failures**. If an error occurs (connection or timeout), `vmanomaly_reader_received_bytes`, `vmanomaly_reader_timeseries_received`, and `vmanomaly_reader_datapoints_received` are not incremented because no valid data was received.
+**Metrics updated in read operations**. Requests update duration and response-code metrics even on handled
+failures. Bytes, time series, datapoints, and parsing durations are recorded only when those values were received
+or parsed. See [reader behaviour metrics](#reader-behaviour-metrics).
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Service logs" %}}
+
 ### Service logs
 
-The `model` component (wrapped in service) logs operations during the fitting and inference stages for each model spec attached to particular [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) `scheduler_alias`. These logs inform about skipped runs, connection or timeout issues, invalid data points, and successful or failed model operations.
+The service logs `fit`, `infer`, and combined `fit_infer`/backtesting work for each model alias and scheduler.
+The `query_key` value may be a composite key containing source labels and an internal hash, rather than only the
+configured query alias.
 
----
+**Skipped runs**. Warnings start with `Skipping run for stage 'STAGE' for model 'MODEL'`. Common reasons are no
+fit or inference partition, no data to infer, no unseen valid data, a missing model instance or on-disk model, an
+unsupported exact-batch path, or no valid output. The service attempts fitting when at least one valid row exists;
+individual models may require more history and report their own error. Skips increment
+`vmanomaly_model_runs_skipped`.
 
-**Skipped runs**. When there are insufficient valid data points to fit or infer using a model, the run is skipped and a `warning` log is generated. This can occur when the query returns no new data or when the data contains invalid values (e.g., `NaN`, `INF`). The skipped run is also reflected in the `vmanomaly_model_runs_skipped` metric. Log messages:
+**Errors during model execution**. Errors start with `Error during stage 'STAGE' for model 'MODEL'` and include
+the composite query key and exception. They increment `vmanomaly_model_run_errors`.
 
-When there are insufficient valid data points (at least 1 for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) and 2 for [offline models](https://docs.victoriametrics.com/anomaly-detection/components/models/#offline-models))
-```text
-[Scheduler {{scheduler_alias}}] Skipping run for stage 'fit' for model '{{model_alias}}' (query_key: {{query_key}}): Not enough valid data to fit: {{valid_values_cnt}}
-```
+**Model instance created during inference**. `Model instance 'MODEL' created ... during inference` is a `DEBUG`
+message for an online model cold start{{% available_from "v1.15.2" anomaly %}}.
 
-When all the received timestamps during an `infer` call have already been processed, meaning the [`anomaly_score`](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) has already been produced for those points
-```text
-[Scheduler {{scheduler_alias}}] Skipping run for stage 'infer' for model '{{model_alias}}' (query_key: {{query_key}}): No unseen data to infer on.
-```
-When the model fails to produce any valid or finite outputs (such as [`anomaly_score`](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score))
-```text
-[Scheduler {{scheduler_alias}}] Skipping run for stage 'infer' for model '{{model_alias}}' (query_key: {{query_key}}): No (valid) datapoints produced.
-```
+**Successful model runs**. `Fitting on VALID/TOTAL valid datapoints` is emitted at `INFO`. At `DEBUG`,
+`Model ... fit completed`, `Inference ran in`, and `Fit-Infer ran in` report stage duration. Combined
+`fit_infer` is used by applicable backtesting/scheduler execution and is not a separate “rolling model” class.
 
----
-
-**Errors during model execution**. If the model fails to fit or infer data due to internal service errors or model spec misconfigurations, an `error` log is generated and the error is also reflected in the `vmanomaly_model_run_errors` metric. This can occur during both `fit` and `infer` stages. Log messages:
-```text
-[Scheduler {{scheduler_alias}}] Error during stage 'fit' for model '{{model_alias}}' (query_key: {{query_key}}): {{error_message}}
-```
-```text
-[Scheduler {{scheduler_alias}}] Error during stage 'infer' for model '{{model_alias}}' (query_key: {{query_key}}): {{error_message}}
-```
-
----
-
-**Model instance created during inference**. In cases where an [online model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) instance is created during the inference stage (without a prior fit{{% available_from "v1.15.2" anomaly %}}), a `debug` log is produced. This helps track models that are created dynamically based on incoming data. Log messages:
-
-```text
-[Scheduler {{scheduler_alias}}] Model instance '{{model_alias}}' created for '{{query_key}}' during inference.
-```
----
-
-**Successful model runs**. When a model successfully fits, logs track the number of valid datapoints processed and the time taken for the operation. These logs are accompanied by updates to [self-monitoring metrics](#models-behaviour-metrics) like `vmanomaly_model_runs`, `vmanomaly_model_run_duration_seconds`, `vmanomaly_model_datapoints_accepted`, and `vmanomaly_model_datapoints_produced`. Log messages:
-
-For [non-rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#non-rolling-models)
-```text
-[Scheduler {{scheduler_alias}}] Fitting on {{valid_values_cnt}}/{{total_values_cnt}} valid datapoints for "{{query_key}}" using model "{{model_alias}}".
-```
-```text
-[Scheduler {{scheduler_alias}}] Model '{{model_alias}}' fit completed in {{model_run_duration}} seconds for {{query_key}}.
-```
-For [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) (combined stage)
-```text
-[Scheduler {{scheduler_alias}}] Fit-Infer on {{datapoint_count}} points for "{{query_key}}" using model "{{model_alias}}".
-```
-
----
-
-**Metrics updated in model runs**. During successful fit or infer operations, the following [self-monitoring metrics](#models-behaviour-metrics) are updated for each run:
-
-- `vmanomaly_model_runs`: Tracks how many times the model ran (`fit`, `infer`, or `fit_infer`) for a specific `query_key`.
-
-- `vmanomaly_model_run_duration_seconds`: Records the total time (in seconds) for the model invocation, based on the results of the `query_key`.
-
-- `vmanomaly_model_datapoints_accepted`: The number of valid datapoints processed by the model during the run.
-
-- `vmanomaly_model_datapoints_produced`: The number of datapoints generated by the model during inference.
-
-- `vmanomaly_models_active`: Tracks the number of models currently **available for infer** for a specific `query_key`.
-
----
-
-**Metrics skipped in case of failures**. If a model run fails due to an error or if no valid data is available, the metrics such as `vmanomaly_model_datapoints_accepted`, `vmanomaly_model_datapoints_produced`, and `vmanomaly_model_run_duration_seconds` are not updated.
-
----
+**Metrics updated in model runs**. Successful stages update runs, duration, accepted/produced datapoints, and
+active-model gauges. Skips and failures update their respective counters; success-only values are not recorded for
+an unsuccessful stage. See [models behaviour metrics](#models-behaviour-metrics).
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Writer logs" %}}
+
 ### Writer logs
 
-The `writer` component logs events during the process of sending produced data (like `anomaly_score` [metrics](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score)) to VictoriaMetrics. This includes data preparation, serialization, and network requests to VictoriaMetrics endpoints. The logs can help identify issues in data transmission, such as connection errors, invalid data points, and track the performance of write requests.
+Writer logs cover serialization and delivery of produced series such as
+[`anomaly_score`](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score).
 
----
+**Starting a write request**. At `DEBUG`, `[Scheduler ...] POST URL with N datapoints, M bytes of payload`
+includes the composite query key and dataframe shape.
 
-**Starting a write request**. A `debug` level log is produced when the `writer` component starts the process of writing data to VictoriaMetrics. It includes details like the number of datapoints, bytes of payload, and the query being written. This is useful for tracking the payload size and performance at the start of the request. Log messages:
+**No valid data points**. `No valid datapoints to save for metric` includes the query key and original dataframe
+shape; no request is sent.
 
-```text
-[Scheduler {{scheduler_alias}}] POST {{url}} with {{N}} datapoints, {{M}} bytes of payload, for {{query_key}}
-```
+**Connection, timeout, or I/O errors**. `Cannot write N points for QUERY` ends with an SSL, connection, timeout,
+or I/O reason. A retriable connection failure first emits `Connection error while writing ... reinitializing
+session and retrying`; the final failed attempt is logged as an error.
 
----
+**Multi-tenancy warnings**. `The label vm_account_id was not found` means a `multitenant` writer will fall back to
+tenant `0:0`. `The label set for the metric ... contains multi-tenancy labels` means labels disagree with the
+configured single tenant. Preserve or align tenant labels and `writer.tenant_id`; see
+[multitenancy support](https://docs.victoriametrics.com/anomaly-detection/components/writer/#multitenancy-support).
 
-**No valid data points**. A `warning` log is generated if there are no valid datapoints to write (i.e., all are `NaN` or unsupported like `INF`). This indicates that the writer will not send any data to VictoriaMetrics. Log messages:
-
-```text
-[Scheduler {{scheduler_alias}}] No valid datapoints to save for metric: {{query_key}}
-```
-
----
-
-**Connection, timeout, or I/O errors**. When the writer fails to send data due to connection, timeout, or I/O errors, an `error` log is generated. These errors often arise from network problems, incorrect URLs, or VictoriaMetrics being unavailable. The log includes details of the failed request and the reason for the failure. Log messages:
-
-```text
-[Scheduler {{scheduler_alias}}] Cannot write {{N}} points for {{query_key}}: connection error {{url}} {{error_message}}
-```
-```text
-[Scheduler {{scheduler_alias}}] Cannot write {{N}} points for {{query_key}}: timeout for {{url}} {{error_message}}
-```
-```text
-[Scheduler {{scheduler_alias}}] Cannot write {{N}} points for {{query_key}}: I/O error for {{url}} {{error_message}}
-```
-
----
-
-**Multi-tenancy warnings**. If the `tenant_id` is set to `multitenant` but the `vm_account_id` label is missing from the query result, or vice versa, a `warning` log is produced{{% available_from "v1.16.2" anomaly %}}. This helps in debugging label set issues that may occur due to the multi-tenant configuration - see [this section for details](https://docs.victoriametrics.com/anomaly-detection/components/writer/#multitenancy-support). Log messages:
-
-```text
-The label vm_account_id was not found in the label set of {{query_key}}, but tenant_id='multitenant' is set in writer...
-```
-```text
-The label set for the metric {{query_key}} contains multi-tenancy labels, but the write endpoint is configured for single-tenant mode (tenant_id != 'multitenant')...
-```
-
----
-
-**Metrics updated in write operations**. During the successful write process of *non-empty data*, the following [self-monitoring metrics](#writer-behaviour-metrics) are updated:
-
-- `vmanomaly_writer_request_duration_seconds`: Records the time (in seconds) taken to complete the write request.
-
-- `vmanomaly_writer_sent_bytes`: Tracks the number of bytes sent in the request.
-
-- `vmanomaly_writer_responses`: Captures the HTTP response code returned by VictoriaMetrics. In case of connection, timeout, or I/O errors, a specific error code (`connection_error`, `timeout`, or `io_error`) is recorded instead.
-
-- `vmanomaly_writer_request_serialize_seconds`: Records the time taken for data serialization.
-
-- `vmanomaly_writer_datapoints_sent`: Counts the number of valid datapoints that were successfully sent.
-
-- `vmanomaly_writer_timeseries_sent`: Tracks the number of timeseries sent to VictoriaMetrics.
-
-**Metrics skipped in case of failures**. If an error occurs (connection, timeout, or I/O error), only `vmanomaly_writer_request_duration_seconds` is updated with appropriate error code. 
+**Metrics updated in write operations**. Request duration is observed for successful and handled failed requests.
+`vmanomaly_writer_responses` records the HTTP status or `ssl_error`, `connection_error`, `timeout`, or `io_error`.
+Serialization duration and prepared time-series count may already be recorded before a failed request; sent bytes
+and datapoints are recorded only after a successful response. See [writer behaviour metrics](#writer-behaviour-metrics).
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
+
+{{% /collapse %}}
+
+{{% collapse name="Scheduler supervision logs" %}}
+
+### Scheduler supervision logs
+
+Scheduler supervision{{% available_from "v1.30.0" anomaly %}} logs a dead worker, automatic restart, successful
+recovery, failed-attempt backoff, and removal after the retry limit. Stable prefixes include `Scheduler ... is not
+alive`, `Scheduler ... restarted successfully`, `restart attempt ... failed`, and `reached max restart attempts`.
+Correlate them with `vmanomaly_scheduler_alive` and `vmanomaly_scheduler_restarts_total`.
+
+[Back to logging sections](#logs-generated-by-vmanomaly)
+
+{{% /collapse %}}
+
+{{% collapse name="Hot-reload logs" %}}
+
+### Hot-reload logs
+
+Hot reload logs config-change detection, validation, staged service restart, success, and rollback. `Reload aborted
+– invalid config` keeps the current runtime unchanged; `Reload apply failed; attempting rollback` starts recovery.
+`Rollback failed` is critical and requests shutdown. A logger-only change emits `Applied component log level changes
+without restarting services`.
+
+[Back to logging sections](#logs-generated-by-vmanomaly)
+
+{{% /collapse %}}
+
+{{% collapse name="Persisted-state logs" %}}
+
+### Persisted-state logs
+
+With `settings.restore_state`, startup logs the stored/runtime version assessment, reusable components, required
+model or reader-data purges, and restored jobs/services. `Persisted state is incompatible` followed by `Dropping
+stored artifacts completely` indicates a full reset; missing or unreadable model files are reported separately.
+
+[Back to logging sections](#logs-generated-by-vmanomaly)
+
+{{% /collapse %}}
+
+{{% collapse name="Query server and task logs" %}}
+
+### Query server and task logs
+
+The query server logs its listening address and datasource-proxy timeouts/failures. Background anomaly-detection
+and autotune failures use `Error in task` and `Error in autotune task`; canceled client requests may still leave a
+background raw query finishing cleanly.
+
+[Back to logging sections](#logs-generated-by-vmanomaly)
+
+{{% /collapse %}}
+
+{{% collapse name="AI Copilot logs" %}}
+
+### AI Copilot logs
+
+AI Copilot{{% available_from "v1.30.0" anomaly %}} reports whether it is initialized, disabled, misconfigured, or
+unable to mount. `Invalid Copilot request state` identifies an incomplete/canceled tool-call history, `Copilot
+request failed` identifies provider execution failure, and `MCP server unreachable` identifies unavailable MCP
+guidance tools.
+
+[Back to logging sections](#logs-generated-by-vmanomaly)
+
+{{% /collapse %}}
+
+</div>

docs/anomaly-detection/components/reader.md

@@ -12,23 +12,24 @@ aliases:
   - /anomaly-detection/components/reader.html
 ---
 
-VictoriaMetrics Anomaly Detection (`vmanomaly`) has an input of Prometheus-compatible metrics from either [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) accessed with [VmReader](#vm-reader) with [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) queries or from [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) / [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) accessed with [VLogsReader](#victorialogs-reader) with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries.
+VictoriaMetrics Anomaly Detection (`vmanomaly`) reads Prometheus-compatible metrics from [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) through [VmReader](#vm-reader) and [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/). It can also derive metrics from [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) or [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) through [VLogsReader](#victorialogs-reader) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/).
 
 Future updates will introduce additional readers, expanding the range of data sources `vmanomaly` can work with.
 
 ## Playgrounds
 
-To ease the development and testing of queries for `vmanomaly`'s input data, following playgrounds can be used for experimenting with MetricsQL and LogsQL queries:
+Use the following playgrounds to develop and test input queries:
 
-Please see respective sections below for specific reader:
 - [MetricsQL playground](#metricsql-playground) for `VmReader`
 - [LogsQL playground](#logsql-playground) for `VLogsReader`
 
 ## VM reader
 
+<div class="collapse-group">
+
 {{% collapse name="Queries format migration (to v1.13.0+)" %}}
 
-> There is backward-compatible change{{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg of [VmReader](#vm-reader). New format allows to specify per-query parameters, like `step` to reduce amount of data read from VictoriaMetrics TSDB and to allow config flexibility. Please see [per-query parameters](#per-query-parameters) section for the details.
+> The backward-compatible `queries` format introduced in v1.13.0 allows [VmReader](#vm-reader) parameters such as `step` to be configured per query. This can reduce the amount of data read from VictoriaMetrics. See [per-query parameters](#per-query-parameters) for details.
 
 Old format like
 
@@ -62,6 +63,8 @@ reader:
 ```
 {{% /collapse %}}
 
+{{% collapse name="VM reader per-query parameters and example" %}}
+
 ### Per-query parameters
 
 There is change {{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg format. Now each query alias supports the next (sub)fields, which *override reader-level parameters*, if set:
@@ -126,6 +129,10 @@ reader:
       offset: '-15s'  # to override reader-wise `offset` and query data 15 seconds earlier to account for data collection delays
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="VM reader config parameters and example" %}}
+
 ### Config parameters
 
 <table class="params">
@@ -263,7 +270,33 @@ BasicAuth password. If set, it will be used to authenticate the request.
 `30s`
             </td>
             <td>
-Timeout for the requests, passed as a string
+Backward-compatible timeout used for both datasource fetches and post-fetch processing when `fetch_timeout` or `processing_timeout` are not set.
+            </td>
+        </tr>
+        <tr>
+            <td>
+
+<span style="white-space: nowrap;">`fetch_timeout`</span>
+            </td>
+            <td>
+
+Not set (`timeout` fallback)
+            </td>
+            <td>
+Optional timeout {{% available_from "v1.30.0" anomaly %}} for each datasource read request. Use values such as `5s`, `30s`, or `1m`.
+            </td>
+        </tr>
+        <tr>
+            <td>
+
+<span style="white-space: nowrap;">`processing_timeout`</span>
+            </td>
+            <td>
+
+Not set (`timeout` fallback)
+            </td>
+            <td>
+Optional timeout {{% available_from "v1.30.0" anomaly %}} for post-fetch processing that prepares returned data for fit or inference. High-cardinality queries may need a larger processing timeout than their datasource fetch timeout.
             </td>
         </tr>
         <tr>
@@ -362,6 +395,19 @@ If True, then query will be performed from the last seen timestamp for a given s
         <tr>
             <td>
 
+<span style="white-space: nowrap;">`query_last_seen_max_lookback`</span>
+            </td>
+            <td>
+
+`None`
+            </td>
+            <td>
+Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far `query_from_last_seen_timestamp` may move a query start into the past to recover skipped inference intervals. When configured below the query step, the effective cap is raised to one step. Examples: `5m`, `1h`.
+            </td>
+        </tr>
+        <tr>
+            <td>
+
 <span style="white-space: nowrap;">`latency_offset`</span>
             </td>
             <td>
@@ -395,7 +441,7 @@ Optional arg{{% available_from "v1.17.0" anomaly %}} overrides how `search.maxPo
 `UTC`
             </td>
             <td>
-Optional argument{{% available_from "v1.18.0" anomaly %}} specifies the [IANA](https://nodatime.org/TimeZones) timezone to account for local shifts, like [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models sensitive to seasonal patterns (e.g., [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)). Defaults to `UTC` if not set and can be overridden on a [per-query basis](#per-query-parameters).
+Optional argument {{% available_from "v1.18.0" anomaly %}} specifies the [IANA](https://nodatime.org/TimeZones) timezone to account for local shifts, like [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models sensitive to seasonal patterns (e.g., [`TemporalEnvelopeModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope), [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)). Defaults to `UTC` if not set and can be overridden on a [per-query basis](#per-query-parameters).
             </td>
         </tr>
         <tr>
@@ -461,15 +507,24 @@ reader:
       # tenant_id: '1:0'  # if set, overrides reader-level tenant_id
       # offset: '-15s'  # if set, overrides reader-level offset
   sampling_period: '1m'
+  timeout: '30s'  # backward-compatible default for both phases
+  fetch_timeout: '30s'  # timeout for each datasource request, overrides `timeout` if set
+  processing_timeout: '1m'  # timeout for preparing fetched series for fit/infer, overrides `timeout` if set
   query_from_last_seen_timestamp: True  # false by default
   latency_offset: '1ms'
   series_processing_batch_size: 8
 ```
 
+{{% /collapse %}}
+
+</div>
+
 ### MetricsQL Playground
 
 To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMetrics MetricsQL Playground](https://play.victoriametrics.com/), which provides an interactive environment to test and visualize your queries against sample data. You can also access embedded version of the playground below:
 
+<div class="collapse-group">
+
 {{% collapse name="VictoriaMetrics Playground" %}}
 
 <div class="position-relative mb-3">
@@ -495,6 +550,8 @@ To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMe
 
 {{% /collapse %}}
 
+</div>
+
 ### mTLS protection
 
 `vmanomaly` supports [mutual TLS (mTLS)](https://en.wikipedia.org/wiki/Mutual_authentication){{% available_from "v1.16.3" anomaly %}} for secure communication across its components, including [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters). This allows for mutual authentication between the client and server when querying or writing data to [VictoriaMetrics Enterprise, configured for mTLS](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#mtls-protection).
@@ -530,7 +587,7 @@ reader:
 
 ### Healthcheck metrics
 
-`VmReader` exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
+`VmReader` exposes [several health metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
 
 
 ## VictoriaLogs reader
@@ -639,6 +696,8 @@ Similarly, [VictoriaTraces LogsQL Playground](https://play-vtraces.victoriametri
 
 You can also access **embedded version of the playground below** (VictoriaLogs datasource):
 
+<div class="collapse-group">
+
 {{% collapse name="VictoriaLogs LogsQL Playground" %}}
 
 <div class="position-relative mb-3">
@@ -664,6 +723,11 @@ You can also access **embedded version of the playground below** (VictoriaLogs d
 
 {{% /collapse %}}
 
+</div>
+
+<div class="collapse-group">
+
+{{% collapse name="VictoriaLogs reader config parameters" %}}
 
 ### Config parameters
 
@@ -800,7 +864,33 @@ Frequency of the points returned. Will be converted to `/select/stats_query_rang
 `30s`
             </td>
             <td>
-(Optional) Specifies the maximum duration to wait for a query to complete before timing out. Can be set on a [per-query basis](#per-query-parameters-1) to override the reader-level setting.
+(Optional) Backward-compatible timeout used for both datasource fetches and post-fetch processing when `fetch_timeout` or `processing_timeout` are not set.
+            </td>
+        </tr>
+        <tr>
+            <td>
+
+<span style="white-space: nowrap;">`fetch_timeout`</span>
+            </td>
+            <td>
+
+Not set (`timeout` fallback)
+            </td>
+            <td>
+Optional timeout {{% available_from "v1.30.0" anomaly %}} for each datasource read request. Use values such as `5s`, `30s`, or `1m`.
+            </td>
+        </tr>
+        <tr>
+            <td>
+
+<span style="white-space: nowrap;">`processing_timeout`</span>
+            </td>
+            <td>
+
+Not set (`timeout` fallback)
+            </td>
+            <td>
+Optional timeout {{% available_from "v1.30.0" anomaly %}} for post-fetch processing that prepares returned data for fit or inference. High-cardinality results may need a larger processing timeout than their datasource fetch timeout.
             </td>
         </tr>
         <tr>
@@ -906,9 +996,26 @@ If a path to a CA bundle file (like `ca.crt`), it will verify the certificate us
 Optional argument {{% available_from "v1.29.7" anomaly %}}, allows specifying the number of time series to process together while preparing data for fit or infer stages. Defaults to `8`. Suggested values are 4-16 for high-cardinality queries.
             </td>
         </tr>
+        <tr>
+            <td>
+
+<span style="white-space: nowrap;">`query_last_seen_max_lookback`</span>
+            </td>
+            <td>
+
+`None`
+            </td>
+            <td>
+Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far last-seen recovery may move a query start into the past. Examples: `5m`, `1h`.
+            </td>
+        </tr>
     </tbody>
 </table>
 
+{{% /collapse %}}
+
+{{% collapse name="VictoriaLogs reader per-query parameters and example" %}}
+
 ### Per-query parameters
 
 The names, types and the logic of the per-query parameters subset used in `VLogsReader` are exactly the same as those of [`VmReader`](#vm-reader), please see [per-query parameters](#per-query-parameters) section above for the details. The only difference is that `expr` parameter should contain a valid [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) expression with `stats` [pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe), as described in [query examples](#query-examples) section above.
@@ -927,7 +1034,9 @@ reader:
   series_processing_batch_size: 8
   data_range: [0, 'inf']  # reader-level
   offset: '0s'  # reader-level
-  timeout: '30s'
+  timeout: '30s'  # backward-compatible default for both phases
+  fetch_timeout: '30s'  # timeout for each datasource request, overrides `timeout` if set
+  processing_timeout: '1m'  # timeout for preparing fetched series for fit/infer, overrides `timeout` if set
   queries:
     # one query returning 1 result fields (avg_duration), it will have __name__ label (series name) as `duration_30m__avg`
     duration_avg_30m:
@@ -954,10 +1063,14 @@ reader:
 # other config sections, like models, schedulers, writer, ...
 ```
 
+{{% /collapse %}}
+
+</div>
+
 ### mTLS protection
 
 Please refer to the [mTLS protection](#mtls-protection) section above for details on how to configure mTLS for `VLogsReader`. It uses the same config parameters as `VmReader` for mTLS setup.
 
 ### Healthcheck metrics
 
-Similarly to `VmReader`, `VLogsReader` also exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
+Like `VmReader`, `VLogsReader` exposes [several health metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).

docs/anomaly-detection/components/scheduler.md

@@ -72,6 +72,8 @@ options={`"scheduler.periodic.PeriodicScheduler"`, `"scheduler.oneoff.OneoffSche
 
 > If `start_from` [parameter](#parameters-1) is used, it's suggested to also set `restore_state: true` in the [Settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) of a config, so that the scheduler can restore its state from the previous run **if terminated or restarted in between scheduled runs** and continue producing anomaly scores without interruptions, otherwise the service will be idle until future `start_from` time is reached. E.g. if `start_from` is set to `20:00` and the service is started and then terminated and restarted at `20:30`, it will not produce any anomaly scores until the next day's `20:00` is reached (+23:30 of being idle), which introduces inconvenience for the users.
 
+> {{% available_from "v1.30.0" anomaly %}} If a periodic scheduler worker exits unexpectedly, the service attempts bounded restarts with exponential backoff instead of shutting down unrelated schedulers. Monitor [`vmanomaly_scheduler_alive`](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) and `vmanomaly_scheduler_restarts_total` to alert on persistent failures.
+
 ### Parameters
 
 For periodic scheduler parameters are defined as differences in times, expressed in difference units, e.g. days, hours, minutes, seconds. Time granularity is defined by the last characters of a string. Examples: `"50s"` (seconds), `"4m"` (minutes), `"3h"` (hours), `"2d"` (days), `"1w"` (weeks).

docs/anomaly-detection/components/server.md

@@ -64,3 +64,15 @@ reader:
 After starting the `vmanomaly` server with the above configuration, UI can be accessed at `<vmanomaly-host>:8490/vmanomaly/vmui/` (e.g. `http://localhost:8490/vmanomaly/vmui/`).
 
 Rest API endpoints (e.g. `/metrics`) can be accessed at `<vmanomaly-host>:8490/vmanomaly/metrics` (e.g. `http://localhost:8490/vmanomaly/metrics`).
+
+### Time-series analysis and autotune API
+
+{{% available_from "v1.30.0" anomaly %}} The server exposes bounded endpoints for UI, MCP, and automation workflows:
+
+- `GET /api/v1/timeseries/characteristics` samples the supplied query and summarizes trend, calendar seasonality, changepoints, gaps, and intermittent or spiky behavior. Use `limit` (default 100) to cap sampled series and pass the production `step` and timezone.
+- `POST /api/v1/autotune/tasks` starts asynchronous shared-model tuning. The request contains the query, candidate `tuned_class_name`, expected `anomaly_percentage`, data-source settings, and optimization parameters.
+- `GET /api/v1/autotune/tasks/{task_id}` returns progress and the concrete suggested `modelConfig` when complete.
+- `DELETE /api/v1/autotune/tasks/{task_id}` cancels pending work cooperatively.
+
+> [!TIP]
+> For a complete request and recommended workflow, see [Shared asynchronous autotune workflow](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow). OpenAPI schemas for the running version are available at `/docs` endpoint of a running `vmanomaly` instance.

docs/anomaly-detection/components/settings.md

@@ -226,6 +226,10 @@ monitoring:
   # other monitoring settings
 ```
 
+<div class="collapse-group">
+
+{{% collapse name="State restoration example" %}}
+
 ### Example
 
 For a configuration with the following models, queries and schedulers:
@@ -305,9 +309,13 @@ This means that the service upon restart:
 1. Won't restore the state of `zscore_online` model, because its `z_threshold` argument **has changed**, retraining from scratch is needed on the last `fit_window` = 24 hours of data for `q1`, `q2` and `q3` (as model's `queries` arg is not set so it defaults to all queries found in the reader).
 2. Will **partially** restore the state of `prophet` model, because its class and schedulers are unchanged, but **only instances trained on timeseries returned by `q1` query**. New fit/infer jobs will be set for new query `q3`. The old query `q2` artifacts will be dropped upon restart - all respective models and data for (`prophet`, `q2`) combination will be removed from the database file and from the disk.
 
+{{% /collapse %}}
+
+</div>
+
 ## Retention
 
-{{% available_from "v1.28.1" anomaly %}} The `retention` argument allows to set a [time-to-live](https://en.wikipedia.org/wiki/Time_to_live) (TTL) for service artifacts, such as stored model instances and their training data. When enabled, the service will periodically check (controlled by `check_interval` period) and clean up model instances that have not been used for inference or refitting within the specified period of time (defined in `ttl` argument as a valid period). This helps to manage resources in long-running deployments by removing stale or unused artifacts.
+{{% available_from "v1.28.1" anomaly %}} The `retention` argument sets a [time to live](https://en.wikipedia.org/wiki/Time_to_live) (TTL) for service artifacts such as stored model instances and training data. At each `check_interval`, the service removes artifacts that have not been used for inference or refitting within `ttl`. This bounds stale resource usage in long-running deployments.
 
 ### Use Cases
 - With **[online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models)** as they continuously create model instances for new timeseries over time during inference calls, especially when combined with [periodic schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) with infrequent `fit_every` (say, `90d`).
@@ -322,7 +330,7 @@ The section is **backward-compatible and disabled by default**, meaning that all
 
 `ttl` argument defines the time-to-live period for model instances and their training data. It should be a valid period string (e.g., `7d` for 7 days, `30d` for 30 days, etc.). If a model instance or its training data has not been used for inference or refitting within this period, it will be considered stale and eligible for cleanup.
 
-> If set higher than respective scheduler's `fit_every` period, the ttl will have no effect, as models will always be refitted before they become stale.
+> If `ttl` is greater than a scheduler's `fit_every`, the model is refitted before it becomes stale and the TTL has no effect.
 
 `check_interval` argument defines how often the service should check for stale artifacts. It should be a valid period string (e.g., `1h` for 1 hour, `24h` for 24 hours, etc.). During each check, the service will evaluate all stored model instances and their training data against the defined `ttl` and remove those that are stale.
 
@@ -401,4 +409,4 @@ settings:
     model: WARNING  # applies to all components with 'model' prefix, such as 'model.zscore_online', 'model.prophet', etc.
     # once commented out in hot-reload mode, will use the default logger level set by --loggerLevel command line argument
     # monitoring.push: critical
-```
+```

docs/anomaly-detection/components/vmanomaly-components-diagram.md

@@ -0,0 +1,15 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+
+The required path is `config.yml` → Scheduler → Reader → Model → Writer. The Reader queries the configured VictoriaMetrics, VictoriaLogs, or VictoriaTraces datasource; the Writer stores inferred anomaly scores in VictoriaMetrics. Monitoring is optional and can push metrics or expose them for collection.
+
+Solid nodes and arrows show the required anomaly-detection path. Dashed nodes and arrows show optional self-monitoring integrations.
+
+![vmanomaly component interaction: the Scheduler starts Reader, Model, and Writer work against a configured datasource; Monitoring is optional](/anomaly-detection/components/vmanomaly-components.svg)
+{style="display:block; width:80%; min-width:320px; margin:1.5rem auto"}

docs/anomaly-detection/components/vmanomaly-components.svg

@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1600" role="img" aria-labelledby="title description">
+  <title id="title">How vmanomaly operates during a scheduled iteration</title>
+  <desc id="description">The required flow starts from config.yml and the Scheduler. The Reader queries either VictoriaMetrics through query_range or VictoriaLogs and VictoriaTraces through stats_query_range, then sends data to a Model. The Model produces anomaly scores, and the Writer stores them in VictoriaMetrics through import. Reader, Model, and Writer can optionally report self-monitoring metrics using push or pull monitoring.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <marker id="arrow-optional" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#666a73" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <style>
+      .label { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
+      .title { font-size: 54px; font-weight: 400; }
+      .service-title { font-size: 46px; font-weight: 400; }
+      .node-text { font-size: 34px; font-weight: 600; text-anchor: middle; }
+      .body { font-size: 28px; }
+      .edge-label { font-size: 25px; }
+      .endpoint-text { font-size: 23px; }
+      .source-text { font-size: 21px; }
+      .required-node { fill: #fff; stroke: #303038; stroke-width: 4; }
+      .optional-node { fill: #fff; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
+      .required-line { fill: none; stroke: #303038; stroke-width: 4; marker-end: url(#arrow); }
+      .optional-line { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; marker-end: url(#arrow-optional); }
+      .boundary { fill: none; stroke: #303038; stroke-width: 4; }
+      .optional-boundary { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
+    </style>
+  </defs>
+
+  <rect width="1920" height="1600" fill="#fff"/>
+  <text x="105" y="72" class="label title">How does vmanomaly operate (scheduled iteration example)</text>
+
+  <!-- Required / optional legend -->
+  <g aria-label="Legend" transform="translate(1425 145)">
+    <rect x="0" y="0" width="135" height="62" class="optional-node"/>
+    <rect x="205" y="0" width="135" height="62" class="required-node"/>
+    <path d="M0 100 H135" class="optional-line"/>
+    <path d="M205 100 H340" class="required-line"/>
+    <text x="67" y="162" class="label body" text-anchor="middle">Optional</text>
+    <text x="272" y="162" class="label body" text-anchor="middle">Required</text>
+  </g>
+
+  <!-- Exactly one configured read datasource is used. -->
+  <g aria-label="Configured datasource choice">
+    <rect x="25" y="315" width="300" height="550" class="optional-boundary"/>
+
+    <g transform="translate(34 326)">
+      <path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
+      <ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
+      <rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
+      <text x="144" y="99" class="label endpoint-text" text-anchor="middle">/query_range</text>
+      <text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
+      <text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
+      <text x="144" y="182" class="label source-text" text-anchor="middle">TSDB</text>
+    </g>
+
+    <text x="178" y="600" class="label body" text-anchor="middle">OR</text>
+
+    <g transform="translate(34 632)">
+      <path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
+      <ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
+      <rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
+      <text x="144" y="99" class="label endpoint-text" text-anchor="middle">/stats_query_range</text>
+      <text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
+      <text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaLogs /</text>
+      <text x="144" y="182" class="label source-text" text-anchor="middle">VictoriaTraces</text>
+    </g>
+  </g>
+
+  <!-- vmanomaly service boundary and components -->
+  <g aria-label="vmanomaly service">
+    <rect x="720" y="145" width="610" height="1145" class="boundary"/>
+    <rect x="1005" y="175" width="270" height="100" class="required-node"/>
+    <text x="1140" y="235" class="label node-text">config.yml</text>
+    <text x="1025" y="352" class="label service-title" text-anchor="middle">vmanomaly</text>
+    <text x="1025" y="405" class="label service-title" text-anchor="middle">service</text>
+
+    <rect x="765" y="448" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="500" class="label node-text">Scheduler</text>
+    <rect x="765" y="590" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="642" class="label node-text">Reader</text>
+    <rect x="765" y="732" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="784" class="label node-text">Model</text>
+    <rect x="765" y="874" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="926" class="label node-text">Writer</text>
+    <rect x="778" y="1180" width="420" height="82" rx="16" class="optional-node"/>
+    <text x="988" y="1235" class="label node-text">Monitoring</text>
+
+    <path d="M1005 225 H900 V448" class="required-line"/>
+    <path d="M1210 486 H1290 V628 H1210" class="required-line"/>
+    <path d="M1210 628 H1290 V770 H1210" class="required-line"/>
+    <path d="M1210 770 H1290 V912 H1210" class="required-line"/>
+
+    <path d="M875 666 V1178" class="optional-line"/>
+    <path d="M987 808 V1178" class="optional-line"/>
+    <path d="M1100 950 V1178" class="optional-line"/>
+  </g>
+
+  <!-- Required flow labels -->
+  <text x="1365" y="455" class="label body">
+    <tspan x="1365" dy="0">1. Get the metrics to</tspan>
+    <tspan x="1365" dy="37">a. fit the model, or</tspan>
+    <tspan x="1365" dy="37">b. produce anomaly scores</tspan>
+  </text>
+  <text x="1365" y="620" class="label body">
+    <tspan x="1365" dy="0">3. Model receives data</tspan>
+    <tspan x="1365" dy="37">to train or infer on</tspan>
+  </text>
+  <text x="1365" y="775" class="label body">
+    <tspan x="1365" dy="0">4. Produce anomaly</tspan>
+    <tspan x="1365" dy="37">scores (inference)</tspan>
+  </text>
+
+  <!-- Datasource request and response -->
+  <path d="M765 610 H338" class="required-line"/>
+  <path d="M338 652 H765" class="required-line"/>
+  <text x="370" y="580" class="label edge-label">2.1 Query request</text>
+  <text x="370" y="700" class="label edge-label">2.2 Metrics response</text>
+
+  <!-- Anomaly-score output -->
+  <g aria-label="VictoriaMetrics write endpoint" transform="translate(26 1145)">
+    <path d="M8 32 V184 C8 224 290 224 290 184 V32 C290 70 8 70 8 32Z" class="required-node"/>
+    <ellipse cx="149" cy="32" rx="141" ry="33" class="required-node"/>
+    <rect x="34" y="72" width="230" height="66" rx="16" class="required-node"/>
+    <text x="149" y="99" class="label endpoint-text" text-anchor="middle">/import</text>
+    <text x="149" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
+    <text x="149" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
+    <text x="149" y="182" class="label source-text" text-anchor="middle">TSDB</text>
+  </g>
+  <path d="M765 912 H650 V1095 L316 1197" class="required-line"/>
+  <rect x="338" y="1018" width="292" height="74" fill="#fff"/>
+  <text x="350" y="1048" class="label edge-label">
+    <tspan x="340" dy="0">5. Write anomaly scores</tspan>
+    <tspan x="340" dy="33">to VictoriaMetrics</tspan>
+  </text>
+
+  <!-- Optional monitoring integrations -->
+  <rect x="748" y="1083" width="490" height="48" fill="#fff"/>
+  <text x="993" y="1116" class="label edge-label" text-anchor="middle">6. Produce self-monitoring metrics</text>
+  <rect x="390" y="1460" width="300" height="118" rx="45" class="optional-node"/>
+  <text x="540" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
+  <text x="540" y="1552" class="label body" text-anchor="middle">push approach</text>
+  <rect x="1490" y="1460" width="300" height="118" rx="45" class="optional-node"/>
+  <text x="1640" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
+  <text x="1640" y="1552" class="label body" text-anchor="middle">pull approach</text>
+  <path d="M850 1262 L540 1460" class="optional-line"/>
+  <path d="M1198 1222 L1640 1460" class="optional-line"/>
+  <rect x="590" y="1350" width="180" height="42" fill="#fff"/>
+  <text x="680" y="1380" class="label edge-label" text-anchor="middle">Push metrics</text>
+  <rect x="1230" y="1340" width="415" height="42" fill="#fff"/>
+  <text x="1438" y="1370" class="label edge-label" text-anchor="middle">HTTP GET /metrics or /health</text>
+</svg>

docs/anomaly-detection/components/writer.md

@@ -17,6 +17,10 @@ Future updates will introduce additional export methods, offering users more fle
 
 ## VM writer
 
+<div class="collapse-group">
+
+{{% collapse name="VM writer config parameters and example" %}}
+
 ### Config parameters
 
 <table class="params">
@@ -35,8 +39,7 @@ Future updates will introduce additional export methods, offering users more fle
             </td>
             <td>
 
-<span style="white-space: nowrap;">`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
-</span>
+`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
             </td>
             <td>
 
@@ -63,10 +66,7 @@ Datasource URL address
 <span style="white-space: nowrap;">`tenant_id`</span>
             </td>
             <td>
-<span>
-
 `0:0`, `multitenant`{{% available_from "v1.16.2" anomaly %}}
-</span>
             </td>
             <td>
 
@@ -253,9 +253,8 @@ Token is passed in the standard format with header: `Authorization: bearer {toke
 `path_to_file`
             </td>
             <td>
-<span>
 Path to a file, which contains token, that is passed in the standard format with header: `Authorization: bearer {token}`{{% available_from "v1.15.9" anomaly %}}
-</span>            </td>
+            </td>
         </tr>
         <tr>
             <td>
@@ -267,9 +266,8 @@ Path to a file, which contains token, that is passed in the standard format with
 `1`
             </td>
             <td>
-<span>
 Number of attempts to retry the connection in case of failure {{% available_from "v1.29.2" anomaly %}}.
-</span>            </td>
+            </td>
         </tr>
     </tbody>
 </table>
@@ -293,6 +291,10 @@ writer:
   connection_retry_attempts: 2  # if not specified, it will be 1 by default
 ```
 
+{{% /collapse %}}
+
+</div>
+
 ### Multitenancy support
 
 > This feature applies to the VictoriaMetrics Cluster version only. Tenants are identified by either `accountID` or `accountID:projectID`. `multitenant` [endpoint](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels){{% available_from "v1.15.9" anomaly %}} is supported for writing data across multiple [tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy). For more details, refer to the VictoriaMetrics Cluster [multitenancy documentation](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).
@@ -334,7 +336,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
 
 ### Healthcheck metrics
 
-`VmWriter` exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics). 
+`VmWriter` exposes [several health metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics).
 
 ### Metrics formatting
 

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -17,7 +17,7 @@ sitemap:
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
 
-![typical setup diagram](guide-vmanomaly-vmalert_overview.webp)
+![Typical vmanomaly observability pipeline](guide-vmanomaly-vmalert_overview.svg)
 
 > **Configurations used throughout this guide can be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-integration/)**
 
@@ -395,7 +395,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.7
+    image: victoriametrics/vmanomaly:v1.30.0
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1706 1069" role="img" aria-labelledby="title description">
+  <title id="title">Typical vmanomaly observability pipeline</title>
+  <desc id="description">vmagent scrapes node-exporter metrics and writes them to VictoriaMetrics. vmanomaly reads those metrics and writes anomaly scores back. Grafana visualizes the results. vmalert evaluates rules based on anomaly scores and sends alerts to Alertmanager.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="10" markerHeight="10" orient="auto-start-reverse">
+      <path d="M0 0 10 5 0 10Z" fill="#252525"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #252525; font-size: 40px; }
+      .box { fill: #fff; stroke: #252525; stroke-width: 3; }
+      .arrow { fill: none; stroke: #252525; stroke-width: 4; marker-end: url(#arrow); }
+    </style>
+  </defs>
+
+  <rect width="1706" height="1069" fill="#fff"/>
+
+  <rect x="20" y="55" width="390" height="100" class="box"/>
+  <text x="215" y="119" text-anchor="middle">node-exporter</text>
+  <rect x="730" y="50" width="395" height="100" class="box"/>
+  <text x="928" y="115" text-anchor="middle">vmagent</text>
+  <path d="M710 100 H435" class="arrow"/>
+  <text x="570" y="56" text-anchor="middle">Scrape metrics</text>
+
+  <rect x="745" y="325" width="400" height="180" class="box"/>
+  <text x="945" y="405" text-anchor="middle">VictoriaMetrics</text>
+  <text x="945" y="457" text-anchor="middle">TSDB</text>
+  <path d="M945 150 V322" class="arrow"/>
+  <text x="1018" y="204">
+    <tspan x="1018" dy="0">Push node</tspan>
+    <tspan x="1018" dy="50">exporter</tspan>
+    <tspan x="1018" dy="50">metrics</tspan>
+  </text>
+
+  <rect x="25" y="340" width="395" height="100" class="box"/>
+  <text x="222" y="405" text-anchor="middle">vmanomaly</text>
+  <path d="M725 365 H440" class="arrow"/>
+  <text x="575" y="319" text-anchor="middle">Read metrics</text>
+  <path d="M440 421 H725" class="arrow"/>
+  <text x="570" y="508" text-anchor="middle">
+    <tspan x="570" dy="0">Write produced</tspan>
+    <tspan x="570" dy="50">anomaly scores</tspan>
+  </text>
+
+  <rect x="1440" y="330" width="245" height="170" class="box"/>
+  <text x="1562" y="430" text-anchor="middle">Grafana</text>
+  <path d="M1160 417 H1418" class="arrow"/>
+  <text x="1285" y="486" text-anchor="middle">
+    <tspan x="1285" dy="0">Visualize the</tspan>
+    <tspan x="1285" dy="50">results</tspan>
+  </text>
+
+  <rect x="750" y="738" width="395" height="100" class="box"/>
+  <text x="948" y="805" text-anchor="middle">vmalert</text>
+  <path d="M946 525 V716" class="arrow"/>
+  <text x="1000" y="644">
+    <tspan x="1000" dy="0">Evaluate rules based</tspan>
+    <tspan x="1000" dy="50">on anomaly scores</tspan>
+  </text>
+
+  <rect x="755" y="948" width="395" height="100" class="box"/>
+  <text x="952" y="1015" text-anchor="middle">alertmanager</text>
+  <path d="M948 858 V928" class="arrow"/>
+  <text x="1004" y="906">Send alerts</text>
+</svg>

docs/anomaly-detection/vmanomaly-sharding-ha-diagram.d2

@@ -0,0 +1,107 @@
+# Render with D2 v0.7.1:
+# d2 --layout elk --theme 0 --pad 20 vmanomaly-sharding-ha-diagram.d2 vmanomaly-sharding-ha-diagram.svg
+
+grid-columns: 1
+grid-gap: 36
+
+classes: {
+  boundary: {
+    style.fill: "#F7F7F8"
+    style.stroke: "#A7AAB2"
+    style.stroke-width: 2
+  }
+  process: {
+    style.fill: "#FFFFFF"
+    style.stroke: "#303038"
+    style.stroke-width: 2
+    style.border-radius: 8
+  }
+  member: {
+    style.fill: "#F1F3F5"
+    style.stroke: "#59616A"
+    style.stroke-width: 2
+    style.border-radius: 8
+  }
+}
+
+flow: "" {
+  grid-columns: 3
+  grid-gap: 36
+  style.fill: transparent
+  style.stroke: transparent
+
+  global: "Global YAML\nconfiguration" {
+    shape: page
+    class: process
+  }
+
+  splitting: Configuration splitting {
+    class: boundary
+    grid-columns: 1
+    grid-gap: 24
+
+    split: "Split by VMANOMALY_SPLIT_BY\ndefault: complete" {
+      class: process
+    }
+    subconfigs: "N valid sub-configurations\nn = 0 ... N-1" {
+      class: process
+      style.multiple: true
+    }
+
+    split -> subconfigs: {
+      style.stroke: "#303038"
+    }
+  }
+
+  placement: Deterministic placement {
+    class: boundary
+    grid-columns: 1
+    grid-gap: 24
+
+    settings: "K members: VMANOMALY_MEMBERS_COUNT\nMember index: VMANOMALY_MEMBER_NUM\nR replicas: VMANOMALY_REPLICATION_FACTOR" {
+      class: process
+    }
+    assign: "Assign every sub-configuration\nto exactly R members" {
+      class: process
+    }
+
+    settings -> assign: {
+      style.stroke: "#303038"
+    }
+  }
+}
+
+members: K vmanomaly members {
+  class: boundary
+  grid-columns: 3
+  grid-gap: 30
+
+  member0: "Member 0\nvmanomaly service 1" {
+    class: member
+  }
+  member1: "Member 1\nvmanomaly service 2" {
+    class: member
+  }
+  memberK: "Member K-1\nvmanomaly service K" {
+    class: member
+  }
+}
+
+flow.global -> flow.splitting.split: {
+  style.stroke: "#303038"
+}
+flow.splitting.subconfigs -> flow.placement.assign: {
+  style.stroke: "#303038"
+}
+flow.placement.assign -> members.member0: "assigned subset" {
+  style.stroke: "#59616A"
+  style.stroke-width: 2
+}
+flow.placement.assign -> members.member1: "assigned subset" {
+  style.stroke: "#59616A"
+  style.stroke-width: 2
+}
+flow.placement.assign -> members.memberK: "assigned subset" {
+  style.stroke: "#59616A"
+  style.stroke-width: 2
+}

docs/anomaly-detection/vmanomaly-sharding-ha-diagram.md

@@ -0,0 +1,13 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+
+The global configuration is split into `N` independently valid sub-configurations. Deterministic placement distributes them across `K` members, and each sub-configuration is assigned to exactly `R` members when replication is enabled. Member `k` processes only its assigned subset.
+
+![vmanomaly sharding and high availability: a global configuration is split into N sub-configurations and replicated across K members](/anomaly-detection/vmanomaly-sharding-ha-diagram.svg)
+{style="display:block; width:50%; min-width:320px; margin:1.5rem auto"}

docs/guides/connecting-vm-components-to-cloud-storage/README.md

@@ -0,0 +1,426 @@
+Several VictoriaMetrics components can connect to cloud storage to read or write object data.
+
+The following table shows the supported types of storage for each component:
+
+| Component | AWS S3 and S3-compatible | Google Cloud Storage | Azure Blob Storage |
+|-----------|----|----------------------|--------------------|
+| [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) | ✅ | ✅ | ✅ |
+| [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/) | ✅ | ✅ | ✅ |
+| [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) | ✅ | ✅ | ✅ |
+| [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) |  ✅ | ✅ | ❌ |
+
+All these components use the same underlying libraries, so the authentication setup is largely the same. The main difference is in the command-line flags: 
+
+- vmalert uses `-s3.*` prefixed flags (e.g., `-s3.credsFilePath`)
+- backup and restore tools use unprefixed flags (e.g., `-credsFilePath`)
+
+See the [component reference](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/#per-component-flag-reference) for details.
+
+## Obtaining credentials
+
+You need to supply credentials so the component can connect to the cloud storage. The setup differs by provider; the sections below cover AWS S3, S3‑compatible systems, GCS, and Azure Blob Storage.
+
+### AWS S3
+
+1. In AWS, [create an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) or role with the minimum permissions for the target bucket (see table below).
+1. [Create an access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) for that IAM identity.
+1. Copy the **Access key ID** and **Secret access key** values. You will use them in the credentials file or environment variables.
+
+| Component       | S3 usage                                                    | Minimum S3 permissions |
+|-----------------|-------------------------------------------------------------|------------------------|
+| vmalert (Enterprise) | Reads alerting/recording rules from bucket.    | `s3:GetObject`, `s3:ListBucket` on the rules bucket/prefix.|
+| vmrestore       | Restores backups from S3 buckets                            | `s3:GetObject`, `s3:ListBucket` on the backup bucket/prefix.|
+| vmbackup        | Uploads backups to S3 and may delete old backup objects during maintenance.| `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject` on the backup bucket/prefix.|
+| vmbackupmanager (Enterprise) | Automates backups using vmbackup behavior.                  | Same as vmbackup. |
+
+
+### S3-compatible storage (MinIO, Ceph)
+
+Generate access keys using your storage system's admin interface or CLI. The credentials follow the same format as AWS S3.
+
+### Google Cloud Storage
+
+1. Open the Google Cloud Console and go to **IAM & Admin > Service Accounts**.
+1. Click **Create service account**, enter a name, and assign a Storage role (for example, Storage Object Admin).
+1. Open the service account, go to **Keys**, then click **Add key > Create new key**.
+1. Choose **JSON** as the key type and click **Create**.
+1. Store the JSON file on the machine running the VictoriaMetrics component.
+
+### Azure Blob Storage
+
+> Azure does not use credential files.
+
+1. Log in to the Azure Portal.
+1. Search for and select **Storage accounts**.
+1. Click on your specific storage account name. (This is your `AZURE_STORAGE_ACCOUNT_NAME`).
+1. In the left menu under **Security + networking**, click **Access keys**.
+1. Copy the key value from either **key1** or **key2** (this is your `AZURE_STORAGE_ACCOUNT_KEY`).
+1. Define the access keys as environment variables.
+    ```sh
+    export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
+    export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
+    ```
+
+## Authenticating with the cloud provider
+
+Provide the credentials as a file or with environment variables, along with the path to the cloud storage bucket. The syntax for the bucket name depends on the cloud provider:
+
+- `s3://`: for AWS S3 and self-hosted S3-compatible storage (MinIO, Ceph)
+- `gs://`: Google Cloud Storage
+- `azblob://`: Azure Blob Storage
+
+### vmbackup and vmrestore
+
+The following example backups to an AWS S3 bucket using a credentials file:
+
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=s3://victoriametrics-backup/backup01 \
+  -credsFilePath=/etc/credentials
+```
+
+In order to restore from the same backup from AWS S3:
+
+```sh
+vmrestore \
+  -src=s3://victoriametrics-backup/backup01 \
+  -storageDataPath=/data \
+  -credsFilePath=/etc/credentials
+
+```
+
+> To use non-AWS S3 buckets such as MinIO or Ceph, you must [supply the `-customS3Endpoint` argument](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/#s3-compatible).
+
+Alternatively, you can set the access keys as environment variables instead of using a credential file:
+
+```sh
+export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
+
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=s3://victoriametrics-backup/backup01
+
+vmrestore \
+  -src=s3://victoriametrics-backup/backup01 \
+  -storageDataPath=/data
+```
+
+
+Backups on Google Cloud Storage use the `gs://` prefix in the destination:
+
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=gs://victoriametrics-backup/backup01 \
+  -credsFilePath=/etc/credentials
+```
+
+You can restore this backup with:
+
+```sh
+vmrestore \
+  -src=gs://victoriametrics-backup/backup01 \
+  -storageDataPath=/data \
+  -credsFilePath=/etc/credentials
+```
+
+On Google Cloud, you can define the path to the JSON credential file with `GOOGLE_APPLICATION_CREDENTIALS`. For example:
+
+```sh
+export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
+
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=gs://victoriametrics-backup/backup01
+
+vmrestore \
+  -src=gs://victoriametrics-backup/backup01 \
+  -storageDataPath=/data
+```
+
+For Azure Blob Storage, use the `azblob://` prefix and rely on environment variables instead of `-credsFilePath`.
+
+```sh
+export AZURE_STORAGE_ACCOUNT_NAME=myaccount
+export AZURE_STORAGE_ACCOUNT_KEY=mykey
+
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=azblob://victoriametrics-backup/backup01
+
+vmrestore \
+  -src=azblob://victoriametrics-backup/backup01 \
+  -storageDataPath=/data
+```
+
+### vmbackupmanager 
+
+> vmbackupmanager only works in the [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) edition.
+
+To manage backups with vmbackupmanager on AWS S3, add the credentials with the `-credsFilePath` flag:
+
+```sh
+vmbackupmanager \
+  -dst=s3://vmstorage-data/backups \
+  -credsFilePath=/etc/credentials \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+Or define the access keys using environment variables:
+
+```sh
+export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
+
+vmbackupmanager \
+  -dst=s3://vmstorage-data/backups \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+> To use non-AWS S3 buckets, you must [supply the `-customS3Endpoint` argument](#s3-compatible).
+
+Automated backups on Google Cloud Storage take the following form:
+
+```sh
+vmbackupmanager \
+  -dst=gs://vmstorage-data/backups \
+  -credsFilePath=/etc/credentials \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+As with vmbackup and vmrestore, you can also define the path to the JSON credential file with `GOOGLE_APPLICATION_CREDENTIALS`:
+
+```sh
+export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
+
+vmbackupmanager \
+  -dst=gs://vmstorage-data/backups \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+vmbackupmanager can also use Azure Blob Storage by defining environment variables:
+
+```sh
+export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
+export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
+
+vmbackupmanager \
+  -dst=azblob://vmstorage-data/backups \
+  -storageDataPath=/vmstorage-data \
+  -snapshot.createURL=http://vmstorage:8482/snapshot/create \
+  -licenseFile=/etc/vm-license
+```
+
+### vmalert
+
+> - vmalert cloud storage command line flags are prefixed with `-s3.` for S3 buckets *and* Google Cloud Storage.
+> - Cloud storage only works in the [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) edition.
+
+Read alerting rules from an S3 bucket. The `-rule` flag accepts a prefix, so it matches all files starting with `alerts_` in the `rules` folder:
+
+```sh
+vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+Instead of a credential file, you can supply the access keys using environment variables:
+
+```sh
+export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
+
+vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+> To use non-AWS S3 buckets, you must [supply the `-s3.customEndpoint` argument](#s3-compatible).
+
+To read rules from Google Cloud Storage:
+
+```sh
+vmalert \
+  -rule=gs://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/gcp-service-account.json \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+If you prefer, you can supply the path to the JSON credential file with the `GOOGLE_APPLICATION_CREDENTIALS` environment variable:
+
+```sh
+export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
+
+vmalert \
+  -rule=gs://my-alert-bucket/rules/alerts_ \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+## Credentials files format
+
+The file format depends on the storage provider.
+
+### S3 credentials
+
+The file uses the standard AWS shared credentials format used by the [AWS CLI](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) and [AWS SDKs](https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html):
+
+```ini
+[default]
+aws_access_key_id=YOUR_AWS_ACCESS_KEY
+aws_secret_access_key=YOUR_AWS_SECRET_ACCESS_KEY
+```
+
+You can define multiple profiles in a single file:
+
+```ini
+[default]
+aws_access_key_id=DEFAULT_ACCESS_KEY
+aws_secret_access_key=DEFAULT_SECRET_KEY
+
+[monitoring]
+aws_access_key_id=MONITORING_ACCESS_KEY
+aws_secret_access_key=MONITORING_SECRET_KEY
+
+[alerts]
+aws_access_key_id=ALERTS_ACCESS_KEY
+aws_secret_access_key=ALERTS_SECRET_KEY
+```
+
+Use the `-configProfile` flag (or `-s3.configProfile` in vmalert) to select a non-default profile:
+
+```sh
+-configProfile=alerts
+```
+
+You can separate credentials from other configuration settings. Put credentials in one file:
+
+```ini
+[default]
+aws_access_key_id=DEFAULT_ACCESS_KEY
+aws_secret_access_key=DEFAULT_SECRET_KEY
+```
+
+And non-sensitive settings in another:
+
+```ini
+[default]
+region=us-east-1
+```
+
+Then pass both files:
+
+```sh
+-configFilePath=/etc/aws-config \
+-credsFilePath=/etc/credentials
+```
+
+### GCS credentials file format
+
+The file is the JSON key downloaded from Google Cloud Console. Its content looks like this:
+
+```json
+{
+  "type": "service_account",
+  "project_id": "project-id",
+  "private_key_id": "key-id",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
+  "client_email": "service-account-email",
+  "client_id": "client-id",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
+}
+```
+
+This is the standard service account key format defined by [Google Cloud IAM](https://developers.google.com/workspace/guides/create-credentials).
+
+### Azure Blob Storage
+
+Azure does not support credentials via file. Use environment variables instead.
+
+```sh
+export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
+export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
+```
+
+## Self-hosted and S3-compatible endpoints
+
+For S3-compatible storage such as MinIO or Ceph, set a custom endpoint with the `-customS3Endpoint` flag  for vmbackup, vmrestore, and vmbackupmanager. For example:
+
+```sh
+vmbackup \
+  -storageDataPath=/data \
+  -snapshot.createURL=http://localhost:8428/snapshot/create \
+  -dst=s3://victoriametrics-backup/backup01 \
+  -customS3Endpoint=http://minio.example.local:9000
+```
+
+On vmalert, use the `-s3.customEndpoint` flag instead:
+
+```sh
+vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -s3.customEndpoint=http://minio.example.local:9000 \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+### Addressing S3-compatible buckets {#s3-compatible}
+
+When connecting to non-AWS S3-compatible buckets, there is an additional flag you might need to configure:
+
+- `-s3ForcePathStyle`: on vmbackupmanager, vmbackup, and vmrestore.
+- `-s3.forcePathStyle`: on vmalert.
+
+The flag changes the expected URL pattern for a bucket.
+
+| Flag value | Address-style | Example | Use with |
+|------------|---------------|---------|----------|
+| `true` (default) | Path-style | `https://endpoint/bucket/key` |  MinIO, Ceph, most S3-compatible storages |
+| `false`        | Virtual host-style | `https://bucket.endpoint/key` | [Aliyun OSS](https://www.aliyun.com/product/oss) and other endpoints that require it |
+
+> The flag only takes effect when you use a custom endpoint (`-customS3Endpoint` or `-s3.customEndpoint` on vmalert). When connecting to real AWS S3, the SDK handles addressing automatically.
+
+## Per-component flag reference
+
+The table below shows how the same concept maps to different flag names across components.
+
+| Concept | vmalert | vmbackup, vmrestore, and vmbackupmanager |
+|---|---|---|---|---|
+| Credentials file | `-s3.credsFilePath` | `-credsFilePath` |
+| Config file | `-s3.configFilePath` | `-configFilePath` |
+| Profile selection | `-s3.configProfile` | `-configProfile` |
+| Custom endpoint | `-s3.customEndpoint` | `-customS3Endpoint` |
+| Force path style | `-s3.forcePathStyle` | `-s3ForcePathStyle` |
+| TLS insecure | N/A | `-s3TLSInsecureSkipVerify` |
+| Storage class | N/A | `-s3StorageClass` |
+
+
+

docs/guides/connecting-vm-components-to-cloud-storage/_index.md

@@ -0,0 +1,12 @@
+---
+weight: 5
+title: Connecting VictoriaMetrics components to cloud storage
+menu:
+  docs:
+    parent: guides
+    weight: 5
+tags:
+  - metrics
+  - guide
+---
+{{% content "README.md" %}}

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -829,12 +829,9 @@ See also [minimum downtime strategy](#minimum-downtime-strategy).
 
 ## Slowness-based re-routing
 
-By default, `vminsert` nodes limit the cluster's overall ingestion rate to the throughput of the slowest `vmstorage` node. 
-This ensures that incoming metrics are evenly distributed across all `vmstorage` nodes. 
-The downside is that a single slow vmstorage node can throttle the entire cluster.
-
-When `-disableRerouting=false` is enabled on `vminsert`, 
-the cluster will automatically re-route writes away from the slowest vmstorage node to preserve maximum ingestion throughput.
+By default{{% available_from "#" %}}, `vminsert` automatically re-routes writes away from the slowest `vmstorage` node
+to preserve maximum ingestion throughput. This prevents a single slow `vmstorage` node
+from throttling the entire cluster.
 
 Re-routing occurs only when all of the following conditions hold:
 - the storage send buffer is full.
@@ -842,11 +839,15 @@ Re-routing occurs only when all of the following conditions hold:
 - the vmstorage cluster have much lower saturation overall.
 - the vmstorage cluster has at least three ready nodes.
 
-Enable slowness-based re-routing when peak write throughput matters more 
-than minimizing the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series)
-or keeping metrics perfectly balanced across nodes.
+Disable slowness-based re-routing with `-disableRerouting=true` when keeping metrics
+perfectly balanced across nodes or minimizing the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) 
+matters more than peak write throughput.
 
-The rerouting and node saturation could be seen at  [VictoriaMetrics - cluster](https://grafana.com/grafana/dashboards/11176) dashboard.
+Slowness-based re-routing is automatically disabled{{% available_from "#" %}} when `-replicationFactor` is greater than `1`,
+because rerouting does not guarantee that replicated copies land on distinct storage nodes,
+which violates the replication contract.
+
+The rerouting and node saturation could be seen at [VictoriaMetrics - cluster](https://grafana.com/grafana/dashboards/11176) dashboard.
 
 ## Capacity planning
 

docs/victoriametrics/LTS-releases.md

@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
 
 ## Currently supported LTS release lines
 
-- v1.136.x - the latest one is [v1.136.4 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.4)
-- v1.122.x - the latest one is [v1.122.19 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.19)
+- v1.148.x - the latest one is [v1.148.0 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
+- v1.136.x - the latest one is [v1.136.14 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.14)

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,7 +26,22 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow specifying individual `url_prefix` list items as a mapping instead of a plain url string, so a group of backend urls can override `load_balancing_policy`, `discover_backend_ips`, `max_concurrent_requests` and `tls_*` options, instead of inheriting them from the enclosing `user` / `url_map` scope. This makes it possible, for example, to prefer a primary backend group over a standby one via the outer `load_balancing_policy`, while independently load-balancing across the targets discovered for each group. Such a group can also be given an optional `name`, used to identify it in [metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting) (falls back to its ordinal position in `url_prefix` when not set). See [these docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#load-balancing) for more details.
+**Update Note 1:** `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the default value of `-disableRerouting` flag has changed from `true` to `false`, enabling [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. If you rely on the old behavior, pass `-disableRerouting` command-line flag to `vminsert`. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
+
+**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints now require `POST` method. Previously, it also accepted `GET` requests. If you use `GET` requests for this endpoint, update your scripts or tooling to use `POST` instead. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
+
+* SECURITY: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): restrict `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints to `POST` method only to prevent some [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery)-based data deletion attacks. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
+
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
+* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enable [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Previously, `-disableRerouting` defaulted to `true`, which limited ingestion throughput to the slowest `vmstorage` node. Now `-disableRerouting` defaults to `false`, so `vminsert` automatically routes data away from the slowest `vmstorage` node, improving overall ingestion performance. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the selected auto-refresh interval in the URL. See [VictoriaLogs#1310](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1310).
+* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
+* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-remoteWrite.obfuscateLabels` flag for hashing values of the specified labels before sending metrics to the corresponding `-remoteWrite.url`. This allows sharing metrics with external systems while keeping sensitive label values hidden. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
+
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
+* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
 
 ## [v1.148.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
 
@@ -38,6 +53,7 @@ Released at 2026-07-20
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): support scraping metrics over Unix domain sockets. The socket path can be configured via the `__unix_socket__` target label. See [#11156](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11156). Thanks to @vinyas-bharadwaj for contribution.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Improve background discovery performance for [http_sd](https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs) discovery. See [#8838](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8838).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add client side least-loaded load-balancing with `DNS` discovery. See [#2388](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2388) and these [vmagent DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmagent/#dns-urls), [vmalert DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmalert/#dns-urls).
 * FEATURE: [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-maxBackfillAge` command-line flag for limiting ingestion of samples with historical timestamps, for example, when older data has been moved between storage tiers (nvme/hdd, hot/cold). See [#11199](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11199). Thanks to @AshwinRamaniPsg for contribution.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): automatically preload relabeling rules configured via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` in the [metrics relabel debug UI](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabel-debugging). See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918).
 
@@ -52,6 +68,7 @@ Released at 2026-07-20
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): return `408 Request Timeout` instead of `400 Bad Request` when the request body isn't received within `-maxQueueDuration`. This prevents `vmagent` from incorrectly downgrading the remote write protocol and dropping data when `vmauth` is used as a proxy for a remote write endpoint. See [#11272](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11272).
+
 ## [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0)
 
 Released at 2026-07-06
@@ -231,7 +248,6 @@ Released at 2026-04-24
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect evaluation of binary operations caused by an ordering bug (e.g. `10 - (3 + 3 + 4)` being evaluated as `10 - 3 + 3 + 4`). The issue was introduced in v1.140.0, v1.136.4, and v1.122.19. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
 
 ## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
-
 Released at 2026-04-10
 
 **Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).

docs/victoriametrics/vmagent.md

@@ -323,7 +323,8 @@ flowchart TB
       H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
       H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
       H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
-      H5 --> H6[[push to <b>-remoteWrite.url</b>]]
+      H5 --> H6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
+      H6 --> H7[[push to <b>-remoteWrite.url</b>]]
 
       %% Right branch
       G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
@@ -331,7 +332,8 @@ flowchart TB
       R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
       R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
       R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
-      R5 --> R6[[push to <b>-remoteWrite.url</b>]]
+      R5 --> R6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
+      R6 --> R7[[push to <b>-remoteWrite.url</b>]]
 ```
 
 Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:
@@ -466,6 +468,43 @@ and `-remoteWrite.streamAggr.config`:
 
 There is also the `-promscrape.configCheckInterval` command-line flag, which can be used to automatically reload configs from the updated `-promscrape.config` file.
 
+## DNS URLs
+
+If `vmagent` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses
+via [DNS A records](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1). The port from the original URL is appended to each discovered IP address.
+Each discovered IP address is used for least-loaded balancing of write requests.
+
+DNS URLs are supported in the following places:
+
+* In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
+  `192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428/api/v1/write` is automatically resolved into
+  `-remoteWrite.url=http://192.168.1.15:8428/api/v1/write`.
+
+DNS URLs are useful when client-side HTTP load balancing is needed. A good example
+is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
+which returns multiple IP addresses for a single hostname.
+
+### DNS URLs and HTTPS
+
+When a `dns+` URL uses the `https` scheme, `vmagent` connects to the discovered
+IP addresses directly. This affects [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security)
+in two ways:
+
+* No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) is sent in the TLS handshake,
+  since the connection target is an IP address rather than a hostname.
+* The server certificate is verified against the IP address, so the verification fails
+  unless the certificate contains the corresponding
+  [IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
+
+To use `dns+` URLs with HTTPS, pass the original hostname via the `-remoteWrite.tlsServerName`
+command-line flag. It is used both as SNI and as the name the server certificate
+is verified against:
+
+```sh
+-remoteWrite.url=https://dns+victoria-metrics:8428/api/v1/write
+-remoteWrite.tlsServerName=victoria-metrics
+```
+
 ## SRV URLs
 
 If `vmagent` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
@@ -476,7 +515,7 @@ SRV URLs are supported in the following places:
 * In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
   `victoria-metrics-host:8428` TCP address, then `-remoteWrite.url=http://srv+victoria-metrics/api/v1/write` is automatically resolved into
   `-remoteWrite.url=http://victoria-metrics-host:8428/api/v1/write`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmagent`
-  uses a randomly chosen address for each connection it establishes to the remote storage.
+   performs per request least-loaded load-balancing.
 
 * In scrape target addresses aka `__address__` label. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#how-to-modify-scrape-urls-in-targets) for details.
 
@@ -646,6 +685,35 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
   /path/to/vmagent -remoteWrite.url=http://127.0.0.1:8428/api/v1/write?extra_label="env=prod"
   ```
 
+## Obfuscating label values
+
+`vmagent` can obfuscate the values of specified labels before sending metrics to `-remoteWrite.url`
+via `-remoteWrite.obfuscateLabels`{{% available_from "#" %}}.
+
+This is useful when one or more `-remoteWrite.url` endpoints point to external monitoring services
+outside the organization, and sensitive label values such as `ip`, `host`, `instance`, or `datacenter`
+must not be exposed.
+
+Label values are replaced with their SHA-256 hex digests. No salt is applied, so values with a small
+or predictable value space can potentially be recovered by brute force.
+
+This feature pairs well with [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange):
+use `-remoteWrite.mdx.enable=true` to forward only VictoriaMetrics self-monitoring metrics to an
+external vendor while hiding sensitive label values with `-remoteWrite.obfuscateLabels`.
+
+Use `-remoteWrite.obfuscateLabels` to list label names whose values should be obfuscated for the
+corresponding `-remoteWrite.url`. Separate multiple label names with `^^`.
+
+```sh
+./vmagent \
+  -remoteWrite.url=http://<external-service1> \
+  -remoteWrite.obfuscateLabels='instance^^datacenter' \
+  -remoteWrite.url=http://<external-service2> \
+  -remoteWrite.obfuscateLabels='instance' \
+  -remoteWrite.url=http://<internal-service> \
+  -remoteWrite.obfuscateLabels='' 
+```
+
 ## Automatically generated metrics
 
 `vmagent` automatically generates the following metrics for each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)

docs/victoriametrics/vmalert.md

@@ -546,7 +546,7 @@ tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags) and
 
 ## Reading rules from object storage
 
-[Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
+The [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
 from object storage:
 
 * `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket
@@ -563,6 +563,48 @@ The following [command-line flags](#flags) can be used for fine-tuning access to
 * `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
 * `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
 
+### S3 (AWS and S3-compatible)
+
+The following example reads rules from an S3 bucket:
+
+```sh
+./bin/vmalert \
+  -rule=s3://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+For S3-compatible backends such as [MinIO](https://www.min.io/) or [Ceph](https://ceph.io/), add `-s3.customEndpoint`:
+
+```sh
+./bin/vmalert \
+  -rule=s3://victoriametrics-alert-rules/rules_ \
+  -s3.credsFilePath=/etc/vmalert/aws-credentials \
+  -s3.customEndpoint=http://minio.example.local:9000 \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for details on creating IAM users, credentials file format, environment variables, and IAM roles.
+
+### Google Cloud Storage (GCS)
+
+The following example reads rules from a GCS bucket:
+
+```sh
+./bin/vmalert \
+  -rule=gs://my-alert-bucket/rules/alerts_ \
+  -s3.credsFilePath=/etc/vmalert/gcp-service-account.json \
+  -datasource.url=http://vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager:9093 \
+  -licenseFile=/etc/vm-license
+```
+
+See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for details on creating service accounts and downloading JSON keys.
+
 ## Topology examples
 
 The following sections are showing how `vmalert` may be used and configured
@@ -1470,6 +1512,59 @@ alert_relabel_configs:
 
 The configuration file can be [hot-reloaded](#hot-config-reload).
 
+## DNS URLs
+
+If `vmalert` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses via DNS A/AAAA records.
+The port from the original URL is appended to each discovered IP address.
+Each discovered IP address is used for least-loaded balancing of write requests.
+
+DNS URLs are supported in the following places:
+
+* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
+  `192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428` is automatically resolved into
+  `-remoteWrite.url=http://192.168.1.15:8428`.
+
+DNS URLs are useful when client-side HTTP load balancing is needed. A good example
+is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
+which returns multiple IP addresses for a single hostname.
+
+### DNS URLs and HTTPS
+
+When a `dns+` URL uses the `https` scheme, `vmalert` connects to the discovered
+IP addresses directly. No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication)
+is sent in the TLS handshake, and the server certificate is verified against the IP address,
+which fails unless the certificate contains the corresponding
+[IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
+
+To use `dns+` URLs with HTTPS, pass the original hostname via the corresponding
+`tlsServerName` command-line flag - `-datasource.tlsServerName`, `-remoteRead.tlsServerName`
+or `-remoteWrite.tlsServerName`. It is used both as SNI and as the name the server
+certificate is verified against:
+
+```sh
+-datasource.url=https://dns+victoria-metrics:8428
+-datasource.tlsServerName=victoria-metrics
+```
+
+Alternatively, issue server certificates with IP SAN entries for every backend IP address.
+Avoid `tlsInsecureSkipVerify` flags for working around this, since they disable
+server certificate verification completely.
+
+## SRV URLs
+
+If `vmalert` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
+record into TCP address with hostname and TCP port, and then use the resulting URL when it needs to connect to it.
+
+SRV URLs are supported in the following places:
+
+* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
+  `victoria-metrics-host:8085`, then `-remoteWrite.url=http://srv+victoria-metrics:8428` is automatically resolved into
+  `-remoteWrite.url=http://victoria-metrics-host:8085`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmalert`
+   performs per request round-robin load-balancing.
+
+SRV URLs are useful when HTTP services run on different TCP ports or when their TCP ports can change over time (for instance, after a restart).
+
+
 ## Contributing
 
 `vmalert` is mostly designed and built by VictoriaMetrics community.