lib/promscrape: ignore proxies for Unix socket targets

Unix socket scrape targets inherit `http.DefaultTransport.Proxy` through
`httputil.NewTransport`. When proxy environment variables are
configured, HTTP requests over Unix sockets are altered as proxy
requests, while HTTPS scrapes send `CONNECT` to the Unix socket and fail
the TLS handshake.

Unix sockets are explicit local transport endpoints and cannot be
reached through HTTP proxies. Clear the transport proxy for targets
configured with `__unix_socket__`, matching the existing rejection of
`proxy_url` for these targets.
The bug was introduced with Unix socket scraping support in v1.148.0  at https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11193

Related PR https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11318

README.md

@@ -2,7 +2,6 @@
 
 [![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
-[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)

app/vmagent/remotewrite/obfuscate.go

@@ -0,0 +1,108 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+type obfuscateLabelsCtx struct {
+	labels []prompb.Label
+
+	// buf holds allocations for cached results below
+	buf []byte
+
+	// cacheResults maps original label values to their SHA-256 hex digests,
+	// avoiding redundant hashing of repeated values within a single batch.
+	cacheResults map[string]string
+}
+
+func (olctx *obfuscateLabelsCtx) reset() {
+	promrelabel.CleanLabels(olctx.labels)
+	olctx.labels = olctx.labels[:0]
+	clear(olctx.cacheResults)
+	olctx.buf = olctx.buf[:0]
+}
+
+var obfuscateLabelsCtxPool = &sync.Pool{
+	New: func() any {
+		return &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+	},
+}
+
+func getObfuscateLabelsCtx() *obfuscateLabelsCtx {
+	return obfuscateLabelsCtxPool.Get().(*obfuscateLabelsCtx)
+}
+
+func putObfuscateLabelsCtx(ctx *obfuscateLabelsCtx) {
+	ctx.reset()
+	obfuscateLabelsCtxPool.Put(ctx)
+}
+
+func (olctx *obfuscateLabelsCtx) obfuscate(tss []prompb.TimeSeries, obfuscateLabels []string) []prompb.TimeSeries {
+	if len(obfuscateLabels) == 0 || len(tss) == 0 {
+		return tss
+	}
+	labels := olctx.labels
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		found := false
+		for _, labelName := range obfuscateLabels {
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], labelName)
+			if tmp == nil {
+				continue
+			}
+			found = true
+			if obfuscatedValue, ok := olctx.cacheResults[tmp.Value]; ok {
+				// fast path: the obfuscated result was calculated before
+				tmp.Value = obfuscatedValue
+				continue
+			}
+
+			digest := sha256.Sum256(bytesutil.ToUnsafeBytes(tmp.Value))
+			buf := olctx.buf
+			bufLen := len(buf)
+			buf = hex.AppendEncode(buf, digest[:])
+			obfuscatedValue := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			olctx.buf = buf
+			olctx.cacheResults[tmp.Value] = obfuscatedValue
+			tmp.Value = obfuscatedValue
+		}
+		if found {
+			ts.Labels = labels[labelsLen:]
+		} else {
+			labels = labels[:labelsLen]
+		}
+	}
+	olctx.labels = labels
+	return tss
+}
+
+func (rwctx *remoteWriteCtx) initObfuscateLabels() {
+	if len(*obfuscateLabels) == 0 {
+		return
+	}
+	idx := rwctx.idx
+	rwObfuscateLabels := obfuscateLabels.GetOptionalArg(idx)
+	rwObfuscateLabelsList := strings.Split(rwObfuscateLabels, "^^")
+
+	for _, label := range rwObfuscateLabelsList {
+		if label == "" {
+			continue
+		}
+		if !slices.Contains(rwctx.obfuscateLabels, label) {
+			rwctx.obfuscateLabels = append(rwctx.obfuscateLabels, label)
+		}
+	}
+}

app/vmagent/remotewrite/obfuscate_test.go

@@ -0,0 +1,186 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func TestRemoteWriteObfuscateLabels(t *testing.T) {
+	f := func(obfuscateLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
+		t.Helper()
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		defer metrics.UnregisterAllMetrics()
+		originValue := *obfuscateLabels
+		defer func() {
+			*obfuscateLabels = originValue
+		}()
+		*obfuscateLabels = []string{obfuscateLabelList}
+		rwctx.initObfuscateLabels()
+
+		outputTss := olctx.obfuscate(inputTss, rwctx.obfuscateLabels)
+
+		if !reflect.DeepEqual(expectedTss, outputTss) {
+			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
+		}
+	}
+
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+
+	// 1. obfuscation is not set.
+	f("",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 1. obfuscation is set for another rwctx.
+	f(",ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 2. obfuscate the value of "ip" label
+	f("ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 3. obfuscate the values of "ip" and "instance"
+	f("ip^^instance",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+					{Name: "instance", Value: "1234"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+					{Name: "instance", Value: sha256Result("1234")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "job", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+
+	// 4. duplicate label names in config must produce single SHA-256, not double
+	f("ip^^ip",
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: "123"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+		[]prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "ip", Value: sha256Result("123")},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: 0},
+				},
+			},
+		},
+	)
+}

app/vmagent/remotewrite/obfuscate_timing_test.go

@@ -0,0 +1,94 @@
+package remotewrite
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func BenchmarkRemoteWriteObfuscateLabels(b *testing.B) {
+	originValue := *obfuscateLabels
+	defer func() {
+		*obfuscateLabels = originValue
+	}()
+	*obfuscateLabels = []string{"ip^^instance"}
+	sha256Result := func(str string) string {
+		sha256Result := sha256.Sum256([]byte(str))
+		return hex.EncodeToString(sha256Result[:])
+	}
+	expected := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("123")},
+				{Name: "instance", Value: sha256Result("12345")},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: sha256Result("1236")},
+				{Name: "instance", Value: sha256Result("some-long-instante-string")},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	defer metrics.UnregisterAllMetrics()
+
+	inputTss := []prompb.TimeSeries{
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "123"},
+				{Name: "instance", Value: "12345"},
+				{Name: "__name__", Value: "http_requests_total"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+		{
+			Labels: []prompb.Label{
+				{Name: "ip", Value: "1236"},
+				{Name: "instance", Value: "some-long-instante-string"},
+				{Name: "__name__", Value: "concurrent_requests"},
+			},
+			Samples: []prompb.Sample{
+				{Value: 1, Timestamp: 0},
+			},
+		},
+	}
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	b.RunParallel(func(pb *testing.PB) {
+		rwctx := &remoteWriteCtx{
+			idx: 0,
+		}
+		olctx := &obfuscateLabelsCtx{
+			cacheResults: make(map[string]string),
+		}
+		rwctx.initObfuscateLabels()
+		var localTss []prompb.TimeSeries
+		for pb.Next() {
+			// always make a shallow copy because obfuscate changes input
+			localTss = localTss[:0]
+			localTss = append(localTss, inputTss...)
+			olctx.reset()
+			outputTss := olctx.obfuscate(localTss, rwctx.obfuscateLabels)
+			if !reflect.DeepEqual(expected, outputTss) {
+				b.Fatalf("unexpected output: got: \n%v\n want: \n%v\n", outputTss, expected)
+			}
+		}
+	})
+
+}

app/vmagent/remotewrite/remotewrite.go

@@ -107,7 +107,12 @@ var (
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
 
 	enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
+		"Can be combined with -remoteWrite.obfuscateLabels to hide sensitive label values in the forwarded metrics. "+
 		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
+	obfuscateLabels = flagutil.NewArrayString("remoteWrite.obfuscateLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
+		"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
+		"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
+		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
 )
 
 var (
@@ -881,6 +886,8 @@ type remoteWriteCtx struct {
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
 
+	obfuscateLabels []string
+
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
 	mdxRowsPreserved       *metrics.Counter
@@ -995,6 +1002,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 		rowsDroppedOnPushFailure:     metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
 	rwctx.initStreamAggrConfig()
+	rwctx.initObfuscateLabels()
 
 	if enableMdx.GetOptionalArg(argIdx) {
 		mdxFilter := mdx.NewFilter()
@@ -1198,24 +1206,41 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
 func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompb.TimeSeries
+	var olctx *obfuscateLabelsCtx
 	defer func() {
-		if rctx == nil {
-			return
+		if v != nil {
+			*v = prompb.ResetTimeSeries(tss)
+			tssPool.Put(v)
+		}
+		if rctx != nil {
+			putRelabelCtx(rctx)
+		}
+		if olctx != nil {
+			putObfuscateLabelsCtx(olctx)
 		}
-		*v = prompb.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
 	}()
 
+	copyTimeSeriesIfNeeded := func() {
+		if v == nil {
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
+	}
+
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		copyTimeSeriesIfNeeded()
 		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
 
+	if len(rwctx.obfuscateLabels) != 0 {
+		copyTimeSeriesIfNeeded()
+		olctx = getObfuscateLabelsCtx()
+		tss = olctx.obfuscate(tss, rwctx.obfuscateLabels)
+	}
+
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 

app/vmalert/datasource/client.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -267,7 +268,11 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
+		err = &httpserver.ErrorWithStatusCode{
+			StatusCode: resp.StatusCode,
+			Err:        fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body),
+		}
+		return nil, err
 	}
 	return resp, nil
 }

app/vmalert/notifier/alert.go

@@ -31,6 +31,8 @@ type Alert struct {
 	State AlertState
 	// Expr contains expression that was executed to generate the Alert
 	Expr string
+	// Interval contains the evaluation interval of the Alert's group
+	Interval time.Duration
 	// ActiveAt defines the moment of time when Alert has become active
 	ActiveAt time.Time
 	// Start defines the moment of time when Alert has become firing
@@ -84,6 +86,7 @@ type AlertTplData struct {
 	Labels    map[string]string
 	Value     float64
 	Expr      string
+	Interval  time.Duration
 	AlertID   uint64
 	GroupID   uint64
 	ActiveAt  time.Time
@@ -96,6 +99,7 @@ var tplHeaders = []string{
 	"{{ $type := .Type }}",
 	"{{ $labels := .Labels }}",
 	"{{ $expr := .Expr }}",
+	"{{ $interval := .Interval }}",
 	"{{ $externalLabels := .ExternalLabels }}",
 	"{{ $externalURL := .ExternalURL }}",
 	"{{ $alertID := .AlertID }}",
@@ -115,6 +119,7 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
 		Type:     a.Type,
 		Labels:   labels,
 		Expr:     a.Expr,
+		Interval: a.Interval,
 		AlertID:  a.ID,
 		GroupID:  a.GroupID,
 		ActiveAt: a.ActiveAt,

app/vmalert/notifier/alert_test.go

@@ -129,6 +129,17 @@ func TestAlertExecTemplate(t *testing.T) {
 		"exprEscapedHTML":   "vm_rows{"label"="bar"}<0",
 	})
 
+	// interval-template
+	f(&Alert{
+		Interval: 10 * time.Second,
+	}, map[string]string{
+		"interval":         "{{ .Interval }}",
+		"intervalVariable": "{{ $interval }}",
+	}, map[string]string{
+		"interval":         "10s",
+		"intervalVariable": "10s",
+	})
+
 	// query
 	f(&Alert{
 		Expr: `vm_rows{"label"="bar"}>0`,

app/vmalert/replay.go

@@ -25,11 +25,12 @@ var (
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
 		"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
-		"Defines how many retries to make before giving up on rule if request for it returns an error.")
+		"Defines how many retries to make before giving up on rule if request for it returns a retriable error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
 		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
+	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
@@ -73,7 +74,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
 
 	for _, cfg := range groupsCfg {
 		ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
-		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency)
+		totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency, *continueWithExecutionErr)
 	}
 	logger.Infof("replay evaluation finished, generated %d samples", totalRows)
 	if err := rw.Close(); err != nil {

app/vmalert/replay_test.go

@@ -8,8 +8,10 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
 type fakeReplayQuerier struct {
@@ -32,6 +34,14 @@ func (fc *fakeRWClient) Close() error {
 }
 
 func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
+	_, err = metricsql.Parse(q)
+	if err != nil {
+		return res, &httpserver.ErrorWithStatusCode{
+			StatusCode: 422,
+			Err:        err,
+		}
+	}
+
 	key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
 	dps, ok := fr.registry[q]
 	if !ok {
@@ -275,4 +285,28 @@ func TestReplay(t *testing.T) {
 			},
 		},
 	}, 10)
+
+	// rule with wrong expression won't break the other rule with continueWithExecutionErr
+	continueWithExecutionErrOld := *continueWithExecutionErr
+	defer func() {
+		*continueWithExecutionErr = continueWithExecutionErrOld
+	}()
+	*continueWithExecutionErr = true
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 1, time.Millisecond, []config.Group{
+		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+		{Rules: []config.Rule{{Record: "bar", Expr: "up ++"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string][]datasource.Metric{
+			"sum(up)": {
+				"12:00:00+12:01:00": {
+					{
+						Timestamps: []int64{1, 2},
+						Values:     []float64{1, 2},
+					},
+				},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	}, 2)
 }

app/vmalert/rule/alerting.go

@@ -530,6 +530,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				ar.logDebugf(ts, a, "INACTIVE => PENDING")
 			}
 			a.Value = m.Values[0]
+			a.Interval = ar.EvalInterval
 			a.Annotations = annotations
 			a.KeepFiringSince = time.Time{}
 			continue
@@ -612,6 +613,7 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
 		Type:      ar.Type.String(),
 		Labels:    ls.origin,
 		Expr:      ar.Expr,
+		Interval:  ar.EvalInterval,
 		AlertID:   hash(ls.processed),
 		GroupID:   ar.GroupID,
 		ActiveAt:  activeAt,
@@ -673,6 +675,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
 		Name:        ar.Name,
 		Type:        ar.Type.String(),
 		Expr:        ar.Expr,
+		Interval:    ar.EvalInterval,
 		For:         ar.For,
 		ActiveAt:    start,
 		Value:       m.Values[0],

app/vmalert/rule/alerting_test.go

@@ -662,6 +662,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -682,6 +683,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "for-firing"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -703,6 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    time.Second,
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -759,6 +762,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -771,6 +775,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
 			Type:        config.NewPrometheusType().String(),
+			Interval:    3 * time.Second,
 			Labels:      map[string]string{"alertname": "multi-series", "foo": "bar"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,
@@ -1134,7 +1139,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		fq.Add(metrics...)
 		fq.SetPartialResponse(isResponsePartial)
 
-		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
+		ts := time.Unix(3600, 0)
+		if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		for hash, expAlert := range alertsExpected {
@@ -1152,12 +1158,14 @@ func TestAlertingRule_Template(t *testing.T) {
 	}
 
 	f(&AlertingRule{
-		Name: "common",
+		Name:         "common",
+		EvalInterval: time.Hour,
 		Labels: map[string]string{
 			"region": "east",
 		},
 		Annotations: map[string]string{
-			"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":   `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+			"dashboard": `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1166,7 +1174,8 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "foo"`,
+				"summary":   `common: Too high connection number for "foo"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1176,7 +1185,8 @@ func TestAlertingRule_Template(t *testing.T) {
 		},
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
 			Annotations: map[string]string{
-				"summary": `common: Too high connection number for "bar"`,
+				"summary":   `common: Too high connection number for "bar"`,
+				"dashboard": "&from=0&to=3600000",
 			},
 			Labels: map[string]string{
 				alertNameLabel: "common",
@@ -1388,7 +1398,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
 		"pod":           "vmalert-0",
-		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1398,7 +1408,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)

app/vmalert/rule/group.go

@@ -548,7 +548,7 @@ func (g *Group) infof(format string, args ...any) {
 }
 
 // Replay performs group replay
-func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int) int {
+func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	var total int
 	step := g.Interval * time.Duration(maxDataPoint)
 	ri := rangeIterator{start: start, end: end, step: step}
@@ -576,7 +576,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 			if !disableProgressBar {
 				bar = pb.StartNew(iterations)
 			}
-			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			if bar != nil {
 				bar.Finish()
 			}
@@ -598,7 +598,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 		rule := g.Rules[i]
 		sem <- struct{}{}
 		wg.Go(func() {
-			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
 			<-sem
 		})
 	}
@@ -618,7 +618,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 	return total
 }
 
-func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
+func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
 	fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
 	// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
 	// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
@@ -633,7 +633,7 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 		start := ri.s
 		end := ri.e
 		wg.Go(func() {
-			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
+			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts, continueWithExecutionErr)
 			if err != nil {
 				logger.Fatalf("rule %q: %s", r, err)
 			}

app/vmalert/rule/rule.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -118,7 +120,7 @@ func (s *ruleState) add(e StateEntry) {
 	s.entries[s.cur] = e
 }
 
-func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
+func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int, continueWithExecutionErr bool) (int, error) {
 	var err error
 	var tss []prompb.TimeSeries
 	for i := range replayRuleRetryAttempts {
@@ -126,6 +128,21 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
 		if err == nil {
 			break
 		}
+		// retry request if possible to tolerate temporary network or datasource unavailability issues
+		var esc *httpserver.ErrorWithStatusCode
+		if errors.As(err, &esc) {
+			statusCode := esc.StatusCode
+			// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
+			// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
+			if statusCode == http.StatusUnprocessableEntity {
+				if continueWithExecutionErr {
+					logger.Errorf("rule %q: %s", r, err)
+					return 0, nil
+				} else {
+					return 0, err
+				}
+			}
+		}
 		logger.Errorf("attempt %d to execute rule %q failed: %s", i+1, r, err)
 		time.Sleep(time.Second)
 	}

app/vmctl/remoteread/remoteread.go

@@ -8,12 +8,14 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -234,9 +236,29 @@ func processResponse(body io.ReadCloser, callback StreamCallback) error {
 	// shouldn't be accounted as an error.
 	for _, res := range readResp.Results {
 		for _, ts := range res.Timeseries {
-			vmTs := convertSamples(ts.Samples, ts.Labels)
-			if err := callback(vmTs); err != nil {
-				return err
+			// A series contains either float samples or native histogram samples.
+			// Both fields are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain samples of both types.
+			if len(ts.Samples) > 0 {
+				vmTs := convertSamples(ts.Samples, ts.Labels)
+				if err := callback(vmTs); err != nil {
+					return err
+				}
+			}
+			if len(ts.Histograms) > 0 {
+				hSamples := make([]histogramSample, 0, len(ts.Histograms))
+				for _, h := range ts.Histograms {
+					hSamples = append(hSamples, histogramSample{
+						timestamp: h.Timestamp,
+						fh:        h.ToFloatHistogram(),
+					})
+				}
+				for _, vmTs := range convertHistograms(hSamples, ts.Labels) {
+					if err := callback(vmTs); err != nil {
+						return err
+					}
+				}
 			}
 		}
 	}
@@ -263,17 +285,45 @@ func processStreamResponse(body io.ReadCloser, callback StreamCallback) error {
 
 		for _, series := range res.ChunkedSeries {
 			samples := make([]prompb.Sample, 0)
+			var hSamples []histogramSample
 			for _, chunk := range series.Chunks {
-				s, err := parseSamples(chunk.Data)
-				if err != nil {
-					return err
+				switch chunk.Type {
+				case prompb.Chunk_XOR, prompb.Chunk_UNKNOWN:
+					// In proto3 the `type` field may be left unset (UNKNOWN) for XOR chunks.
+					// Prometheus remote.proto: "REQUIREMENT: when using proto3, this field
+					// MUST be set when using anything else than XOR". Senders before native
+					// histograms support (Prometheus < 2.40) do not set this field at all,
+					// so UNKNOWN chunks must be parsed as XOR ones.
+					s, err := parseSamples(chunk.Data)
+					if err != nil {
+						return err
+					}
+					samples = append(samples, s...)
+				case prompb.Chunk_HISTOGRAM, prompb.Chunk_FLOAT_HISTOGRAM:
+					hs, err := parseHistograms(chunk.Type, chunk.Data)
+					if err != nil {
+						return err
+					}
+					hSamples = append(hSamples, hs...)
+				default:
+					return fmt.Errorf("unsupported chunk encoding %q", chunk.Type)
 				}
-				samples = append(samples, s...)
 			}
 
-			ts := convertSamples(samples, series.Labels)
-			if err := callback(ts); err != nil {
-				return err
+			// A series contains either XOR chunks or native histogram chunks.
+			// Both are processed independently, since a series may switch
+			// from float to native histogram representation at some point in time,
+			// so the requested time range may contain chunks of both types.
+			if len(samples) > 0 {
+				ts := convertSamples(samples, series.Labels)
+				if err := callback(ts); err != nil {
+					return err
+				}
+			}
+			for _, ts := range convertHistograms(hSamples, series.Labels) {
+				if err := callback(ts); err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -312,6 +362,151 @@ func parseSamples(chunk []byte) ([]prompb.Sample, error) {
 	return samples, it.Err()
 }
 
+// histogramSample represents a single native histogram sample.
+type histogramSample struct {
+	timestamp int64
+	fh        *histogram.FloatHistogram
+}
+
+func parseHistograms(encoding prompb.Chunk_Encoding, chunk []byte) ([]histogramSample, error) {
+	var enc chunkenc.Encoding
+	switch encoding {
+	case prompb.Chunk_HISTOGRAM:
+		enc = chunkenc.EncHistogram
+	case prompb.Chunk_FLOAT_HISTOGRAM:
+		enc = chunkenc.EncFloatHistogram
+	default:
+		return nil, fmt.Errorf("unsupported histogram chunk encoding %q", encoding)
+	}
+	c, err := chunkenc.FromData(enc, chunk)
+	if err != nil {
+		return nil, fmt.Errorf("error read chunk: %w", err)
+	}
+
+	var hSamples []histogramSample
+	it := c.Iterator(nil)
+	for {
+		typ := it.Next()
+		if typ == chunkenc.ValNone {
+			break
+		}
+		switch typ {
+		case chunkenc.ValHistogram:
+			ts, h := it.AtHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        h.ToFloat(nil),
+			})
+		case chunkenc.ValFloatHistogram:
+			ts, fh := it.AtFloatHistogram(nil)
+			hSamples = append(hSamples, histogramSample{
+				timestamp: ts,
+				fh:        fh,
+			})
+		default:
+			// Skip unsupported values
+			continue
+		}
+	}
+	if err := it.Err(); err != nil {
+		return nil, fmt.Errorf("error iterate over chunks: %w", err)
+	}
+
+	return hSamples, nil
+}
+
+// convertHistograms converts native histogram samples into VictoriaMetrics histogram
+// time series in the same way as VictoriaMetrics converts native histograms
+// received via Prometheus remote write protocol: every native histogram sample
+// is converted into `<name>_count` and `<name>_sum` series plus a set of
+// `<name>_bucket` series with `vmrange` labels containing non-cumulative bucket counts.
+// The only difference is that for native histograms with custom buckets (NHCB)
+// bucket bounds are taken from the custom values, while the remote write protocol
+// parser ignores custom values and estimates the bounds with the exponential formula.
+// See https://prometheus.io/docs/specs/native_histograms/#data-model
+func convertHistograms(hSamples []histogramSample, labels []prompb.Label) []*vm.TimeSeries {
+	if len(hSamples) == 0 {
+		return nil
+	}
+
+	labelPairs := make([]vm.LabelPair, 0, len(labels))
+	nameValue := ""
+	for _, label := range labels {
+		if label.Name == "__name__" {
+			nameValue = label.Value
+			continue
+		}
+		labelPairs = append(labelPairs, vm.LabelPair{Name: label.Name, Value: label.Value})
+	}
+	// the metric has no name, skip it in the same way as VictoriaMetrics does
+	// when it receives a native histogram without the metric name via remote write protocol.
+	if nameValue == "" {
+		return nil
+	}
+
+	countSeries := &vm.TimeSeries{
+		Name:       nameValue + "_count",
+		LabelPairs: labelPairs,
+	}
+	sumSeries := &vm.TimeSeries{
+		Name:       nameValue + "_sum",
+		LabelPairs: labelPairs,
+	}
+	bucketSeries := make(map[string]*vm.TimeSeries)
+	// vmranges preserves the order of bucketSeries creation
+	// in order to get deterministic results.
+	var vmranges []string
+
+	for _, hs := range hSamples {
+		fh := hs.fh
+		countSeries.Timestamps = append(countSeries.Timestamps, hs.timestamp)
+		countSeries.Values = append(countSeries.Values, fh.Count)
+		sumSeries.Timestamps = append(sumSeries.Timestamps, hs.timestamp)
+		sumSeries.Values = append(sumSeries.Values, fh.Sum)
+
+		it := fh.AllBucketIterator()
+		for it.Next() {
+			b := it.At()
+			if b.Count <= 0 {
+				continue
+			}
+			vmrange := formatVmrange(b.Lower, b.Upper)
+			s := bucketSeries[vmrange]
+			if s == nil {
+				bucketLabelPairs := make([]vm.LabelPair, len(labelPairs), len(labelPairs)+1)
+				copy(bucketLabelPairs, labelPairs)
+				bucketLabelPairs = append(bucketLabelPairs, vm.LabelPair{Name: "vmrange", Value: vmrange})
+				s = &vm.TimeSeries{
+					Name:       nameValue + "_bucket",
+					LabelPairs: bucketLabelPairs,
+				}
+				bucketSeries[vmrange] = s
+				vmranges = append(vmranges, vmrange)
+			}
+			s.Timestamps = append(s.Timestamps, hs.timestamp)
+			s.Values = append(s.Values, b.Count)
+		}
+	}
+
+	tss := make([]*vm.TimeSeries, 0, 2+len(vmranges))
+	tss = append(tss, countSeries, sumSeries)
+	for _, vmrange := range vmranges {
+		tss = append(tss, bucketSeries[vmrange])
+	}
+	return tss
+}
+
+// formatVmrange formats the given bucket bounds into `vmrange` label value
+// in the same way as VictoriaMetrics does for native histograms
+// received via Prometheus remote write protocol.
+func formatVmrange(lower, upper float64) string {
+	b := make([]byte, 0, 24)
+	b = strconv.AppendFloat(b, lower, 'e', 3, 64)
+	b = append(b, "..."...)
+	b = strconv.AppendFloat(b, upper, 'e', 3, 64)
+	return string(b)
+}
+
 type keyValue struct {
 	key   string
 	value string

app/vmctl/remoteread/remoteread_test.go

@@ -0,0 +1,334 @@
+package remoteread
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage/remote"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+)
+
+func testHistogram(mul int64) *histogram.Histogram {
+	return &histogram.Histogram{
+		Schema:          0,
+		Count:           uint64(10 * mul),
+		Sum:             25.5 * float64(mul),
+		ZeroThreshold:   0.001,
+		ZeroCount:       uint64(2 * mul),
+		PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}},
+		PositiveBuckets: []int64{1 * mul, 2 * mul},
+		NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+		NegativeBuckets: []int64{4 * mul},
+	}
+}
+
+func TestConvertHistograms(t *testing.T) {
+	f := func(hSamples []histogramSample, labels []prompb.Label, expected []*vm.TimeSeries) {
+		t.Helper()
+
+		tss := convertHistograms(hSamples, labels)
+		if !reflect.DeepEqual(tss, expected) {
+			t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, expected)
+		}
+	}
+
+	// series without samples
+	f(nil, []prompb.Label{{Name: "__name__", Value: "foo"}}, nil)
+
+	// series without the metric name must be skipped
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+	}, []prompb.Label{{Name: "job", Value: "bar"}}, nil)
+
+	// native histogram must be converted to _count, _sum and _bucket series
+	// in the same way as VictoriaMetrics does for Prometheus remote write protocol
+	labels := []prompb.Label{
+		{Name: "__name__", Value: "request_duration_seconds"},
+		{Name: "job", Value: "bar"},
+	}
+	jobLabel := []vm.LabelPair{{Name: "job", Value: "bar"}}
+	bucketLabels := func(vmrange string) []vm.LabelPair {
+		return []vm.LabelPair{
+			{Name: "job", Value: "bar"},
+			{Name: "vmrange", Value: vmrange},
+		}
+	}
+	f([]histogramSample{
+		{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
+		{timestamp: 2000, fh: testHistogram(2).ToFloat(nil)},
+	}, labels, []*vm.TimeSeries{
+		{
+			Name:       "request_duration_seconds_count",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{10, 20},
+		},
+		{
+			Name:       "request_duration_seconds_sum",
+			LabelPairs: jobLabel,
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{25.5, 51},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e+00...-5.000e-01"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{4, 8},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("-1.000e-03...1.000e-03"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{2, 4},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("5.000e-01...1.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{1, 2},
+		},
+		{
+			Name:       "request_duration_seconds_bucket",
+			LabelPairs: bucketLabels("1.000e+00...2.000e+00"),
+			Timestamps: []int64{1000, 2000},
+			Values:     []float64{3, 6},
+		},
+	})
+}
+
+func TestParseHistograms(t *testing.T) {
+	c := chunkenc.NewHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+	if _, _, _, err := app.AppendHistogram(nil, 0, 2000, testHistogram(2), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse histogram chunk: %s", err)
+	}
+	if len(hSamples) != 2 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 2", len(hSamples))
+	}
+	for i, expected := range []struct {
+		timestamp int64
+		count     float64
+		sum       float64
+	}{
+		{timestamp: 1000, count: 10, sum: 25.5},
+		{timestamp: 2000, count: 20, sum: 51},
+	} {
+		if hSamples[i].timestamp != expected.timestamp {
+			t.Fatalf("unexpected timestamp; got %d; want %d", hSamples[i].timestamp, expected.timestamp)
+		}
+		if hSamples[i].fh.Count != expected.count {
+			t.Fatalf("unexpected count; got %f; want %f", hSamples[i].fh.Count, expected.count)
+		}
+		if hSamples[i].fh.Sum != expected.sum {
+			t.Fatalf("unexpected sum; got %f; want %f", hSamples[i].fh.Sum, expected.sum)
+		}
+	}
+
+	// unsupported chunk encoding must return error
+	if _, err := parseHistograms(prompb.Chunk_XOR, c.Bytes()); err == nil {
+		t.Fatalf("expecting non-nil error for unsupported chunk encoding")
+	}
+}
+
+func TestProcessResponse(t *testing.T) {
+	readResp := &prompb.ReadResponse{
+		Results: []*prompb.QueryResult{
+			{
+				Timeseries: []*prompb.TimeSeries{
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "cpu_usage"},
+							{Name: "job", Value: "bar"},
+						},
+						Samples: []prompb.Sample{
+							{Timestamp: 1000, Value: 1.5},
+						},
+					},
+					{
+						Labels: []prompb.Label{
+							{Name: "__name__", Value: "request_duration_seconds"},
+							{Name: "job", Value: "bar"},
+						},
+						Histograms: []prompb.Histogram{
+							prompb.FromIntHistogram(1000, testHistogram(1)),
+						},
+					},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(readResp)
+	if err != nil {
+		t.Fatalf("cannot marshal ReadResponse: %s", err)
+	}
+	compressed := snappy.Encode(nil, data)
+
+	var tss []*vm.TimeSeries
+	err = processResponse(io.NopCloser(bytes.NewReader(compressed)), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process response: %s", err)
+	}
+
+	// 1 float series + _count + _sum + 4 buckets
+	if len(tss) != 7 {
+		t.Fatalf("unexpected number of time series; got %d; want 7", len(tss))
+	}
+	if tss[0].Name != "cpu_usage" || !reflect.DeepEqual(tss[0].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[1].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[1])
+	}
+	if tss[2].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[2].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[2])
+	}
+	for _, ts := range tss[3:] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+}
+
+type nopFlusher struct{}
+
+func (nopFlusher) Flush() {}
+
+func TestProcessStreamResponse(t *testing.T) {
+	// build a histogram chunk
+	hc := chunkenc.NewHistogramChunk()
+	hApp, err := hc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create histogram chunk appender: %s", err)
+	}
+	if _, _, _, err := hApp.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
+		t.Fatalf("cannot append histogram: %s", err)
+	}
+
+	// build a float chunk
+	xc := chunkenc.NewXORChunk()
+	xApp, err := xc.Appender()
+	if err != nil {
+		t.Fatalf("cannot create xor chunk appender: %s", err)
+	}
+	xApp.Append(0, 1000, 1.5)
+
+	res := &prompb.ChunkedReadResponse{
+		ChunkedSeries: []*prompb.ChunkedSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "request_duration_seconds"},
+					{Name: "job", Value: "bar"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_HISTOGRAM, Data: hc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "cpu_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					{Type: prompb.Chunk_XOR, Data: xc.Bytes()},
+				},
+			},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "memory_usage"},
+				},
+				Chunks: []prompb.Chunk{
+					// the `type` field may be unset for XOR chunks,
+					// such chunks must be parsed as XOR ones
+					{Type: prompb.Chunk_UNKNOWN, Data: xc.Bytes()},
+				},
+			},
+		},
+	}
+	data, err := proto.Marshal(res)
+	if err != nil {
+		t.Fatalf("cannot marshal ChunkedReadResponse: %s", err)
+	}
+	var buf bytes.Buffer
+	cw := remote.NewChunkedWriter(&buf, nopFlusher{})
+	if _, err := cw.Write(data); err != nil {
+		t.Fatalf("cannot write chunked response: %s", err)
+	}
+
+	var tss []*vm.TimeSeries
+	err = processStreamResponse(io.NopCloser(&buf), func(ts *vm.TimeSeries) error {
+		tss = append(tss, ts)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("cannot process stream response: %s", err)
+	}
+
+	// _count + _sum + 4 buckets + 1 float series + 1 float series from UNKNOWN chunk
+	if len(tss) != 8 {
+		t.Fatalf("unexpected number of time series; got %d; want 8", len(tss))
+	}
+	if tss[0].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[0].Values, []float64{10}) {
+		t.Fatalf("unexpected _count series: %v", tss[0])
+	}
+	if tss[1].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[1].Values, []float64{25.5}) {
+		t.Fatalf("unexpected _sum series: %v", tss[1])
+	}
+	for _, ts := range tss[2:6] {
+		if ts.Name != "request_duration_seconds_bucket" {
+			t.Fatalf("unexpected bucket series name %q", ts.Name)
+		}
+	}
+	if tss[6].Name != "cpu_usage" || !reflect.DeepEqual(tss[6].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series: %v", tss[6])
+	}
+	if tss[7].Name != "memory_usage" || !reflect.DeepEqual(tss[7].Values, []float64{1.5}) {
+		t.Fatalf("unexpected float series from UNKNOWN chunk: %v", tss[7])
+	}
+}
+
+func TestParseFloatHistograms(t *testing.T) {
+	c := chunkenc.NewFloatHistogramChunk()
+	app, err := c.Appender()
+	if err != nil {
+		t.Fatalf("cannot create chunk appender: %s", err)
+	}
+	fh := testHistogram(1).ToFloat(nil)
+	if _, _, _, err := app.AppendFloatHistogram(nil, 0, 1000, fh, true); err != nil {
+		t.Fatalf("cannot append float histogram: %s", err)
+	}
+
+	hSamples, err := parseHistograms(prompb.Chunk_FLOAT_HISTOGRAM, c.Bytes())
+	if err != nil {
+		t.Fatalf("cannot parse float histogram chunk: %s", err)
+	}
+	if len(hSamples) != 1 {
+		t.Fatalf("unexpected number of histogram samples; got %d; want 1", len(hSamples))
+	}
+	if hSamples[0].timestamp != 1000 {
+		t.Fatalf("unexpected timestamp; got %d; want 1000", hSamples[0].timestamp)
+	}
+	if hSamples[0].fh.Count != 10 {
+		t.Fatalf("unexpected count; got %f; want 10", hSamples[0].fh.Count)
+	}
+}

app/vmselect/main.go

@@ -369,6 +369,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}
@@ -388,6 +392,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
+		if r.Method != "POST" {
+			http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
+			return true
+		}
 		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
 			return true
 		}

apptest/tests/prometheus_mock_storage.go

@@ -59,7 +59,7 @@ func (ms *PrometheusMockStorage) Read(_ context.Context, query *prompb.Query, so
 		}
 
 		if !notMatch {
-			q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples})
+			q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples, Histograms: s.Histograms})
 		}
 	}
 

apptest/tests/remote_read_server.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
@@ -86,10 +87,17 @@ func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
 						samples = append(samples, sample)
 					}
 				}
+				var histograms []prompb.Histogram
+				for _, h := range s.Histograms {
+					if h.Timestamp >= startTs && h.Timestamp < endTs {
+						histograms = append(histograms, h)
+					}
+				}
 				var series prompb.TimeSeries
-				if len(samples) > 0 {
+				if len(samples) > 0 || len(histograms) > 0 {
 					series.Labels = s.Labels
 					series.Samples = samples
+					series.Histograms = histograms
 				}
 				ts[i] = &series
 			}
@@ -317,6 +325,37 @@ func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64)
 	return samples
 }
 
+// GenerateRemoteReadHistogramSeries generates a remote read series
+// with native histogram samples within the given time range.
+func GenerateRemoteReadHistogramSeries(start, end, numOfSamples int64) []*prompb.TimeSeries {
+	timeSeries := &prompb.TimeSeries{
+		Labels: []prompb.Label{
+			{Name: labels.MetricName, Value: "vm_histogram_metric"},
+			{Name: "job", Value: "0"},
+		},
+	}
+
+	delta := (end - start) / numOfSamples
+	mul := int64(0)
+	for t := start; t != end; t += delta {
+		mul++
+		h := &histogram.Histogram{
+			Schema:          0,
+			Count:           uint64(10 * mul),
+			Sum:             25.5 * float64(mul),
+			ZeroThreshold:   0.001,
+			ZeroCount:       uint64(2 * mul),
+			PositiveSpans:   []histogram.Span{{Offset: 0, Length: 2}},
+			PositiveBuckets: []int64{mul, 2 * mul},
+			NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+			NegativeBuckets: []int64{4 * mul},
+		}
+		timeSeries.Histograms = append(timeSeries.Histograms, prompb.FromIntHistogram(t*1000, h))
+	}
+
+	return []*prompb.TimeSeries{timeSeries}
+}
+
 func labelsToLabelsProto(ls labels.Labels) []prompb.Label {
 	result := make([]prompb.Label, 0, ls.Len())
 	ls.Range(func(l labels.Label) {

apptest/tests/vmctl_remote_read_mogration_test.go

@@ -75,6 +75,118 @@ func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
 	testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
 }
 
+func TestSingleVmctlRemoteReadNativeHistograms(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadServer, vmctlFlags)
+}
+
+func TestSingleVmctlRemoteReadStreamNativeHistograms(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--remote-read-use-stream=true`,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadStreamServer, vmctlFlags)
+}
+
+// testRemoteReadNativeHistograms verifies that native histograms are migrated
+// as _count, _sum and _bucket series with vmrange labels in the same way
+// as VictoriaMetrics converts native histograms received via Prometheus remote write protocol.
+func testRemoteReadNativeHistograms(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T, series []*prompb.TimeSeries) *RemoteReadServer, vmctlFlags []string) {
+	t := tc.T()
+	t.Helper()
+
+	series := GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2)
+
+	rrs := newRemoteReadServer(t, series)
+	defer rrs.Close()
+
+	vmctlFlags = append(vmctlFlags, `--remote-read-src-addr=`+rrs.HTTPAddr())
+	tc.MustStartVmctl("vmctl", vmctlFlags)
+
+	sut.ForceFlush(t)
+
+	tc.Assert(&apptest.AssertOptions{
+		Retries: 300,
+		Msg:     `unexpected native histogram metrics stored on vmsingle via the prometheus protocol`,
+		Got: func() any {
+			got := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
+				Start: "2025-06-11T15:31:10Z",
+				End:   "2025-06-11T15:32:20Z",
+			})
+			got.Sort()
+			return got.Data.Result
+		},
+		Want: expectedNativeHistogramQueryResult(),
+		CmpOpts: []cmp.Option{
+			cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		},
+	})
+}
+
+// expectedNativeHistogramQueryResult returns the series expected to be stored in VictoriaMetrics
+// after migrating the series generated by GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2).
+func expectedNativeHistogramQueryResult() []*apptest.QueryResult {
+	metric := func(name, vmrange string) map[string]string {
+		m := map[string]string{
+			"__name__": name,
+			"job":      "0",
+		}
+		if vmrange != "" {
+			m["vmrange"] = vmrange
+		}
+		return m
+	}
+	samples := func(v1, v2 float64) []*apptest.Sample {
+		return []*apptest.Sample{
+			{Timestamp: 1749655870000, Value: v1},
+			{Timestamp: 1749655875000, Value: v2},
+		}
+	}
+	resp := &apptest.PrometheusAPIV1QueryResponse{
+		Data: &apptest.QueryData{
+			Result: []*apptest.QueryResult{
+				{Metric: metric("vm_histogram_metric_count", ""), Samples: samples(10, 20)},
+				{Metric: metric("vm_histogram_metric_sum", ""), Samples: samples(25.5, 51)},
+				{Metric: metric("vm_histogram_metric_bucket", "-1.000e+00...-5.000e-01"), Samples: samples(4, 8)},
+				{Metric: metric("vm_histogram_metric_bucket", "-1.000e-03...1.000e-03"), Samples: samples(2, 4)},
+				{Metric: metric("vm_histogram_metric_bucket", "5.000e-01...1.000e+00"), Samples: samples(1, 2)},
+				{Metric: metric("vm_histogram_metric_bucket", "1.000e+00...2.000e+00"), Samples: samples(3, 6)},
+			},
+		},
+	}
+	// sort in the same way as the exported result
+	resp.Sort()
+	return resp.Data.Result
+}
+
 func testRemoteReadProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
 	t := tc.T()
 	t.Helper()

docs/anomaly-detection/FAQ.md

@@ -118,27 +118,6 @@ To visualize and interact with both [self-monitoring metrics](https://docs.victo
 - {{% available_from "v1.26.0" anomaly %}} For rapid exploration of how different models, their configurations and included domain knowledge impacts the results of anomaly detection, use the built-in [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
 
-## Is vmanomaly stateful?
-By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
-
-Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
-
-## Config hot-reloading
-
-`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
-
-## Environment variables
-
-`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
-
-## Deploying vmanomaly
-
-`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
-
-## Migration
-
-For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
-
 ## Choosing the right model for vmanomaly
 
 Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect:
@@ -155,16 +134,6 @@ Please refer to [respective blogpost on anomaly types and alerting heuristics](h
 
 Still not 100% sure what to use? We are [here to help](https://docs.victoriametrics.com/anomaly-detection/#get-in-touch).
 
-## Can AI help configure vmanomaly?
-
-Yes. The available tools serve different workflows:
-
-- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
-- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
-- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
-
-Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
-
 ## Incorporating domain knowledge
 
 Anomaly detection models can significantly improve when incorporating business-specific assumptions about the data and what constitutes an anomaly. `vmanomaly` supports various [business-side configuration parameters](https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args) across all built-in models to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** and **align model behavior with business needs**, for example:
@@ -222,39 +191,15 @@ models:
     provide_series: ['anomaly_score', 'y', 'yhat', 'yhat_lower', 'yhat_upper']
 ```
 
-## Alert generation in vmanomaly
-While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
+## Can AI help configure vmanomaly?
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
+Yes. The available tools serve different workflows:
 
-Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
+- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
+- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
+- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
 
-```yaml
-groups:
-- name: VMAnomalyAlerts
-  interval: 60s
-  rules:
-  - alert: HighAnomalyScore
-    expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
-    for: 5m  # adjust to your needs based on data frequency and alerting policies
-    labels:
-      severity: warning
-      query_alias: explore
-      model_alias: default
-      scheduler_alias: periodic
-      preset: ui
-    annotations:
-      summary: High anomaly score detected.
-      description: Anomaly score exceeded threshold ({{ $value }}) for more than
-        {{ $for }} for query {{ $labels.for }}.
-```
-
-> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
-
-> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
-
-## Preventing alert fatigue
-Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
+Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
 
 ## How to backtest particular configuration on historical data?
 
@@ -417,6 +362,61 @@ groups:
       description: "Disk usage is forecasted to exceed 95% in the next 3 days for instance {{ $labels.instance }}. Forecasted value: {{ $value }}."
 ```
 
+## Alert generation in vmanomaly
+While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
+
+<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
+
+Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
+
+```yaml
+groups:
+- name: VMAnomalyAlerts
+  interval: 60s
+  rules:
+  - alert: HighAnomalyScore
+    expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
+    for: 5m  # adjust to your needs based on data frequency and alerting policies
+    labels:
+      severity: warning
+      query_alias: explore
+      model_alias: default
+      scheduler_alias: periodic
+      preset: ui
+    annotations:
+      summary: High anomaly score detected.
+      description: Anomaly score exceeded threshold ({{ $value }}) for more than
+        {{ $for }} for query {{ $labels.for }}.
+```
+
+> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
+
+> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
+
+## Preventing alert fatigue
+Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
+
+## Deploying vmanomaly
+
+`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
+
+## Environment variables
+
+`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
+
+## Is vmanomaly stateful?
+By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
+
+Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
+
+## Config hot-reloading
+
+`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
+
+## Migration
+
+For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
+
 ## Resource consumption of vmanomaly
 `vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), and the complexity of the employed [models](https://docs.victoriametrics.com/anomaly-detection/components/models/). Its resource usage is directly related to these factors, making it adaptable to various operational scales. Various optimizations are available to balance between RAM usage, processing speed, and model capacity. These options are described in the sections below.
 

docs/anomaly-detection/README.md

@@ -27,7 +27,7 @@ Key functions:
 
 The diagram below illustrates how `vmanomaly` fits into an observability setup, such as detecting anomalies in metrics collected by `node_exporter`:
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
+<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
 
 ## How does it work?
 
@@ -40,7 +40,7 @@ VictoriaMetrics Anomaly Detection **continuously re-fit and apply machine learni
   - **Confidence intervals** (`[yhat_lower, yhat_upper]`)  
   These outputs integrate seamlessly into downstream applications, making it easier to **visually inspect anomalies**, e.g. in respective [Grafana dashboards](https://docs.victoriametrics.com/anomaly-detection/presets/#grafana-dashboard).
 
-<img src="https://docs.victoriametrics.com/anomaly-detection/components/vmanomaly-components.webp" alt="node_exporter_example_diagram" style="width:80%"/>
+{{% content "components/vmanomaly-components-diagram.md" %}}
 
 ## Key benefits
 

docs/anomaly-detection/Scaling-vmanomaly.md

@@ -78,9 +78,7 @@ These [sub-configurations](#sub-configuration) can be assigned to a specific sha
 
 Additionally, a replication factor `R ≥ 1` ensures [high availability](#high-availability) by enforcing redundancy across shards.
 
-<p></p>
-
-![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
+{{% content "vmanomaly-sharding-ha-diagram.md" %}}
 
 > Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
 
@@ -130,9 +128,7 @@ Similar to other VictoriaMetrics ecosystem components, like [VMAgent](https://do
 
 When `VMANOMALY_REPLICATION_FACTOR` > 1, each [sub-config](#sub-configuration) `n` from `{0, N-1}` is assigned to exactly `R` nodes. This ensures redundancy, preventing single-node failures from causing data loss.
 
-<p></p>
-
-![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
+{{% content "vmanomaly-sharding-ha-diagram.md" %}}
 
 > Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
 

docs/anomaly-detection/components/README.md

@@ -28,7 +28,7 @@ Below, you will find an example illustrating how the components of `vmanomaly` i
 
 > [Reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [Writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) also support [multitenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy), so you can read/write from/to different locations - see `tenant_id` param description.
 
-![vmanomaly-components](vmanomaly-components.webp)
+{{% content "vmanomaly-components-diagram.md" %}}
 
 ## Example config
 

docs/anomaly-detection/components/autotune.svg

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 968 420" role="img" aria-labelledby="title description">
+  <title id="title">AutoTunedModel tuning and inference lifecycle</title>
+  <desc id="description">The tuning process tests and scores model candidates across n time-series splits until the trial count or timeout is reached. Each fold fits a candidate on training data and predicts anomalies on its validation segment. The best model is then used on inference data until the next fit call.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M0 0 10 5 0 10Z" fill="#202124"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #202124; font-size: 18px; }
+      .line { fill: none; stroke: #202124; stroke-width: 2.5; }
+      .arrow { fill: none; stroke: #202124; stroke-width: 2.5; marker-end: url(#arrow); }
+      .dash { fill: none; stroke: #202124; stroke-width: 2.5; stroke-dasharray: 8 9; }
+      .box { fill: #fff; stroke: #202124; stroke-width: 1.5; }
+    </style>
+  </defs>
+
+  <rect width="968" height="420" fill="#fff"/>
+  <rect x="112" y="10" width="242" height="83" class="box"/>
+  <text x="233" y="38" text-anchor="middle">
+    <tspan x="233" dy="0">test and score candidates</tspan>
+    <tspan x="233" dy="21">until `n_trials` or `timeout`</tspan>
+    <tspan x="233" dy="21">is reached</tspan>
+  </text>
+  <path d="M354 49 V67" class="arrow"/>
+  <text x="355" y="90" text-anchor="middle">Tuning</text>
+  <text x="355" y="112" text-anchor="middle">process</text>
+
+  <path d="M162 157 V136 H549 V157" class="line"/>
+  <path d="M355 112 V136" class="line"/>
+  <path d="M399 93 H653 V152" class="line"/>
+
+  <path d="M113 161 H75 V350 H113" class="line"/>
+  <path d="M75 236 H39" class="line"/>
+  <text x="28" y="250" transform="rotate(-90 28 250)" text-anchor="middle">n splits</text>
+
+  <text x="102" y="227">Fold 1</text>
+  <text x="102" y="255">Fold 2</text>
+  <text x="102" y="283">Fold 3</text>
+  <text x="120" y="327">...</text>
+
+  <text x="260" y="179" text-anchor="middle">Fit</text>
+  <text x="260" y="202" text-anchor="middle">candidate</text>
+  <text x="413" y="179" text-anchor="middle">Predict</text>
+  <text x="413" y="202" text-anchor="middle">anomalies</text>
+
+  <path d="M173 221 H354 V207" class="line"/>
+  <path d="M232 249 H412 V235" class="line"/>
+  <path d="M293 280 H472 V265" class="line"/>
+  <path d="M354 221 H412" class="dash"/>
+  <path d="M412 249 H474" class="dash"/>
+  <path d="M472 280 H513" class="dash"/>
+
+  <path d="M548 144 V415" class="dash" style="stroke-width:1.5;stroke-dasharray:4 6"/>
+  <rect x="577" y="152" width="136" height="61" rx="12" class="box"/>
+  <text x="645" y="187" text-anchor="middle">best model</text>
+  <path d="M653 213 V348" class="arrow"/>
+  <text x="815" y="168" text-anchor="middle">
+    <tspan x="815" dy="0">used to predict</tspan>
+    <tspan x="815" dy="21">on inference data</tspan>
+    <tspan x="815" dy="21">until the next `fit` call</tspan>
+  </text>
+
+  <path d="M40 350 H895" class="arrow"/>
+  <text x="455" y="402" text-anchor="middle">Training data</text>
+  <text x="623" y="402" text-anchor="middle">Inference data</text>
+  <text x="856" y="402">Time axis, t</text>
+</svg>

docs/anomaly-detection/components/model-lifecycle-multivariate.svg

@@ -0,0 +1,139 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
+  <title id="title">Multivariate models lifecycle</title>
+  <desc id="description">MetricsQL queries retrieve an aligned set of series from VictoriaMetrics. Reader data fits one shared multivariate model. The exact same series set produces one anomaly score series with the intersected label set; inference is skipped when the fit and inference series sets differ. Writer stores produced scores in VictoriaMetrics.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
+      .title { font-size: 50px; font-weight: 400; }
+      .node { font-size: 34px; font-weight: 400; }
+      .label { font-size: 29px; }
+      .blue-label { font-size: 26px; }
+      .small { font-size: 25px; }
+      .box { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .group { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
+      .line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
+      .blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
+      .blue { fill: #1478c9; }
+    </style>
+  </defs>
+
+  <rect width="1920" height="1295" fill="#fff"/>
+  <text x="20" y="70" class="title">Multivariate Models Lifecycle</text>
+
+  <!-- VictoriaMetrics and query configuration -->
+  <rect x="535" y="175" width="420" height="215" class="box"/>
+  <text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
+  <text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
+  <rect x="270" y="270" width="150" height="155" class="box"/>
+  <text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
+  <text x="285" y="360" class="label">MetricsQL</text>
+  <text x="285" y="395" class="label">queries</text>
+  <path d="M420 346 H535" class="line"/>
+
+  <!-- Reader and datasource exchange -->
+  <rect x="580" y="500" width="310" height="100" class="box"/>
+  <text x="735" y="562" class="node" text-anchor="middle">Reader</text>
+  <path d="M580 545 H455 V365 H535" class="line"/>
+  <text x="465" y="478" class="label">1. Request data</text>
+  <path d="M955 270 H1040 V550 H890" class="line"/>
+  <text x="810" y="478" class="label">2. Get metrics</text>
+
+  <!-- Historical fit data returned by the configured queries -->
+  <g aria-label="Historical fit data">
+    <rect x="1090" y="85" width="430" height="505" class="group"/>
+    <rect x="1120" y="135" width="340" height="170" class="group"/>
+    <text x="1140" y="180" class="node">Query 1</text>
+    <rect x="1135" y="195" width="300" height="42" class="box"/>
+    <text x="1150" y="228" class="label">Metric 1.1</text>
+    <text x="1150" y="262" class="label">...</text>
+    <rect x="1135" y="265" width="300" height="42" class="box"/>
+    <text x="1150" y="298" class="label">Metric 1.M₁</text>
+    <text x="1305" y="355" class="node" text-anchor="middle">...</text>
+    <rect x="1120" y="390" width="340" height="170" class="group"/>
+    <text x="1140" y="435" class="node">Query N</text>
+    <rect x="1135" y="450" width="300" height="42" class="box"/>
+    <text x="1150" y="483" class="label">Metric N.1</text>
+    <text x="1150" y="517" class="label">...</text>
+    <rect x="1135" y="520" width="300" height="42" class="box"/>
+    <text x="1150" y="553" class="label">Metric N.Mₙ</text>
+  </g>
+
+  <!-- One shared model is fitted on the complete aligned set -->
+  <rect x="1215" y="795" width="310" height="115" class="box"/>
+  <text x="1370" y="865" class="node" text-anchor="middle">Model</text>
+  <rect x="1365" y="635" width="155" height="130" class="box"/>
+  <text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1382" y="723" class="label">Model</text>
+  <text x="1382" y="757" class="label">config</text>
+  <path d="M1305 590 V795" class="line"/>
+  <text x="1055" y="672" class="label">
+    <tspan x="1055" dy="0">3. Fit one model</tspan>
+    <tspan x="1055" dy="38">on all historical series</tspan>
+  </text>
+  <path d="M1443 765 V795" class="line"/>
+
+  <!-- Inference data must contain the same set of series -->
+  <g aria-label="Inference data">
+    <rect x="25" y="615" width="430" height="500" class="group"/>
+    <rect x="55" y="645" width="340" height="170" class="group"/>
+    <text x="75" y="690" class="node">Query 1</text>
+    <rect x="70" y="705" width="300" height="42" class="box"/>
+    <text x="85" y="738" class="label">Metric 1.1</text>
+    <text x="85" y="772" class="label">...</text>
+    <rect x="70" y="775" width="300" height="42" class="box"/>
+    <text x="85" y="808" class="label">Metric 1.M₁</text>
+    <text x="225" y="870" class="node" text-anchor="middle">...</text>
+    <rect x="55" y="900" width="340" height="175" class="group"/>
+    <text x="75" y="945" class="node">Query N</text>
+    <rect x="70" y="960" width="300" height="42" class="box"/>
+    <text x="85" y="993" class="label">Metric N.1</text>
+    <text x="85" y="1027" class="label">...</text>
+    <rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
+    <text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
+  </g>
+  <path d="M580 575 H500 V650 H455" class="line"/>
+  <text x="465" y="680" class="label">4. Provide inference data</text>
+
+  <!-- Single multivariate model registry -->
+  <g aria-label="Multivariate model registry">
+    <rect x="580" y="750" width="420" height="480" class="group"/>
+    <text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
+    <rect x="610" y="835" width="360" height="220" class="model-group"/>
+    <rect x="640" y="885" width="300" height="115" class="box"/>
+    <text x="790" y="955" class="node" text-anchor="middle">Model (single)</text>
+  </g>
+  <path d="M455 1050 H580" class="line"/>
+  <path d="M1215 852 H1000" class="line"/>
+
+  <!-- Joint output and mismatched-series skip path -->
+  <rect x="1600" y="1015" width="285" height="105" class="box"/>
+  <text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
+  <rect x="1740" y="805" width="145" height="145" class="box"/>
+  <text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1758" y="900" class="label">Writer</text>
+  <text x="1758" y="935" class="label">config</text>
+  <path d="M1813 950 V1015" class="line"/>
+  <path d="M1000 1040 H1600" class="line"/>
+  <text x="1295" y="965" class="label" text-anchor="middle">
+    <tspan x="1295" dy="0">5.a Produce one anomaly-score series</tspan>
+    <tspan x="1295" dy="38">with the intersected label set</tspan>
+  </text>
+  <path d="M1000 1110 H1600" class="blue-line"/>
+  <text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
+    <tspan x="1320" dy="0">5.b Skip inference when the fit and inference</tspan>
+    <tspan x="1320" dy="32">series sets differ;</tspan>
+    <tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
+  </text>
+
+  <!-- Persist the joint anomaly score -->
+  <path d="M1743 1015 V80 H745 V175" class="line"/>
+  <text x="1170" y="55" class="label" text-anchor="middle">6. Write one anomaly-score series (label set = intersection)</text>
+</svg>

docs/anomaly-detection/components/model-lifecycle-univariate.svg

@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
+  <title id="title">Univariate models lifecycle</title>
+  <desc id="description">MetricsQL queries retrieve multiple series from VictoriaMetrics. Reader data fits one model per series in the model registry. Known series produce individual anomaly scores for Writer; an unseen series is skipped until a fitted model exists. Writer stores produced scores in VictoriaMetrics.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
+      .title { font-size: 50px; font-weight: 400; }
+      .node { font-size: 34px; font-weight: 400; }
+      .label { font-size: 29px; }
+      .blue-label { font-size: 26px; }
+      .small { font-size: 25px; }
+      .box { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .group { fill: #fff; stroke: #303038; stroke-width: 3; }
+      .model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
+      .line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
+      .blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
+      .blue { fill: #1478c9; }
+    </style>
+  </defs>
+
+  <rect width="1920" height="1295" fill="#fff"/>
+  <text x="20" y="70" class="title">Univariate Models Lifecycle</text>
+
+  <!-- VictoriaMetrics and query configuration -->
+  <rect x="535" y="175" width="420" height="215" class="box"/>
+  <text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
+  <text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
+  <rect x="270" y="270" width="150" height="155" class="box"/>
+  <text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
+  <text x="285" y="360" class="label">MetricsQL</text>
+  <text x="285" y="395" class="label">queries</text>
+  <path d="M420 346 H535" class="line"/>
+
+  <!-- Reader and datasource exchange -->
+  <rect x="580" y="500" width="310" height="100" class="box"/>
+  <text x="735" y="562" class="node" text-anchor="middle">Reader</text>
+  <path d="M580 545 H455 V365 H535" class="line"/>
+  <text x="465" y="478" class="label">1. Request data</text>
+  <path d="M955 270 H1040 V550 H890" class="line"/>
+  <text x="810" y="478" class="label">2. Get metrics</text>
+
+  <!-- Historical fit data returned by the configured queries -->
+  <g aria-label="Historical fit data">
+    <rect x="1090" y="85" width="430" height="505" class="group"/>
+    <rect x="1120" y="135" width="340" height="170" class="group"/>
+    <text x="1140" y="180" class="node">Query 1</text>
+    <rect x="1135" y="195" width="300" height="42" class="box"/>
+    <text x="1150" y="228" class="label">Metric 1.1</text>
+    <text x="1150" y="262" class="label">...</text>
+    <rect x="1135" y="265" width="300" height="42" class="box"/>
+    <text x="1150" y="298" class="label">Metric 1.M₁</text>
+    <text x="1305" y="355" class="node" text-anchor="middle">...</text>
+    <rect x="1120" y="390" width="340" height="170" class="group"/>
+    <text x="1140" y="435" class="node">Query N</text>
+    <rect x="1135" y="450" width="300" height="42" class="box"/>
+    <text x="1150" y="483" class="label">Metric N.1</text>
+    <text x="1150" y="517" class="label">...</text>
+    <rect x="1135" y="520" width="300" height="42" class="box"/>
+    <text x="1150" y="553" class="label">Metric N.Mₙ</text>
+  </g>
+
+  <!-- Model fitting -->
+  <rect x="1215" y="795" width="310" height="115" class="box"/>
+  <text x="1370" y="865" class="node" text-anchor="middle">Model</text>
+  <rect x="1365" y="635" width="155" height="130" class="box"/>
+  <text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1382" y="723" class="label">Model</text>
+  <text x="1382" y="757" class="label">config</text>
+  <path d="M1305 590 V795" class="line"/>
+  <text x="1055" y="672" class="label">
+    <tspan x="1055" dy="0">3. Fit models</tspan>
+    <tspan x="1055" dy="38">on historical data</tspan>
+  </text>
+  <path d="M1443 765 V795" class="line"/>
+
+  <!-- Inference data, including a new unseen series -->
+  <g aria-label="Inference data">
+    <rect x="25" y="615" width="430" height="500" class="group"/>
+    <rect x="55" y="645" width="340" height="170" class="group"/>
+    <text x="75" y="690" class="node">Query 1 (has fit model)</text>
+    <rect x="70" y="705" width="300" height="42" class="box"/>
+    <text x="85" y="738" class="label">Metric 1.1</text>
+    <text x="85" y="772" class="label">...</text>
+    <rect x="70" y="775" width="300" height="42" class="box"/>
+    <text x="85" y="808" class="label">Metric 1.M₁</text>
+    <text x="225" y="870" class="node" text-anchor="middle">...</text>
+    <rect x="55" y="900" width="340" height="175" class="group"/>
+    <text x="75" y="945" class="node">Query N</text>
+    <rect x="70" y="960" width="300" height="42" class="box"/>
+    <text x="85" y="993" class="label">Metric N.1</text>
+    <text x="85" y="1027" class="label">...</text>
+    <rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
+    <text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
+  </g>
+  <path d="M580 575 H500 V650 H455" class="line"/>
+  <text x="465" y="680" class="label">4. Provide inference data</text>
+
+  <!-- One fitted model per known series -->
+  <g aria-label="Univariate model registry">
+    <rect x="580" y="750" width="420" height="480" class="group"/>
+    <text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
+    <rect x="610" y="830" width="360" height="170" class="model-group"/>
+    <rect x="630" y="865" width="300" height="42" class="box"/>
+    <text x="645" y="898" class="label">Model 1.1</text>
+    <text x="645" y="932" class="label">...</text>
+    <rect x="630" y="935" width="300" height="42" class="box"/>
+    <text x="645" y="968" class="label">Model 1.M₁</text>
+    <text x="790" y="1045" class="node" text-anchor="middle">...</text>
+    <rect x="610" y="1070" width="360" height="135" class="model-group"/>
+    <rect x="630" y="1090" width="300" height="42" class="box"/>
+    <text x="645" y="1123" class="label">Model N.1</text>
+    <rect x="630" y="1145" width="300" height="42" class="box"/>
+    <text x="645" y="1178" class="label">Model N.Mₙ</text>
+  </g>
+  <path d="M455 1050 H580" class="line"/>
+  <path d="M1215 852 H1000" class="line"/>
+
+  <!-- Known-series output and unseen-series skip path -->
+  <rect x="1600" y="1015" width="285" height="105" class="box"/>
+  <text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
+  <rect x="1740" y="805" width="145" height="145" class="box"/>
+  <text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
+  <text x="1758" y="900" class="label">Writer</text>
+  <text x="1758" y="935" class="label">config</text>
+  <path d="M1813 950 V1015" class="line"/>
+  <path d="M1000 1040 H1600" class="line"/>
+  <text x="1275" y="975" class="label" text-anchor="middle">
+    <tspan x="1275" dy="0">5.a Produce anomaly scores</tspan>
+    <tspan x="1275" dy="38">for known series</tspan>
+  </text>
+  <path d="M1000 1110 H1600" class="blue-line"/>
+  <text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
+    <tspan x="1320" dy="0">5.b Skip inference until a fitted model exists</tspan>
+    <tspan x="1320" dy="32">for Metric N.Mₖ;</tspan>
+    <tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
+  </text>
+
+  <!-- Persist anomaly scores -->
+  <path d="M1743 1015 V80 H745 V175" class="line"/>
+  <text x="1130" y="55" class="label" text-anchor="middle">6. Write back produced anomaly scores</text>
+</svg>

docs/anomaly-detection/components/models.md

@@ -68,6 +68,10 @@ models:
 
 Common arguments supported by every model were introduced in [v1.10.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1100).
 
+<div class="collapse-group">
+
+{{% collapse name="Queries" %}}
+
 ### Queries
 
 The `queries` argument selects the [reader queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) used to fit and run a particular model{{% available_from "v1.10.0" anomaly %}}. Every series returned by a selected query is passed to that model.
@@ -93,6 +97,10 @@ models:
     queries: ['q1', 'q2', 'q3']  # i.e., if your `queries` in `reader` section has exactly q1, q2, q3 aliases
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Schedulers" %}}
+
 ### Schedulers
 
 The `schedulers` argument selects the [schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) that run a particular model{{% available_from "v1.11.0" anomaly %}}.
@@ -118,6 +126,10 @@ models:
     schedulers: ['s1', 's2', 's3']  # i.e., if your `schedulers` section has exactly s1, s2, s3 aliases
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Provide series" %}}
+
 ### Provide series
 
 The `provide_series` argument{{% available_from "v1.12.0" anomaly %}} limits the [model output](#vmanomaly-output) sent to the writer. For example, a model may produce `['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']` by default, while the following configuration writes only `anomaly_score` for each input series:
@@ -131,6 +143,10 @@ models:
 
 > If `provide_series` is not specified in model config, the model will produce its default [model-dependent output](#vmanomaly-output). The output can't be less than `['anomaly_score']`. Even if `timestamp` column is omitted, it will be implicitly added to `provide_series` list, as it's required for metrics to be properly written.
 
+{{% /collapse %}}
+
+{{% collapse name="Detection direction" %}}
+
 ### Detection direction
 The `detection_direction` argument{{% available_from "v1.13.0" anomaly %}} can reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive) when domain knowledge indicates that only values above or below the expected value are anomalous. Available values are `both`, `above_expected`, and `below_expected`.
 
@@ -190,6 +206,10 @@ reader:
 # other components like writer, schedule, monitoring
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Minimal deviation from expected" %}}
+
 ### Minimal deviation from expected
 
 `min_dev_from_expected`{{% available_from "v1.13.0" anomaly %}} argument is designed to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** in scenarios where deviations between the actual value (`y`) and the expected value (`yhat`) are **relatively** high. Such deviations can cause models to generate high [anomaly scores](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score). However, these deviations may not be significant enough in **absolute values** from a business perspective to be considered anomalies. This parameter ensures that anomaly scores for data points where `|y - yhat| < min_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it is set to `0` to maintain backward compatibility.
@@ -238,6 +258,10 @@ models:
     queries: ['normal_behavior']  # use the default where it's not needed
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Minimal relative deviation from expected" %}}
+
 ### Minimal relative deviation from expected
 
 {{% available_from "v1.29.1" anomaly %}} `min_rel_dev_from_expected` argument serves a similar purpose to `min_dev_from_expected` (see [section above](#minimal-deviation-from-expected)), but focuses on **relative deviations** rather than absolute ones. It is designed to reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive) in scenarios where the relative deviation between the actual value (`y`) and the expected value (`yhat`) is high, but the absolute deviation is not significant enough to be considered an anomaly from a business perspective. This parameter ensures that anomaly scores for data points where `|y - yhat| / |yhat| < min_rel_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it is set to `0` to maintain backward compatibility. 
@@ -278,6 +302,10 @@ models:
 ```
   
 
+{{% /collapse %}}
+
+{{% collapse name="Group by" %}}
+
 ### Group by
 
 > The `groupby` argument works only in combination with [multivariate models](#multivariate-models).
@@ -316,6 +344,10 @@ models:
     groupby: [host]
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Scale" %}}
+
 ### Scale
 
 Previously available only to [ProphetModel](#prophet) and [OnlineQuantileModel](#online-seasonal-quantile),  the `scale` {{% available_from "v1.20.0" anomaly %}} parameter is now applicable to all models that support generating predictions (`yhat`, `yhat_lower`, `yhat_upper`). Also, it is **two-sided** now, represented as a list of two positive float values, allowing separate scaling for the intervals `[yhat, yhat_upper]` and `[yhat_lower, yhat]`. The new margins are calculated as:
@@ -346,6 +378,10 @@ models:
     scale: [1.2, 0.75]
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Clip predictions" %}}
+
 ### Clip predictions
 
 A post-processing step to **clip model predictions** (`yhat`, `yhat_lower`, and `yhat_upper` series) to the configured [`data_range` values](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) in `VmReader` is available.
@@ -400,6 +436,10 @@ models:
     ]
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Score outside data range" %}}
+
 ### Score outside data range
 
 The `anomaly_score_outside_data_range` {{% available_from "v1.20.0" anomaly %}} parameter allows overriding the default **anomaly score (`1.01`)** assigned when actual values (`y`) fall **outside the defined `data_range` if defined in [reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/)**. This provides greater flexibility for **alerting rule configurations** and enables **clearer visual differentiation** between different types of anomalies:
@@ -445,6 +485,10 @@ models:
     anomaly_score_outside_data_range: 3.0  
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="Decay" %}}
+
 ### Decay
 
 > The `decay` argument works only in combination with [online models](#online-models) like [`ZScoreOnlineModel`](#online-z-score) or [`OnlineQuantileModel`](#online-seasonal-quantile).
@@ -477,6 +521,10 @@ models:
     queries: ['q1']
 ```
 
+{{% /collapse %}}
+
+</div>
+
 
 ## Model types
 
@@ -502,7 +550,7 @@ If during an inference, you got a series having **new labelset** (not present in
 
 **Examples:** [Prophet](#prophet), [Holt-Winters](#holt-winters)
 
-![vmanomaly-model-type-univariate](model-lifecycle-univariate.webp)
+![Univariate model lifecycle](model-lifecycle-univariate.svg)
 
 
 ### Multivariate Models
@@ -517,9 +565,34 @@ If during an inference, you got a **different amount of series** or some series
 
 **Implications:** Multivariate models are a go-to default, when your queries returns **fixed** amount of **individual** time series (say, some aggregations), to be used for adding cross-series (and cross-query) context, useful for catching [collective anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#collective-anomalies) or [novelties](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#novelties) (expanded to multi-input scenario). For example, you may set it up for anomaly detection of CPU usage in different modes (`idle`, `user`, `system`, etc.) and use its cross-dependencies to detect **unseen (in fit data)** behavior.
 
-**Examples:** [IsolationForest](#isolation-forest-multivariate)
+**Examples:** [Temporal Envelope](#temporal-envelope), [Isolation Forest](#isolation-forest-multivariate)
 
-![vmanomaly-model-type-multivariate](model-lifecycle-multivariate.webp)
+![Multivariate model lifecycle](model-lifecycle-multivariate.svg)
+
+The following configuration applies both models to the same aligned input series. Start with Temporal Envelope when temporal profiles and online adaptation matter; use Isolation Forest as an offline alternative when feature-space outliers are the primary concern.
+
+```yaml
+models:
+  service_dependency_envelope:
+    class: temporal_envelope_multivariate
+    queries: [request_rate, error_rate, latency]
+    groupby: [cluster]
+    dependency_rank: 8
+    score_aggregation: l2
+    seasonalities: [hod_smooth, dow_smooth]
+    provide_series: [anomaly_score]
+
+  service_dependency_isolation_forest:
+    class: isolation_forest_multivariate
+    queries: [request_rate, error_rate, latency]
+    groupby: [cluster]
+    contamination: 0.01
+    seasonal_features: [hod, dow]
+    args:
+      n_estimators: 100
+      random_state: 42
+    provide_series: [anomaly_score]
+```
 
 
 ### Online Models
@@ -664,7 +737,7 @@ models:
 
 </div>
 
-![vmanomaly-autotune-schema](autotune.webp)
+![AutoTunedModel tuning and inference lifecycle](autotune.svg)
 
 #### Shared asynchronous autotune workflow
 
@@ -1344,6 +1417,10 @@ This guide shows how to:
 
 > The file containing the model must be written in [Python](https://www.python.org/) 3.14 or later. A custom model runs inside the `vmanomaly` Python environment, so keep its dependencies compatible with the target image and keep the module available when restoring serialized model state after a restart.
 
+<div class="collapse-group">
+
+{{% collapse name="Custom model implementation guide" %}}
+
 ### 1. Custom model
 
 Create `custom_model.py` with a `CustomModel` class derived from `Model`. A concrete model must implement:
@@ -1510,6 +1587,10 @@ The writer emits one `custom_anomaly_score` series for each input series. It ret
 {__name__="custom_anomaly_score", for="churn_rate",     model_alias="custom_model", scheduler_alias="s1", run="test-format"}
 ```
 
+{{% /collapse %}}
+
+</div>
+
 ## Deprecations
 
 {{% collapse name="Deprecated model types and models" %}}

docs/anomaly-detection/components/monitoring.md

@@ -24,6 +24,10 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
 - Adding `preset` and `scheduler_alias` keys to [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics) and [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics) metrics for consistency in multi-[scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) setups.
 - Renaming [Counters](https://prometheus.io/docs/concepts/metric_types/#counter) `vmanomaly_reader_response_count` to `vmanomaly_reader_responses` and `vmanomaly_writer_response_count` to `vmanomaly_writer_responses`.
 
+<div class="collapse-group">
+
+{{% collapse name="Pull model config parameters" %}}
+
 ## Pull Model Config parameters
 
 <table class="params">
@@ -60,6 +64,10 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
     </tbody>
 </table>
 
+{{% /collapse %}}
+
+{{% collapse name="Push config parameters" %}}
+
 ## Push Config parameters
 
 By default, metrics are pushed only after the completion of specific stages, e.g., `fit`, `infer`, or `fit_infer` (for each [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) if using a multi-scheduler configuration).
@@ -227,6 +235,10 @@ Path to a file with the client certificate key, i.e. `client.key`{{% available_f
     </tbody>
 </table>
 
+{{% /collapse %}}
+
+</div>
+
 ## Monitoring section config example
 
 ``` yaml
@@ -260,6 +272,10 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
 - [Model metrics](#models-behaviour-metrics)
 - [Writer metrics](#writer-behaviour-metrics)
 
+<div class="collapse-group">
+
+{{% collapse name="Startup metrics" %}}
+
 ### Startup metrics
 
 <table class="params">
@@ -385,6 +401,10 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Reader behaviour metrics" %}}
+
 ### Reader behaviour metrics
 Label names [description](#labelnames)
 
@@ -509,6 +529,10 @@ Label names [description](#labelnames)
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Models behaviour metrics" %}}
+
 ### Models behaviour metrics
 Label names [description](#labelnames)
 
@@ -635,6 +659,10 @@ Label names [description](#labelnames)
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Writer behaviour metrics" %}}
+
 ### Writer behaviour metrics
 Label names [description](#labelnames)
 
@@ -746,6 +774,10 @@ Label names [description](#labelnames)
 
 [Back to metric sections](#metrics-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+</div>
+
 ### Labelnames
 
 * `stage` - model execution stage: `fit`, `infer`, or `fit_infer` for a combined fit/inference scheduler run. See [model types](https://docs.victoriametrics.com/anomaly-detection/components/models/#model-types).
@@ -793,6 +825,10 @@ and the [command-line arguments](https://docs.victoriametrics.com/anomaly-detect
 - [Query server and task logs](#query-server-and-task-logs)
 - [AI Copilot logs](#ai-copilot-logs)
 
+<div class="collapse-group">
+
+
+{{% collapse name="Startup logs" %}}
 
 ### Startup logs
 
@@ -812,6 +848,10 @@ server addresses, hot-reload state, and active schedulers. The most useful prefi
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Reader logs" %}}
+
 ### Reader logs
 
 Reader logs cover endpoint checks, request splitting, network failures, response parsing, and coordination between
@@ -863,6 +903,10 @@ or parsed. See [reader behaviour metrics](#reader-behaviour-metrics).
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Service logs" %}}
+
 ### Service logs
 
 The service logs `fit`, `infer`, and combined `fit_infer`/backtesting work for each model alias and scheduler.
@@ -891,6 +935,10 @@ an unsuccessful stage. See [models behaviour metrics](#models-behaviour-metrics)
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Writer logs" %}}
+
 ### Writer logs
 
 Writer logs cover serialization and delivery of produced series such as
@@ -918,6 +966,10 @@ and datapoints are recorded only after a successful response. See [writer behavi
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Scheduler supervision logs" %}}
+
 ### Scheduler supervision logs
 
 Scheduler supervision{{% available_from "v1.30.0" anomaly %}} logs a dead worker, automatic restart, successful
@@ -927,6 +979,10 @@ Correlate them with `vmanomaly_scheduler_alive` and `vmanomaly_scheduler_restart
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Hot-reload logs" %}}
+
 ### Hot-reload logs
 
 Hot reload logs config-change detection, validation, staged service restart, success, and rollback. `Reload aborted
@@ -936,6 +992,10 @@ without restarting services`.
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Persisted-state logs" %}}
+
 ### Persisted-state logs
 
 With `settings.restore_state`, startup logs the stored/runtime version assessment, reusable components, required
@@ -944,6 +1004,10 @@ stored artifacts completely` indicates a full reset; missing or unreadable model
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="Query server and task logs" %}}
+
 ### Query server and task logs
 
 The query server logs its listening address and datasource-proxy timeouts/failures. Background anomaly-detection
@@ -952,6 +1016,10 @@ background raw query finishing cleanly.
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
 
+{{% /collapse %}}
+
+{{% collapse name="AI Copilot logs" %}}
+
 ### AI Copilot logs
 
 AI Copilot{{% available_from "v1.30.0" anomaly %}} reports whether it is initialized, disabled, misconfigured, or
@@ -960,3 +1028,7 @@ request failed` identifies provider execution failure, and `MCP server unreachab
 guidance tools.
 
 [Back to logging sections](#logs-generated-by-vmanomaly)
+
+{{% /collapse %}}
+
+</div>

docs/anomaly-detection/components/reader.md

@@ -25,6 +25,8 @@ Use the following playgrounds to develop and test input queries:
 
 ## VM reader
 
+<div class="collapse-group">
+
 {{% collapse name="Queries format migration (to v1.13.0+)" %}}
 
 > The backward-compatible `queries` format introduced in v1.13.0 allows [VmReader](#vm-reader) parameters such as `step` to be configured per query. This can reduce the amount of data read from VictoriaMetrics. See [per-query parameters](#per-query-parameters) for details.
@@ -61,6 +63,8 @@ reader:
 ```
 {{% /collapse %}}
 
+{{% collapse name="VM reader per-query parameters and example" %}}
+
 ### Per-query parameters
 
 There is change {{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg format. Now each query alias supports the next (sub)fields, which *override reader-level parameters*, if set:
@@ -125,6 +129,10 @@ reader:
       offset: '-15s'  # to override reader-wise `offset` and query data 15 seconds earlier to account for data collection delays
 ```
 
+{{% /collapse %}}
+
+{{% collapse name="VM reader config parameters and example" %}}
+
 ### Config parameters
 
 <table class="params">
@@ -507,10 +515,16 @@ reader:
   series_processing_batch_size: 8
 ```
 
+{{% /collapse %}}
+
+</div>
+
 ### MetricsQL Playground
 
 To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMetrics MetricsQL Playground](https://play.victoriametrics.com/), which provides an interactive environment to test and visualize your queries against sample data. You can also access embedded version of the playground below:
 
+<div class="collapse-group">
+
 {{% collapse name="VictoriaMetrics Playground" %}}
 
 <div class="position-relative mb-3">
@@ -536,6 +550,8 @@ To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMe
 
 {{% /collapse %}}
 
+</div>
+
 ### mTLS protection
 
 `vmanomaly` supports [mutual TLS (mTLS)](https://en.wikipedia.org/wiki/Mutual_authentication){{% available_from "v1.16.3" anomaly %}} for secure communication across its components, including [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters). This allows for mutual authentication between the client and server when querying or writing data to [VictoriaMetrics Enterprise, configured for mTLS](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#mtls-protection).
@@ -680,6 +696,8 @@ Similarly, [VictoriaTraces LogsQL Playground](https://play-vtraces.victoriametri
 
 You can also access **embedded version of the playground below** (VictoriaLogs datasource):
 
+<div class="collapse-group">
+
 {{% collapse name="VictoriaLogs LogsQL Playground" %}}
 
 <div class="position-relative mb-3">
@@ -705,6 +723,11 @@ You can also access **embedded version of the playground below** (VictoriaLogs d
 
 {{% /collapse %}}
 
+</div>
+
+<div class="collapse-group">
+
+{{% collapse name="VictoriaLogs reader config parameters" %}}
 
 ### Config parameters
 
@@ -989,6 +1012,10 @@ Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far last-seen
     </tbody>
 </table>
 
+{{% /collapse %}}
+
+{{% collapse name="VictoriaLogs reader per-query parameters and example" %}}
+
 ### Per-query parameters
 
 The names, types and the logic of the per-query parameters subset used in `VLogsReader` are exactly the same as those of [`VmReader`](#vm-reader), please see [per-query parameters](#per-query-parameters) section above for the details. The only difference is that `expr` parameter should contain a valid [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) expression with `stats` [pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe), as described in [query examples](#query-examples) section above.
@@ -1036,6 +1063,10 @@ reader:
 # other config sections, like models, schedulers, writer, ...
 ```
 
+{{% /collapse %}}
+
+</div>
+
 ### mTLS protection
 
 Please refer to the [mTLS protection](#mtls-protection) section above for details on how to configure mTLS for `VLogsReader`. It uses the same config parameters as `VmReader` for mTLS setup.

docs/anomaly-detection/components/settings.md

@@ -226,6 +226,10 @@ monitoring:
   # other monitoring settings
 ```
 
+<div class="collapse-group">
+
+{{% collapse name="State restoration example" %}}
+
 ### Example
 
 For a configuration with the following models, queries and schedulers:
@@ -305,6 +309,10 @@ This means that the service upon restart:
 1. Won't restore the state of `zscore_online` model, because its `z_threshold` argument **has changed**, retraining from scratch is needed on the last `fit_window` = 24 hours of data for `q1`, `q2` and `q3` (as model's `queries` arg is not set so it defaults to all queries found in the reader).
 2. Will **partially** restore the state of `prophet` model, because its class and schedulers are unchanged, but **only instances trained on timeseries returned by `q1` query**. New fit/infer jobs will be set for new query `q3`. The old query `q2` artifacts will be dropped upon restart - all respective models and data for (`prophet`, `q2`) combination will be removed from the database file and from the disk.
 
+{{% /collapse %}}
+
+</div>
+
 ## Retention
 
 {{% available_from "v1.28.1" anomaly %}} The `retention` argument sets a [time to live](https://en.wikipedia.org/wiki/Time_to_live) (TTL) for service artifacts such as stored model instances and training data. At each `check_interval`, the service removes artifacts that have not been used for inference or refitting within `ttl`. This bounds stale resource usage in long-running deployments.

docs/anomaly-detection/components/vmanomaly-components-diagram.md

@@ -0,0 +1,15 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+
+The required path is `config.yml` → Scheduler → Reader → Model → Writer. The Reader queries the configured VictoriaMetrics, VictoriaLogs, or VictoriaTraces datasource; the Writer stores inferred anomaly scores in VictoriaMetrics. Monitoring is optional and can push metrics or expose them for collection.
+
+Solid nodes and arrows show the required anomaly-detection path. Dashed nodes and arrows show optional self-monitoring integrations.
+
+![vmanomaly component interaction: the Scheduler starts Reader, Model, and Writer work against a configured datasource; Monitoring is optional](/anomaly-detection/components/vmanomaly-components.svg)
+{style="display:block; width:80%; min-width:320px; margin:1.5rem auto"}

docs/anomaly-detection/components/vmanomaly-components.svg

@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1600" role="img" aria-labelledby="title description">
+  <title id="title">How vmanomaly operates during a scheduled iteration</title>
+  <desc id="description">The required flow starts from config.yml and the Scheduler. The Reader queries either VictoriaMetrics through query_range or VictoriaLogs and VictoriaTraces through stats_query_range, then sends data to a Model. The Model produces anomaly scores, and the Writer stores them in VictoriaMetrics through import. Reader, Model, and Writer can optionally report self-monitoring metrics using push or pull monitoring.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <marker id="arrow-optional" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
+      <path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#666a73" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
+    </marker>
+    <style>
+      .label { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
+      .title { font-size: 54px; font-weight: 400; }
+      .service-title { font-size: 46px; font-weight: 400; }
+      .node-text { font-size: 34px; font-weight: 600; text-anchor: middle; }
+      .body { font-size: 28px; }
+      .edge-label { font-size: 25px; }
+      .endpoint-text { font-size: 23px; }
+      .source-text { font-size: 21px; }
+      .required-node { fill: #fff; stroke: #303038; stroke-width: 4; }
+      .optional-node { fill: #fff; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
+      .required-line { fill: none; stroke: #303038; stroke-width: 4; marker-end: url(#arrow); }
+      .optional-line { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; marker-end: url(#arrow-optional); }
+      .boundary { fill: none; stroke: #303038; stroke-width: 4; }
+      .optional-boundary { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
+    </style>
+  </defs>
+
+  <rect width="1920" height="1600" fill="#fff"/>
+  <text x="105" y="72" class="label title">How does vmanomaly operate (scheduled iteration example)</text>
+
+  <!-- Required / optional legend -->
+  <g aria-label="Legend" transform="translate(1425 145)">
+    <rect x="0" y="0" width="135" height="62" class="optional-node"/>
+    <rect x="205" y="0" width="135" height="62" class="required-node"/>
+    <path d="M0 100 H135" class="optional-line"/>
+    <path d="M205 100 H340" class="required-line"/>
+    <text x="67" y="162" class="label body" text-anchor="middle">Optional</text>
+    <text x="272" y="162" class="label body" text-anchor="middle">Required</text>
+  </g>
+
+  <!-- Exactly one configured read datasource is used. -->
+  <g aria-label="Configured datasource choice">
+    <rect x="25" y="315" width="300" height="550" class="optional-boundary"/>
+
+    <g transform="translate(34 326)">
+      <path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
+      <ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
+      <rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
+      <text x="144" y="99" class="label endpoint-text" text-anchor="middle">/query_range</text>
+      <text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
+      <text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
+      <text x="144" y="182" class="label source-text" text-anchor="middle">TSDB</text>
+    </g>
+
+    <text x="178" y="600" class="label body" text-anchor="middle">OR</text>
+
+    <g transform="translate(34 632)">
+      <path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
+      <ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
+      <rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
+      <text x="144" y="99" class="label endpoint-text" text-anchor="middle">/stats_query_range</text>
+      <text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
+      <text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaLogs /</text>
+      <text x="144" y="182" class="label source-text" text-anchor="middle">VictoriaTraces</text>
+    </g>
+  </g>
+
+  <!-- vmanomaly service boundary and components -->
+  <g aria-label="vmanomaly service">
+    <rect x="720" y="145" width="610" height="1145" class="boundary"/>
+    <rect x="1005" y="175" width="270" height="100" class="required-node"/>
+    <text x="1140" y="235" class="label node-text">config.yml</text>
+    <text x="1025" y="352" class="label service-title" text-anchor="middle">vmanomaly</text>
+    <text x="1025" y="405" class="label service-title" text-anchor="middle">service</text>
+
+    <rect x="765" y="448" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="500" class="label node-text">Scheduler</text>
+    <rect x="765" y="590" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="642" class="label node-text">Reader</text>
+    <rect x="765" y="732" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="784" class="label node-text">Model</text>
+    <rect x="765" y="874" width="445" height="76" rx="16" class="required-node"/>
+    <text x="987" y="926" class="label node-text">Writer</text>
+    <rect x="778" y="1180" width="420" height="82" rx="16" class="optional-node"/>
+    <text x="988" y="1235" class="label node-text">Monitoring</text>
+
+    <path d="M1005 225 H900 V448" class="required-line"/>
+    <path d="M1210 486 H1290 V628 H1210" class="required-line"/>
+    <path d="M1210 628 H1290 V770 H1210" class="required-line"/>
+    <path d="M1210 770 H1290 V912 H1210" class="required-line"/>
+
+    <path d="M875 666 V1178" class="optional-line"/>
+    <path d="M987 808 V1178" class="optional-line"/>
+    <path d="M1100 950 V1178" class="optional-line"/>
+  </g>
+
+  <!-- Required flow labels -->
+  <text x="1365" y="455" class="label body">
+    <tspan x="1365" dy="0">1. Get the metrics to</tspan>
+    <tspan x="1365" dy="37">a. fit the model, or</tspan>
+    <tspan x="1365" dy="37">b. produce anomaly scores</tspan>
+  </text>
+  <text x="1365" y="620" class="label body">
+    <tspan x="1365" dy="0">3. Model receives data</tspan>
+    <tspan x="1365" dy="37">to train or infer on</tspan>
+  </text>
+  <text x="1365" y="775" class="label body">
+    <tspan x="1365" dy="0">4. Produce anomaly</tspan>
+    <tspan x="1365" dy="37">scores (inference)</tspan>
+  </text>
+
+  <!-- Datasource request and response -->
+  <path d="M765 610 H338" class="required-line"/>
+  <path d="M338 652 H765" class="required-line"/>
+  <text x="370" y="580" class="label edge-label">2.1 Query request</text>
+  <text x="370" y="700" class="label edge-label">2.2 Metrics response</text>
+
+  <!-- Anomaly-score output -->
+  <g aria-label="VictoriaMetrics write endpoint" transform="translate(26 1145)">
+    <path d="M8 32 V184 C8 224 290 224 290 184 V32 C290 70 8 70 8 32Z" class="required-node"/>
+    <ellipse cx="149" cy="32" rx="141" ry="33" class="required-node"/>
+    <rect x="34" y="72" width="230" height="66" rx="16" class="required-node"/>
+    <text x="149" y="99" class="label endpoint-text" text-anchor="middle">/import</text>
+    <text x="149" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
+    <text x="149" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
+    <text x="149" y="182" class="label source-text" text-anchor="middle">TSDB</text>
+  </g>
+  <path d="M765 912 H650 V1095 L316 1197" class="required-line"/>
+  <rect x="338" y="1018" width="292" height="74" fill="#fff"/>
+  <text x="350" y="1048" class="label edge-label">
+    <tspan x="340" dy="0">5. Write anomaly scores</tspan>
+    <tspan x="340" dy="33">to VictoriaMetrics</tspan>
+  </text>
+
+  <!-- Optional monitoring integrations -->
+  <rect x="748" y="1083" width="490" height="48" fill="#fff"/>
+  <text x="993" y="1116" class="label edge-label" text-anchor="middle">6. Produce self-monitoring metrics</text>
+  <rect x="390" y="1460" width="300" height="118" rx="45" class="optional-node"/>
+  <text x="540" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
+  <text x="540" y="1552" class="label body" text-anchor="middle">push approach</text>
+  <rect x="1490" y="1460" width="300" height="118" rx="45" class="optional-node"/>
+  <text x="1640" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
+  <text x="1640" y="1552" class="label body" text-anchor="middle">pull approach</text>
+  <path d="M850 1262 L540 1460" class="optional-line"/>
+  <path d="M1198 1222 L1640 1460" class="optional-line"/>
+  <rect x="590" y="1350" width="180" height="42" fill="#fff"/>
+  <text x="680" y="1380" class="label edge-label" text-anchor="middle">Push metrics</text>
+  <rect x="1230" y="1340" width="415" height="42" fill="#fff"/>
+  <text x="1438" y="1370" class="label edge-label" text-anchor="middle">HTTP GET /metrics or /health</text>
+</svg>

docs/anomaly-detection/components/writer.md

@@ -17,6 +17,10 @@ Future updates will introduce additional export methods, offering users more fle
 
 ## VM writer
 
+<div class="collapse-group">
+
+{{% collapse name="VM writer config parameters and example" %}}
+
 ### Config parameters
 
 <table class="params">
@@ -35,8 +39,7 @@ Future updates will introduce additional export methods, offering users more fle
             </td>
             <td>
 
-<span style="white-space: nowrap;">`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
-</span>
+`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
             </td>
             <td>
 
@@ -63,10 +66,7 @@ Datasource URL address
 <span style="white-space: nowrap;">`tenant_id`</span>
             </td>
             <td>
-<span>
-
 `0:0`, `multitenant`{{% available_from "v1.16.2" anomaly %}}
-</span>
             </td>
             <td>
 
@@ -253,9 +253,8 @@ Token is passed in the standard format with header: `Authorization: bearer {toke
 `path_to_file`
             </td>
             <td>
-<span>
 Path to a file, which contains token, that is passed in the standard format with header: `Authorization: bearer {token}`{{% available_from "v1.15.9" anomaly %}}
-</span>            </td>
+            </td>
         </tr>
         <tr>
             <td>
@@ -267,9 +266,8 @@ Path to a file, which contains token, that is passed in the standard format with
 `1`
             </td>
             <td>
-<span>
 Number of attempts to retry the connection in case of failure {{% available_from "v1.29.2" anomaly %}}.
-</span>            </td>
+            </td>
         </tr>
     </tbody>
 </table>
@@ -293,6 +291,10 @@ writer:
   connection_retry_attempts: 2  # if not specified, it will be 1 by default
 ```
 
+{{% /collapse %}}
+
+</div>
+
 ### Multitenancy support
 
 > This feature applies to the VictoriaMetrics Cluster version only. Tenants are identified by either `accountID` or `accountID:projectID`. `multitenant` [endpoint](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels){{% available_from "v1.15.9" anomaly %}} is supported for writing data across multiple [tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy). For more details, refer to the VictoriaMetrics Cluster [multitenancy documentation](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -17,7 +17,7 @@ sitemap:
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
 
-![typical setup diagram](guide-vmanomaly-vmalert_overview.webp)
+![Typical vmanomaly observability pipeline](guide-vmanomaly-vmalert_overview.svg)
 
 > **Configurations used throughout this guide can be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-integration/)**
 

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1706 1069" role="img" aria-labelledby="title description">
+  <title id="title">Typical vmanomaly observability pipeline</title>
+  <desc id="description">vmagent scrapes node-exporter metrics and writes them to VictoriaMetrics. vmanomaly reads those metrics and writes anomaly scores back. Grafana visualizes the results. vmalert evaluates rules based on anomaly scores and sends alerts to Alertmanager.</desc>
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="10" markerHeight="10" orient="auto-start-reverse">
+      <path d="M0 0 10 5 0 10Z" fill="#252525"/>
+    </marker>
+    <style>
+      text { font-family: Arial, Helvetica, sans-serif; fill: #252525; font-size: 40px; }
+      .box { fill: #fff; stroke: #252525; stroke-width: 3; }
+      .arrow { fill: none; stroke: #252525; stroke-width: 4; marker-end: url(#arrow); }
+    </style>
+  </defs>
+
+  <rect width="1706" height="1069" fill="#fff"/>
+
+  <rect x="20" y="55" width="390" height="100" class="box"/>
+  <text x="215" y="119" text-anchor="middle">node-exporter</text>
+  <rect x="730" y="50" width="395" height="100" class="box"/>
+  <text x="928" y="115" text-anchor="middle">vmagent</text>
+  <path d="M710 100 H435" class="arrow"/>
+  <text x="570" y="56" text-anchor="middle">Scrape metrics</text>
+
+  <rect x="745" y="325" width="400" height="180" class="box"/>
+  <text x="945" y="405" text-anchor="middle">VictoriaMetrics</text>
+  <text x="945" y="457" text-anchor="middle">TSDB</text>
+  <path d="M945 150 V322" class="arrow"/>
+  <text x="1018" y="204">
+    <tspan x="1018" dy="0">Push node</tspan>
+    <tspan x="1018" dy="50">exporter</tspan>
+    <tspan x="1018" dy="50">metrics</tspan>
+  </text>
+
+  <rect x="25" y="340" width="395" height="100" class="box"/>
+  <text x="222" y="405" text-anchor="middle">vmanomaly</text>
+  <path d="M725 365 H440" class="arrow"/>
+  <text x="575" y="319" text-anchor="middle">Read metrics</text>
+  <path d="M440 421 H725" class="arrow"/>
+  <text x="570" y="508" text-anchor="middle">
+    <tspan x="570" dy="0">Write produced</tspan>
+    <tspan x="570" dy="50">anomaly scores</tspan>
+  </text>
+
+  <rect x="1440" y="330" width="245" height="170" class="box"/>
+  <text x="1562" y="430" text-anchor="middle">Grafana</text>
+  <path d="M1160 417 H1418" class="arrow"/>
+  <text x="1285" y="486" text-anchor="middle">
+    <tspan x="1285" dy="0">Visualize the</tspan>
+    <tspan x="1285" dy="50">results</tspan>
+  </text>
+
+  <rect x="750" y="738" width="395" height="100" class="box"/>
+  <text x="948" y="805" text-anchor="middle">vmalert</text>
+  <path d="M946 525 V716" class="arrow"/>
+  <text x="1000" y="644">
+    <tspan x="1000" dy="0">Evaluate rules based</tspan>
+    <tspan x="1000" dy="50">on anomaly scores</tspan>
+  </text>
+
+  <rect x="755" y="948" width="395" height="100" class="box"/>
+  <text x="952" y="1015" text-anchor="middle">alertmanager</text>
+  <path d="M948 858 V928" class="arrow"/>
+  <text x="1004" y="906">Send alerts</text>
+</svg>

docs/anomaly-detection/vmanomaly-sharding-ha-diagram.d2

@@ -0,0 +1,107 @@
+# Render with D2 v0.7.1:
+# d2 --layout elk --theme 0 --pad 20 vmanomaly-sharding-ha-diagram.d2 vmanomaly-sharding-ha-diagram.svg
+
+grid-columns: 1
+grid-gap: 36
+
+classes: {
+  boundary: {
+    style.fill: "#F7F7F8"
+    style.stroke: "#A7AAB2"
+    style.stroke-width: 2
+  }
+  process: {
+    style.fill: "#FFFFFF"
+    style.stroke: "#303038"
+    style.stroke-width: 2
+    style.border-radius: 8
+  }
+  member: {
+    style.fill: "#F1F3F5"
+    style.stroke: "#59616A"
+    style.stroke-width: 2
+    style.border-radius: 8
+  }
+}
+
+flow: "" {
+  grid-columns: 3
+  grid-gap: 36
+  style.fill: transparent
+  style.stroke: transparent
+
+  global: "Global YAML\nconfiguration" {
+    shape: page
+    class: process
+  }
+
+  splitting: Configuration splitting {
+    class: boundary
+    grid-columns: 1
+    grid-gap: 24
+
+    split: "Split by VMANOMALY_SPLIT_BY\ndefault: complete" {
+      class: process
+    }
+    subconfigs: "N valid sub-configurations\nn = 0 ... N-1" {
+      class: process
+      style.multiple: true
+    }
+
+    split -> subconfigs: {
+      style.stroke: "#303038"
+    }
+  }
+
+  placement: Deterministic placement {
+    class: boundary
+    grid-columns: 1
+    grid-gap: 24
+
+    settings: "K members: VMANOMALY_MEMBERS_COUNT\nMember index: VMANOMALY_MEMBER_NUM\nR replicas: VMANOMALY_REPLICATION_FACTOR" {
+      class: process
+    }
+    assign: "Assign every sub-configuration\nto exactly R members" {
+      class: process
+    }
+
+    settings -> assign: {
+      style.stroke: "#303038"
+    }
+  }
+}
+
+members: K vmanomaly members {
+  class: boundary
+  grid-columns: 3
+  grid-gap: 30
+
+  member0: "Member 0\nvmanomaly service 1" {
+    class: member
+  }
+  member1: "Member 1\nvmanomaly service 2" {
+    class: member
+  }
+  memberK: "Member K-1\nvmanomaly service K" {
+    class: member
+  }
+}
+
+flow.global -> flow.splitting.split: {
+  style.stroke: "#303038"
+}
+flow.splitting.subconfigs -> flow.placement.assign: {
+  style.stroke: "#303038"
+}
+flow.placement.assign -> members.member0: "assigned subset" {
+  style.stroke: "#59616A"
+  style.stroke-width: 2
+}
+flow.placement.assign -> members.member1: "assigned subset" {
+  style.stroke: "#59616A"
+  style.stroke-width: 2
+}
+flow.placement.assign -> members.memberK: "assigned subset" {
+  style.stroke: "#59616A"
+  style.stroke-width: 2
+}

docs/anomaly-detection/vmanomaly-sharding-ha-diagram.md

@@ -0,0 +1,13 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+
+The global configuration is split into `N` independently valid sub-configurations. Deterministic placement distributes them across `K` members, and each sub-configuration is assigned to exactly `R` members when replication is enabled. Member `k` processes only its assigned subset.
+
+![vmanomaly sharding and high availability: a global configuration is split into N sub-configurations and replicated across K members](/anomaly-detection/vmanomaly-sharding-ha-diagram.svg)
+{style="display:block; width:50%; min-width:320px; margin:1.5rem auto"}

docs/victoriametrics/changelog/CHANGELOG.md

@@ -28,13 +28,23 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 **Update Note 1:** `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the default value of `-disableRerouting` flag has changed from `true` to `false`, enabling [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. If you rely on the old behavior, pass `-disableRerouting` command-line flag to `vminsert`. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
 
+**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints now require `POST` method. Previously, it also accepted `GET` requests. If you use `GET` requests for this endpoint, update your scripts or tooling to use `POST` instead. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
+
+* SECURITY: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): restrict `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints to `POST` method only to prevent some [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery)-based data deletion attacks. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
+
+* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-remoteWrite.obfuscateLabels` flag for hashing values of the specified labels before sending metrics to the corresponding `-remoteWrite.url`. This allows sharing metrics with external systems while keeping sensitive label values hidden. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-replay.continueWithExecutionErr` flag to allow continuing to replay other rules when a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit. See [11313](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11313).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add template variable `$interval` to expose the alerting rule group's evaluation interval. This allows generating dashboard links with a lookback window relative to the rule's interval, for example `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`. See this issue [#11232](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11232) for more details. Thanks to @1solomonwakhungu for contribution.
+* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, 
+`NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support [Prometheus native histograms](https://prometheus.io/docs/specs/native_histograms/) migration in [remote read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/). Native histograms are converted into `_count`, `_sum` and `_bucket` series with `vmrange` labels in the same way as VictoriaMetrics converts native histograms received via Prometheus remote write protocol, except that for native histograms with custom buckets the original bucket bounds are preserved instead of being estimated with the exponential formula. Previously native histograms were silently ignored in `SAMPLES` mode, while in stream mode the migration failed with `EOF` error. See [#11292](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11292). Thanks to @liuxu623 for contribution.
 * FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enable [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Previously, `-disableRerouting` defaulted to `true`, which limited ingestion throughput to the slowest `vmstorage` node. Now `-disableRerouting` defaults to `false`, so `vminsert` automatically routes data away from the slowest `vmstorage` node, improving overall ingestion performance. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the selected auto-refresh interval in the URL. See [VictoriaLogs#1310](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1310).
-* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
-* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
 
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): ignore HTTP proxy environment variables when scraping targets over Unix domain sockets. See [#11318](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11318). Thanks to @lwmacct for contribution.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
 
@@ -243,7 +253,6 @@ Released at 2026-04-24
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect evaluation of binary operations caused by an ordering bug (e.g. `10 - (3 + 3 + 4)` being evaluated as `10 - 3 + 3 + 4`). The issue was introduced in v1.140.0, v1.136.4, and v1.122.19. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
 
 ## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
-
 Released at 2026-04-10
 
 **Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).

docs/victoriametrics/vmagent.md

@@ -323,7 +323,8 @@ flowchart TB
       H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
       H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
       H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
-      H5 --> H6[[push to <b>-remoteWrite.url</b>]]
+      H5 --> H6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
+      H6 --> H7[[push to <b>-remoteWrite.url</b>]]
 
       %% Right branch
       G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
@@ -331,7 +332,8 @@ flowchart TB
       R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
       R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
       R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
-      R5 --> R6[[push to <b>-remoteWrite.url</b>]]
+      R5 --> R6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
+      R6 --> R7[[push to <b>-remoteWrite.url</b>]]
 ```
 
 Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:
@@ -683,6 +685,35 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
   /path/to/vmagent -remoteWrite.url=http://127.0.0.1:8428/api/v1/write?extra_label="env=prod"
   ```
 
+## Obfuscating label values
+
+`vmagent` can obfuscate the values of specified labels before sending metrics to `-remoteWrite.url`
+via `-remoteWrite.obfuscateLabels`{{% available_from "#" %}}.
+
+This is useful when one or more `-remoteWrite.url` endpoints point to external monitoring services
+outside the organization, and sensitive label values such as `ip`, `host`, `instance`, or `datacenter`
+must not be exposed.
+
+Label values are replaced with their SHA-256 hex digests. No salt is applied, so values with a small
+or predictable value space can potentially be recovered by brute force.
+
+This feature pairs well with [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange):
+use `-remoteWrite.mdx.enable=true` to forward only VictoriaMetrics self-monitoring metrics to an
+external vendor while hiding sensitive label values with `-remoteWrite.obfuscateLabels`.
+
+Use `-remoteWrite.obfuscateLabels` to list label names whose values should be obfuscated for the
+corresponding `-remoteWrite.url`. Separate multiple label names with `^^`.
+
+```sh
+./vmagent \
+  -remoteWrite.url=http://<external-service1> \
+  -remoteWrite.obfuscateLabels='instance^^datacenter' \
+  -remoteWrite.url=http://<external-service2> \
+  -remoteWrite.obfuscateLabels='instance' \
+  -remoteWrite.url=http://<internal-service> \
+  -remoteWrite.obfuscateLabels='' 
+```
+
 ## Automatically generated metrics
 
 `vmagent` automatically generates the following metrics for each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)

docs/victoriametrics/vmalert.md

@@ -341,6 +341,7 @@ The following variables are available in templating:
 | $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag.                                                                                    | `Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})`                                                                                                             |
 | $externalURL or .ExternalURL       | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy.                                             | `Visit {{ $externalURL }} for more details`                                                                                                                                            |
 | $isPartial or .IsPartial           | Indicates whether the latest rule query response from the datasource(that supports returning `isPartial` option, such as vmcluster) could be partial. | `{{ if $isPartial }}WARNING: The latest alert state may be a false alarm due to a partial response from the datasource.{{ end }}`                                                      |
+| $interval or .Interval             | Alerting rule group's evaluation interval.                                                                                                            | `http://vm-grafana.com/<dashboard-id>?viewPanel=<panel-id>&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}` |
 
 Additionally, `vmalert` provides some extra templating functions listed in [template functions](#template-functions) and [reusable templates](#reusable-templates).
 

docs/victoriametrics/vmalert_common_flags.md

@@ -404,6 +404,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
      Optional TLS server name to use for connections to -remoteWrite.url. By default, the server name from -remoteWrite.url is used
   -remoteWrite.url string
      Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
+  -replay.continueWithExecutionErr bool
+     Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int
@@ -411,7 +413,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -replay.ruleEvaluationConcurrency int
      The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery. (default 1)
   -replay.ruleRetryAttempts int
-     Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
+     Defines how many retries to make before giving up on rule if request for it returns a retriable error. (default 5)
   -replay.rulesDelay duration
      Delay before evaluating the next rule within the group. Is important for chained rules. Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting. (default 1s)
   -replay.timeFrom string

lib/promscrape/client.go

@@ -90,7 +90,11 @@ func newClient(ctx context.Context, sw *ScrapeWork) (*client, error) {
 	}
 
 	tr := httputil.NewTransport(false, "vm_promscrape")
-	if proxyURLFunc != nil {
+	if sw.UnixSocket != "" {
+		// Unix sockets are direct local endpoints and cannot be reached via HTTP proxies.
+		// Proxy could be set implicitly via global env variable HTTP_PROXY
+		tr.Proxy = nil
+	} else if proxyURLFunc != nil {
 		tr.Proxy = proxyURLFunc
 	}
 	tr.TLSHandshakeTimeout = 10 * time.Second

lib/storage/storage_test.go

@@ -2843,9 +2843,8 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 	}
 	var searchTimeRange TimeRange
 
-	// Zero search time range is adjusted to globalIndexTimeRange regardless
-	// whether the -disablePerDayIndex flag is set or not.
-	searchTimeRange = TimeRange{}
+	// Search time range is the same as globalIndexTimeRange.
+	searchTimeRange = globalIndexTimeRange
 	f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
 	f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
 	f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
@@ -2853,9 +2852,6 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 
 	// The search time range is smaller than a month (and therefore < 40 days)
 	// and is fully included into the partition idb time range.
-	// If -disablePerDayIndex is set, the effective search time range is
-	// expected to be globalIndexTimeRange. Otherwise it must remain the same
-	// after the adjustment.
 	searchTimeRange = TimeRange{
 		MinTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
 		MaxTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
@@ -2866,11 +2862,6 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 	f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
 
 	// The search time range is the same as partition idb time range.
-	// If -disablePerDayIndex is set, the effective search time range is
-	// expected to be globalIndexTimeRange for both legacy and parition idb.
-	// Otherwise:
-	// - For the legacy idb: it must remain the same
-	// - For the partition idb: it must be replaced with globalIndexTimeRange.
 	searchTimeRange = partitionIDBTimeRange
 	f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
 	f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
@@ -2879,11 +2870,6 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 
 	// The search time range is smaller than 40 days and fully includes the
 	// partition idb time range.
-	// If -disablePerDayIndex is set, the effective search time range is
-	// expected to be globalIndexTimeRange for both legacy and parition idb.
-	// Otherwise:
-	// - For the legacy idb: it must remain the same
-	// - For the partition idb: it must be replaced with globalIndexTimeRange.
 	searchTimeRange = TimeRange{
 		MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
 		MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
@@ -2895,10 +2881,6 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 
 	// The search time range is 41 days and fully includes the partition idb
 	// time range.
-	// If -disablePerDayIndex is set, the effective search time range is
-	// expected to be globalIndexTimeRange for both legacy and parition idb.
-	// Otherwise it must be replaced with globalIndexTimeRange for both legacy
-	// and partition idbs.
 	searchTimeRange = TimeRange{
 		MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
 		MaxTimestamp: partitionIDBTimeRange.MinTimestamp + 41*msecPerDay,
@@ -2910,12 +2892,6 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 
 	// The search time range is smaller than 40 days and overlaps with partition
 	// idb time range on the left.
-	// If -disablePerDayIndex is set, the effective search time range is
-	// expected to be globalIndexTimeRange for both legacy and parition idb.
-	// Otherwise:
-	// - For the legacy idb: it must remain the same
-	// - For the partition idb: the MinTimestamp must be adjusted to match the
-	// partition idb time range MinTimestamp.
 	searchTimeRange = TimeRange{
 		MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
 		MaxTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
@@ -2930,12 +2906,6 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 
 	// The search time range is smaller than 40 days and overlaps with partition
 	// idb time range on the right.
-	// If -disablePerDayIndex is set, the effective search time range is
-	// expected to be globalIndexTimeRange for both legacy and parition idb.
-	// Otherwise:
-	// - For the legacy idb, it must remain the same
-	// - For the partition idb: its MaxTimestamp must be adjusted to match the
-	//   partition idb time range MaxTimestamp.
 	searchTimeRange = TimeRange{
 		MinTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
 		MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
@@ -2949,7 +2919,7 @@ func TestStorageAdjustTimeRange(t *testing.T) {
 	f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
 }
 
-type testStorageSearchWithoutPerDayIndexOptions struct {
+type testStorageSearchWithoutIndexOptions struct {
 	mrs                []MetricRow
 	assertSearchResult func(t *testing.T, s *Storage, tr TimeRange, want any)
 	alwaysPerTimeRange bool // If true, use wantPerTimeRange instead of wantAll
@@ -2958,16 +2928,15 @@ type testStorageSearchWithoutPerDayIndexOptions struct {
 	wantEmpty          any
 }
 
-// testStorageSearchWithoutPerDayIndex tests how the search behaves when the
+// testStorageSearchWithoutIndex tests how the search behaves when the
 // per-day index is disabled. This function is expected to be called by
 // functions that test a particular search operation, such as GetTSDBStatus(),
 // SearchMetricNames(), etc.
-func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWithoutPerDayIndexOptions) {
+func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutIndexOptions) {
 	defer testRemoveAll(t)
 
-	// The data is inserted and the search is performed when the per-day index
-	// is enabled.
-	t.Run("InsertAndSearchWithPerDayIndex", func(t *testing.T) {
+	// The data is inserted and the search is performed when per-day index is enabled.
+	t.Run("Add-Global-PerDay/Search-Global-PerDay", func(t *testing.T) {
 		s := MustOpenStorage(t.Name(), OpenOptions{
 			DisablePerDayIndex: false,
 		})
@@ -2979,9 +2948,8 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
 		s.MustClose()
 	})
 
-	//  The data is inserted and the search is performed when the per-day index
-	//  is disabled.
-	t.Run("InsertAndSearchWithoutPerDayIndex", func(t *testing.T) {
+	// The data is inserted and the search is performed when per-day index is disabled.
+	t.Run("Add-Global-noPerDay/Search-Global-noPerDay", func(t *testing.T) {
 		s := MustOpenStorage(t.Name(), OpenOptions{
 			DisablePerDayIndex: true,
 		})
@@ -2996,9 +2964,9 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
 		s.MustClose()
 	})
 
-	// The data is inserted when the per-day index is enabled but the search is
-	// performed when the per-day index is disabled.
-	t.Run("InsertWithPerDayIndexSearchWithout", func(t *testing.T) {
+	// The data is inserted when per-day index are enabled.
+	// The search is performed when per-day index is disabled.
+	t.Run("Add-Global-PerDay/Search-Global-noPerDay", func(t *testing.T) {
 		s := MustOpenStorage(t.Name(), OpenOptions{
 			DisablePerDayIndex: false,
 		})
@@ -3018,10 +2986,11 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
 		s.MustClose()
 	})
 
-	// The data is inserted when the per-day index is disabled but the search is
-	// performed when the per-day index is enabled. This case also shows that
-	// registering metric names recovers the per-day index.
-	t.Run("InsertWithoutPerDayIndexSearchWith", func(t *testing.T) {
+	// The data is inserted when per-day index is disabled.
+	// The search is performed when per-day index is enabled.
+	// This case also shows that registering metric names recovers the per-day
+	// index.
+	t.Run("Add-Global-noPerDay/Search-Global-PerDay", func(t *testing.T) {
 		s := MustOpenStorage(t.Name(), OpenOptions{
 			DisablePerDayIndex: true,
 		})
@@ -3048,13 +3017,13 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
 	})
 }
 
-func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
+func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
 	const (
 		days = 4
 		rows = 10
 	)
 	rng := rand.New(rand.NewSource(1))
-	opts := testStorageSearchWithoutPerDayIndexOptions{
+	opts := testStorageSearchWithoutIndexOptions{
 		wantEmpty:        &TSDBStatus{},
 		wantPerTimeRange: make(map[TimeRange]any),
 		wantAll:          &TSDBStatus{TotalSeries: days * rows},
@@ -3094,16 +3063,16 @@ func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
 		}
 	}
 
-	testStorageSearchWithoutPerDayIndex(t, &opts)
+	testStorageSearchWithoutIndex(t, &opts)
 }
 
-func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
+func TestStorageSearchMetricNamesWithoutIndex(t *testing.T) {
 	const (
 		days = 4
 		rows = 10
 	)
 	rng := rand.New(rand.NewSource(1))
-	opts := testStorageSearchWithoutPerDayIndexOptions{
+	opts := testStorageSearchWithoutIndexOptions{
 		wantEmpty:        []string{},
 		wantPerTimeRange: make(map[TimeRange]any),
 		wantAll:          []string{},
@@ -3155,16 +3124,16 @@ func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
 		}
 	}
 
-	testStorageSearchWithoutPerDayIndex(t, &opts)
+	testStorageSearchWithoutIndex(t, &opts)
 }
 
-func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
+func TestStorageSearchLabelNamesWithoutIndex(t *testing.T) {
 	const (
 		days = 4
 		rows = 10
 	)
 	rng := rand.New(rand.NewSource(1))
-	opts := testStorageSearchWithoutPerDayIndexOptions{
+	opts := testStorageSearchWithoutIndexOptions{
 		wantEmpty:        []string{},
 		wantPerTimeRange: make(map[TimeRange]any),
 		wantAll:          []string{},
@@ -3209,17 +3178,17 @@ func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
 		}
 	}
 
-	testStorageSearchWithoutPerDayIndex(t, &opts)
+	testStorageSearchWithoutIndex(t, &opts)
 }
 
-func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
+func TestStorageSearchLabelValuesWithoutIndex(t *testing.T) {
 	const (
 		days      = 4
 		rows      = 10
 		labelName = "job"
 	)
 	rng := rand.New(rand.NewSource(1))
-	opts := testStorageSearchWithoutPerDayIndexOptions{
+	opts := testStorageSearchWithoutIndexOptions{
 		wantEmpty:        []string{},
 		wantPerTimeRange: make(map[TimeRange]any),
 		wantAll:          []string{},
@@ -3263,17 +3232,17 @@ func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
 		}
 	}
 
-	testStorageSearchWithoutPerDayIndex(t, &opts)
+	testStorageSearchWithoutIndex(t, &opts)
 }
 
-func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
+func TestStorageSearchTagValueSuffixesWithoutIndex(t *testing.T) {
 	const (
 		days           = 4
 		rows           = 10
 		tagValuePrefix = "metric."
 	)
 	rng := rand.New(rand.NewSource(1))
-	opts := testStorageSearchWithoutPerDayIndexOptions{
+	opts := testStorageSearchWithoutIndexOptions{
 		wantEmpty:        []string{},
 		wantPerTimeRange: make(map[TimeRange]any),
 		wantAll:          []string{},
@@ -3313,16 +3282,16 @@ func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
 		}
 	}
 
-	testStorageSearchWithoutPerDayIndex(t, &opts)
+	testStorageSearchWithoutIndex(t, &opts)
 }
 
-func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
+func TestStorageSearchGraphitePathsWithoutIndex(t *testing.T) {
 	const (
 		days = 4
 		rows = 10
 	)
 	rng := rand.New(rand.NewSource(1))
-	opts := testStorageSearchWithoutPerDayIndexOptions{
+	opts := testStorageSearchWithoutIndexOptions{
 		wantEmpty:        []string{},
 		wantPerTimeRange: make(map[TimeRange]any),
 		wantAll:          []string{},
@@ -3363,16 +3332,16 @@ func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
 		}
 	}
 
-	testStorageSearchWithoutPerDayIndex(t, &opts)
+	testStorageSearchWithoutIndex(t, &opts)
 }
 
-func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
+func TestStorageQueryWithoutIndex(t *testing.T) {
 	const (
 		days = 4
 		rows = 10
 	)
 	rng := rand.New(rand.NewSource(1))
-	opts := testStorageSearchWithoutPerDayIndexOptions{
+	opts := testStorageSearchWithoutIndexOptions{
 		wantEmpty:          []MetricRow(nil),
 		wantPerTimeRange:   make(map[TimeRange]any),
 		alwaysPerTimeRange: true,
@@ -3413,7 +3382,7 @@ func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
 		}
 	}
 
-	testStorageSearchWithoutPerDayIndex(t, &opts)
+	testStorageSearchWithoutIndex(t, &opts)
 }
 
 func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
@@ -3455,12 +3424,12 @@ func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
 		}
 	}
 
-	t.Run("disablePerDayIndex=false", func(t *testing.T) {
-		f(t, false)
-	})
-	t.Run("disablePerDayIndex=true", func(t *testing.T) {
-		f(t, true)
-	})
+	for _, disablePerDayIndex := range []bool{false, true} {
+		name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
+		t.Run(name, func(t *testing.T) {
+			f(t, disablePerDayIndex)
+		})
+	}
 }
 
 // testSearchMetricIDs returns metricIDs for the given tfss and tr.
@@ -3497,13 +3466,13 @@ func testCountAllMetricIDs(s *Storage, tr TimeRange) int {
 	return len(ids)
 }
 
-func TestStorageRegisterMetricNamesForVariousDataPatternsConcurrently(t *testing.T) {
+func TestStorageRegisterMetricNamesVariousDataPatterns(t *testing.T) {
 	testStorageVariousDataPatternsConcurrently(t, true, func(s *Storage, mrs []MetricRow) {
 		s.RegisterMetricNames(nil, mrs)
 	})
 }
 
-func TestStorageAddRowsForVariousDataPatternsConcurrently(t *testing.T) {
+func TestStorageAddRowsVariousDataPatterns(t *testing.T) {
 	testStorageVariousDataPatternsConcurrently(t, false, func(s *Storage, mrs []MetricRow) {
 		s.AddRows(mrs, defaultPrecisionBits)
 	})
@@ -3520,27 +3489,18 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
 
 	const concurrency = 4
 
-	disablePerDayIndex := false
-	t.Run("perDayIndexes/serial", func(t *testing.T) {
-		testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
-	})
-	t.Run("perDayIndexes/concurrentRows", func(t *testing.T) {
-		testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
-	})
-	t.Run("perDayIndexes/concurrentBatches", func(t *testing.T) {
-		testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
-	})
-
-	disablePerDayIndex = true
-	t.Run("noPerDayIndexes/serial", func(t *testing.T) {
-		testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
-	})
-	t.Run("noPerDayIndexes/concurrentRows", func(t *testing.T) {
-		testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
-	})
-	t.Run("noPerDayIndexes/concurrentBatches", func(t *testing.T) {
-		testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
-	})
+	for _, disablePerDayIndex := range []bool{false, true} {
+		prefix := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
+		t.Run(prefix+"/serial", func(t *testing.T) {
+			testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
+		})
+		t.Run(prefix+"/concurrentRows", func(t *testing.T) {
+			testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
+		})
+		t.Run(prefix+"/concurrentBatches", func(t *testing.T) {
+			testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
+		})
+	}
 }
 
 // testStorageVariousDataPatterns tests the ingestion of different combinations