docs/vmanomaly: fix outdated anchors and param names (#11064)

Updated /docs/anomaly-detection/ section content for stale anchors, metrics renames, and typos in param names

PR https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11064

app/vmagent/remotewrite/remotewrite.go

@@ -79,7 +79,8 @@ var (
 		"writing them to remote storage. "+
 		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
 		"By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
-		"This option may be used for improving data compression for the stored metrics")
+		"This option may be used for improving data compression for the stored metrics. "+
+		"See also -remoteWrite.significantFigures")
 	sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
 		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+

app/vmalert/rule/group.go

@@ -95,6 +95,7 @@ type groupMetrics struct {
 	iterationTotal    *metrics.Counter
 	iterationDuration *metrics.Summary
 	iterationMissed   *metrics.Counter
+	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
 }
 
@@ -330,6 +331,7 @@ func (g *Group) Init() {
 	g.metrics.iterationTotal = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
 	g.metrics.iterationDuration = g.metrics.set.NewSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
 	g.metrics.iterationMissed = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	g.metrics.iterationReset = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_reset_total{%s}`, labels))
 	g.metrics.iterationInterval = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
 		i := g.Interval.Seconds()
 		return i
@@ -474,14 +476,16 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 			if missed < 0 {
 				// missed can become < 0 due to irregular delays during evaluation
 				// which can result in time.Since(evalTS) < g.Interval;
-				// or the system wall clock was changed backward
-				missed = 0
+				// or the system wall clock was changed backward,
+				// Reset the evalTS to the current time.
 				evalTS = time.Now()
+				g.metrics.iterationReset.Inc()
+			} else {
+				evalTS = evalTS.Add((missed + 1) * g.Interval)
 			}
 			if missed > 0 {
 				g.metrics.iterationMissed.Inc()
 			}
-			evalTS = evalTS.Add((missed + 1) * g.Interval)
 
 			eval(evalCtx, evalTS)
 		}

app/vmalert/web.go

@@ -11,6 +11,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
@@ -160,12 +162,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
-		gf, err := newGroupsFilter(r)
+		af, err := newAlertsFilter(r)
 		if err != nil {
 			errJson(w, r, err)
 			return true
 		}
-		data, err := rh.listAlerts(gf)
+		data, err := rh.listAlerts(af)
 		if err != nil {
 			errJson(w, r, err)
 			return true
@@ -325,6 +327,48 @@ func (gf *groupsFilter) matches(group *rule.Group) bool {
 	return true
 }
 
+type alertsFilter struct {
+	gf    *groupsFilter
+	match [][]metricsql.LabelFilter
+}
+
+func getMatchFilters(matches []string) ([][]metricsql.LabelFilter, *httpserver.ErrorWithStatusCode) {
+	if len(matches) == 0 {
+		return nil, nil
+	}
+	tfss := make([][]metricsql.LabelFilter, 0, len(matches))
+	for _, s := range matches {
+		expr, err := metricsql.Parse(s)
+		if err != nil {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": failed to parse %q: %w`, s, err), http.StatusBadRequest)
+		}
+		me, ok := expr.(*metricsql.MetricExpr)
+		if !ok {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": expecting metricSelector; got %q`, expr.AppendString(nil)), http.StatusBadRequest)
+		}
+		if len(me.LabelFilterss) == 0 {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": labelFilterss cannot be empty`), http.StatusBadRequest)
+		}
+		tfss = append(tfss, me.LabelFilterss...)
+	}
+	return tfss, nil
+}
+
+func newAlertsFilter(r *http.Request) (*alertsFilter, *httpserver.ErrorWithStatusCode) {
+	gf, err := newGroupsFilter(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var af alertsFilter
+	af.gf = gf
+	af.match, err = getMatchFilters(r.Form["match[]"])
+	if err != nil {
+		return nil, err
+	}
+	return &af, nil
+}
+
 // see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
 type rulesFilter struct {
 	gf             *groupsFilter
@@ -335,6 +379,7 @@ type rulesFilter struct {
 	maxGroups      int
 	pageNum        int
 	search         string
+	match          [][]metricsql.LabelFilter
 	extendedStates bool
 }
 
@@ -355,7 +400,10 @@ func newRulesFilter(r *http.Request) (*rulesFilter, *httpserver.ErrorWithStatusC
 			return nil, errResponse(fmt.Errorf(`invalid parameter "type": not supported value %q`, ruleTypeParam), http.StatusBadRequest)
 		}
 	}
-
+	rf.match, err = getMatchFilters(r.Form["match[]"])
+	if err != nil {
+		return nil, err
+	}
 	states := vs["state"]
 	if len(states) == 0 {
 		states = vs["filter"]
@@ -416,12 +464,47 @@ func (rf *rulesFilter) matchesRule(r *rule.ApiRule) bool {
 	if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
 		return false
 	}
+	if !areLabelsMatch(r.Labels, rf.match) {
+		return false
+	}
 	if len(rf.states) == 0 {
 		return true
 	}
 	return slices.Contains(rf.states, r.State)
 }
 
+func areLabelsMatch(labels map[string]string, matches [][]metricsql.LabelFilter) bool {
+	if len(matches) == 0 {
+		return true
+	}
+	// labels need to match at least one of the provided match[] arg
+	return slices.ContainsFunc(matches, func(filters []metricsql.LabelFilter) bool {
+		for _, mf := range filters {
+			if !isLabelFilterMatch(labels[mf.Label], mf) {
+				return false
+			}
+		}
+		return true
+	})
+}
+
+func isLabelFilterMatch(s string, match metricsql.LabelFilter) bool {
+	if !match.IsRegexp {
+		if match.IsNegative {
+			return s != match.Value
+		}
+		return s == match.Value
+	}
+	re, err := metricsql.CompileRegexpAnchored(match.Value)
+	if err != nil {
+		return false
+	}
+	if match.IsNegative {
+		return !re.MatchString(s)
+	}
+	return re.MatchString(s)
+}
+
 func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
@@ -543,14 +626,14 @@ func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	return gAlerts
 }
 
-func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
+func (rh *requestHandler) listAlerts(af *alertsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
 	lr := listAlertsResponse{Status: "success"}
 	lr.Data.Alerts = make([]*rule.ApiAlert, 0)
 	for _, group := range rh.m.groups {
-		if !gf.matches(group) {
+		if !af.gf.matches(group) {
 			continue
 		}
 		g := group.ToAPI()
@@ -558,7 +641,11 @@ func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.Erro
 			if r.Type != rule.TypeAlerting {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, r.Alerts...)
+			for _, alert := range r.Alerts {
+				if areLabelsMatch(alert.Labels, af.match) {
+					lr.Data.Alerts = append(lr.Data.Alerts, alert)
+				}
+			}
 		}
 	}
 

app/vmalert/web.qtpl

@@ -348,7 +348,7 @@
                 typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
                 count := len(ns)
             %}
-            <div class="w-100 flex-column vm-group">
+            <div class="w-100 flex-column">
                 <span class="d-flex justify-content-between" id="group-{%s typeK %}">
                     <a href="#group-{%s typeK %}">{%s typeK %} ({%d count %})</a>
                     <span
@@ -361,7 +361,7 @@
                 <div id="item-{%s typeK %}" class="collapse show">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
-                            <tr class="vm-item">
+                            <tr>
                                 <th scope="col">Labels</th>
                                 <th scope="col">Address</th>
                             </tr>

app/vmalert/web.qtpl.go

@@ -1115,7 +1115,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
 
 //line app/vmalert/web.qtpl:350
 			qw422016.N().S(`
-            <div class="w-100 flex-column vm-group">
+            <div class="w-100 flex-column">
                 <span class="d-flex justify-content-between" id="group-`)
 //line app/vmalert/web.qtpl:352
 			qw422016.E().S(typeK)
@@ -1152,7 +1152,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
 			qw422016.N().S(`" class="collapse show">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
-                            <tr class="vm-item">
+                            <tr>
                                 <th scope="col">Labels</th>
                                 <th scope="col">Address</th>
                             </tr>

app/vmalert/web_test.go

@@ -10,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -37,12 +39,14 @@ func TestHandler(t *testing.T) {
 			Concurrency: 1,
 			Rules: []config.Rule{
 				{
-					ID:    0,
-					Alert: "alert",
+					ID:     0,
+					Alert:  "alert",
+					Labels: map[string]string{"job": "foo"},
 				},
 				{
 					ID:     1,
 					Record: "record",
+					Labels: map[string]string{"job": "bar"},
 				},
 			},
 		}, fq, 1*time.Minute, nil)
@@ -128,6 +132,18 @@ func TestHandler(t *testing.T) {
 		if length := len(lr.Data.Alerts); length != 2 {
 			t.Fatalf("expected 2 alert got %d", length)
 		}
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+`/api/v1/alerts?match[]={job="foo"}`, &lr, 200)
+		if length := len(lr.Data.Alerts); length != 3 {
+			t.Fatalf("expected 3 alerts got %d", length)
+		}
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+`/api/v1/alerts?match[]={job="bar"}`, &lr, 200)
+		if length := len(lr.Data.Alerts); length != 0 {
+			t.Fatalf("expected 0 alerts got %d", length)
+		}
 	})
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
 		expAlert := rule.NewAlertAPI(ar, ar.GetAlerts()[0])
@@ -242,6 +258,13 @@ func TestHandler(t *testing.T) {
 		check("/vmalert/api/v1/rules?datasource_type=graphite", 200, 1, 2)
 		check("/vmalert/api/v1/rules?datasource_type=graphiti", 400, 0, 0)
 
+		// invalid match[] params
+		check(`/vmalert/api/v1/rules?match[]={job=!"foo"}`, 400, 0, 0)
+		check(`/vmalert/api/v1/rules?match[]={job="foo"}`, 200, 3, 3)
+		check(`/vmalert/api/v1/rules?match[]={job="bar"}`, 200, 3, 3)
+		check(`/vmalert/api/v1/rules?match[]={job="bar"}&match[]={job="foo"}`, 200, 3, 6)
+		check(`/vmalert/api/v1/rules?match[]={job="barzz"}`, 200, 0, 0)
+
 		// no filtering expected due to bad params
 		check("/api/v1/rules?type=badParam", 400, 0, 0)
 		check("/api/v1/rules?foo=bar", 200, 3, 6)
@@ -367,3 +390,116 @@ func TestEmptyResponse(t *testing.T) {
 		}
 	})
 }
+
+func TestMatchesRule(t *testing.T) {
+	parseMatch := func(t *testing.T, selectors []string) [][]metricsql.LabelFilter {
+		t.Helper()
+		var match [][]metricsql.LabelFilter
+		for _, s := range selectors {
+			expr, err := metricsql.Parse(s)
+			if err != nil {
+				t.Fatalf("failed to parse selector %q: %v", s, err)
+			}
+			me, ok := expr.(*metricsql.MetricExpr)
+			if !ok {
+				t.Fatalf("expected MetricExpr for %q, got %T", s, expr)
+			}
+			match = append(match, me.LabelFilterss...)
+		}
+		return match
+	}
+
+	f := func(t *testing.T, selectors []string, labels map[string]string, wantMatch bool) {
+		t.Helper()
+		rf := &rulesFilter{
+			gf:    &groupsFilter{},
+			match: parseMatch(t, selectors),
+		}
+		r := &rule.ApiRule{Labels: labels}
+		got := rf.matchesRule(r)
+		if got != wantMatch {
+			t.Fatalf("matchesRule(%v) with selectors %v: got %v, want %v",
+				labels, selectors, got, wantMatch)
+		}
+	}
+
+	f(t, nil, map[string]string{"foo": "bar"}, true)
+
+	f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "bar"}, true)
+	f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "baz"}, false)
+
+	f(t, []string{`{foo="bar"}`}, map[string]string{"bar": "baz"}, false)
+	f(t, []string{`{foo=""}`}, map[string]string{"bar": "baz"}, true)
+
+	f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "baz"}, true)
+	f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "bar"}, false)
+
+	f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "bar"}, true)
+	f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "baz"}, false)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "baz"}, true)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "bar"}, true)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "foo"}, false)
+
+	f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "baz"}, true)
+	f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "bar"}, false)
+
+	// single match[] with multiple filters
+	f(t,
+		[]string{`{job="foo",instance="bar"}`},
+		map[string]string{"job": "foo", "instance": "bar"},
+		true,
+	)
+	f(t,
+		[]string{`{job="foo",instance="bar"}`},
+		map[string]string{"job": "other", "instance": "bar"},
+		false,
+	)
+
+	f(t,
+		[]string{`{foo="bar",baz=~"b.*"}`},
+		map[string]string{"foo": "bar", "baz": "bazinga"},
+		true,
+	)
+	f(t,
+		[]string{`{foo="bar",baz=~"b.*"}`},
+		map[string]string{"foo": "other", "baz": "bazinga"},
+		false,
+	)
+
+	// multiple matches[]
+	f(t,
+		[]string{`{foo="bar"}`, `{foo="baz"}`},
+		map[string]string{"foo": "baz"},
+		true,
+	)
+	f(t,
+		[]string{`{foo="bar"}`, `{foo="baz"}`},
+		map[string]string{"foo": "unknown"},
+		false,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"bar": "bazinga"},
+		true,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"foo": "bartender"},
+		true,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"foo": "other", "bar": "other"},
+		false,
+	)
+	f(t,
+		[]string{`{job="foo",instance="bar"}`, `{foo="bar"}`},
+		map[string]string{"foo": "bar"},
+		true,
+	)
+	f(t,
+		[]string{`{job="foo", instance="bar"}`, `{foo="bar"}`},
+		map[string]string{"instance": "barr", "job": "foo"},
+		false,
+	)
+}

app/vmauth/auth_config.go

@@ -889,7 +889,8 @@ func reloadAuthConfig() (bool, error) {
 	}
 
 	mp := authUsers.Load()
-	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
+	jwtc := jwtAuthCache.Load()
+	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp)+len(jwtc.users), *authConfigPath)
 	return true, nil
 }
 

app/vmauth/main.go

@@ -317,7 +317,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tk
 	defer ui.endConcurrencyLimit()
 
 	// Process the request.
-	processRequest(w, r, ui, tkn)
+	processRequest(w, r, ui, tkn, userName)
 }
 
 func beginConcurrencyLimit(ctx context.Context) error {
@@ -391,7 +391,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 	return bb, nil
 }
 
-func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
+func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token, userName string) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
 	isDefault := false
@@ -409,7 +409,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 			if ui.DumpRequestOnErrors {
 				di = debugInfo(u, r)
 			}
-			httpserver.Errorf(w, r, "missing route for %q%s", u.String(), di)
+			httpserver.Errorf(w, r, "user %s missing route for %q%s", userName, u.String(), di)
 			return
 		}
 		up, hc = ui.DefaultURL, ui.HeadersConf
@@ -455,7 +455,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), ui.name()),
+		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), userName),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)

app/vmauth/main_test.go

@@ -307,6 +307,24 @@ statusCode=200
 requested_url={BACKEND}/bar/a/b`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
+	// correct authorization but unexisted path, hence missing route error.
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_map:
+  - src_paths:
+    - "/api/v1/write"
+    url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:secret@some-host.com/a/b"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=400
+user foo missing route for "http://foo:secret@some-host.com/a/b"`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
 	// verify how path cleanup works
 	cfgStr = `
 unauthorized_user:
@@ -403,7 +421,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=400
-missing route for "http://some-host.com/abc?de=fg"`
+user unauthorized missing route for "http://some-host.com/abc?de=fg"`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
 	// missing default_url and default url_prefix for unauthorized user with dump_request_on_errors enabled
@@ -419,7 +437,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=400
-missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
+user unauthorized missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
 Pass-Header: abc
 Some-Header: foobar
 X-Forwarded-For: 12.34.56.78
@@ -461,7 +479,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
 	// all the backend_urls are unavailable for authorized user
@@ -501,7 +519,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 0 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 0 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	netutil.Resolver = origResolver
 
@@ -518,7 +536,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	if n := retries.Load(); n != 2 {
 		t.Fatalf("unexpected number of retries; got %d; want 2", n)
@@ -545,6 +563,31 @@ requested_url={BACKEND}/path2/foo/?de=fg`
 	if n := retries.Load(); n != 2 {
 		t.Fatalf("unexpected number of retries; got %d; want 2", n)
 	}
+
+	// make sure that empty config value erases client extra filters and extra labels
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz&extra_filters[]=&extra_label=&extra_filters=`
+	requestURL = "http://some-host.com/abc/def?some_arg=some_value&extra_filters[]=baz&extra_label=tenant=admin&extra_filters=bar"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		h := w.Header()
+		h.Set("Connection", "close")
+		h.Set("Foo", "bar")
+
+		var bb bytes.Buffer
+		if err := r.Header.Write(&bb); err != nil {
+			panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
+		}
+		fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
+	}
+	responseExpected = `
+statusCode=200
+Foo: bar
+requested_url={BACKEND}/foo/abc/def?bar=baz&extra_filters=&extra_filters%5B%5D=&extra_label=&some_arg=some_value
+Pass-Header: abc
+User-Agent: vmauth
+X-Forwarded-For: 12.34.56.78, 42.2.3.84`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
 }
 
 func TestJWTRequestHandler(t *testing.T) {

app/vmctl/flags.go

@@ -146,7 +146,8 @@ var (
 			Name:  vmRoundDigits,
 			Value: 100,
 			Usage: "Round metric values to the given number of decimal digits after the point. " +
-				"This option may be used for increasing on-disk compression level for the stored metrics",
+				"This option may be used for increasing on-disk compression level for the stored metrics. " +
+				"See also --vm-significant-figures option",
 		},
 		&cli.StringSliceFlag{
 			Name:  vmExtraLabel,
@@ -500,6 +501,96 @@ var (
 	}
 )
 
+const (
+	mimirPath             = "mimir-path"
+	mimirTenantID         = "mimir-tenant-id"
+	mimirConcurrency      = "mimir-concurrency"
+	mimirFilterTimeStart  = "mimir-filter-time-start"
+	mimirFilterTimeEnd    = "mimir-filter-time-end"
+	mimirFilterLabel      = "mimir-filter-label"
+	mimirFilterLabelValue = "mimir-filter-label-value"
+
+	mimirCredsFilePath           = "mimir-creds-file-path"
+	mimirConfigFilePath          = "mimir-config-file-path"
+	mimirConfigProfile           = "mimir-config-profile"
+	mimirCustomS3Endpoint        = "mimir-custom-s3-endpoint"
+	mimirS3ForcePathStyle        = "mimir-s3-force-path-style"
+	mimirS3TLSInsecureSkipVerify = "mimir-s3-tls-insecure-skip-verify"
+	mimirSSEKMSKeyID             = "mimir-s3-sse-kms-key-id"
+	mimirSSEAlgorithm            = "mimir-s3-sse-algorithm"
+)
+
+var (
+	mimirFlags = []cli.Flag{
+		&cli.StringFlag{
+			Name:     mimirPath,
+			Usage:    "Path to Mimir storage bucket or local folder.",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:  mimirTenantID,
+			Usage: "Tenant ID for Mimir storage",
+		},
+		&cli.IntFlag{
+			Name:  mimirConcurrency,
+			Usage: "Number of concurrently running block readers",
+			Value: 1,
+		},
+		&cli.StringFlag{
+			Name:     mimirFilterTimeStart,
+			Usage:    "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     mimirFilterTimeEnd,
+			Usage:    "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:  mimirFilterLabel,
+			Usage: "Mimir label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
+		},
+		&cli.StringFlag{
+			Name:  mimirFilterLabelValue,
+			Usage: fmt.Sprintf("Regular expression to filter label from %q flag.", mimirFilterLabel),
+			Value: ".*",
+		},
+		&cli.StringFlag{
+			Name:  mimirCredsFilePath,
+			Usage: "Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
+		},
+		&cli.StringFlag{
+			Name:  mimirConfigFilePath,
+			Usage: "Path to file with S3 configs. Configs are loaded from default location if not set. See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
+		},
+		&cli.StringFlag{
+			Name:  mimirConfigProfile,
+			Usage: "Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used",
+		},
+		&cli.StringFlag{
+			Name:  mimirCustomS3Endpoint,
+			Usage: "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set",
+		},
+		&cli.BoolFlag{
+			Name:  mimirS3ForcePathStyle,
+			Usage: "Prefixing endpoint with bucket name when set false, true by default.",
+			Value: true,
+		},
+		&cli.BoolFlag{
+			Name:  mimirS3TLSInsecureSkipVerify,
+			Usage: "Whether to skip TLS verification when connecting to the S3 endpoint.",
+		},
+		&cli.StringFlag{
+			Name:  mimirSSEKMSKeyID,
+			Usage: "SSE KMS Key ID for use with S3-compatible storages.",
+		},
+		&cli.StringFlag{
+			Name:  mimirSSEAlgorithm,
+			Usage: "SSE algorithm for use with S3-compatible storages.",
+		},
+	}
+)
+
 const (
 	vmNativeFilterMatch       = "vm-native-filter-match"
 	vmNativeFilterTimeStart   = "vm-native-filter-time-start"

app/vmctl/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/mimir"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -297,12 +298,12 @@ func main() {
 				},
 			},
 			{
-				Name:   "thanos",
-				Usage:  "Migrate time series from Thanos blocks (supports raw and downsampled data)",
-				Flags:  mergeFlags(globalFlags, thanosFlags, vmFlags),
+				Name:   "mimir",
+				Usage:  "Migrate time series from Mimir object storage or local filesystem",
+				Flags:  mergeFlags(globalFlags, mimirFlags, vmFlags),
 				Before: beforeFn,
 				Action: func(c *cli.Context) error {
-					fmt.Println("Thanos import mode")
+					fmt.Println("Mimir import mode")
 
 					vmCfg, err := initConfigVM(c)
 					if err != nil {
@@ -314,6 +315,54 @@ func main() {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
 
+					mCfg := mimir.Config{
+						Filter: mimir.Filter{
+							TimeMin:    c.String(mimirFilterTimeStart),
+							TimeMax:    c.String(mimirFilterTimeEnd),
+							Label:      c.String(mimirFilterLabel),
+							LabelValue: c.String(mimirFilterLabelValue),
+						},
+						Path:                    c.String(mimirPath),
+						TenantID:                c.String(mimirTenantID),
+						CredsFilePath:           c.String(mimirCredsFilePath),
+						ConfigFilePath:          c.String(mimirConfigFilePath),
+						ConfigProfile:           c.String(mimirConfigProfile),
+						CustomS3Endpoint:        c.String(mimirCustomS3Endpoint),
+						S3ForcePathStyle:        c.Bool(mimirS3ForcePathStyle),
+						S3TLSInsecureSkipVerify: c.Bool(mimirS3TLSInsecureSkipVerify),
+						SSEKMSKeyID:             c.String(mimirSSEKMSKeyID),
+						SSEAlgorithm:            c.String(mimirSSEAlgorithm),
+					}
+					cl, err := mimir.NewClient(ctx, mCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create mimir client: %s", err)
+					}
+
+					pp := prometheusProcessor{
+						cl:        cl,
+						im:        importer,
+						cc:        c.Int(mimirConcurrency),
+						isVerbose: c.Bool(globalVerbose),
+					}
+					return pp.run(ctx)
+				},
+			},
+			{
+				Name:   "thanos",
+				Usage:  "Migrate time series from Thanos blocks (supports raw and downsampled data)",
+				Flags:  mergeFlags(globalFlags, thanosFlags, vmFlags),
+				Before: beforeFn,
+				Action: func(c *cli.Context) error {
+					fmt.Println("Thanos import mode")
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
+
+					importer, err = vm.NewImporter(ctx, vmCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create VM importer: %s", err)
+					}
 					thanosCfg := thanos.Config{
 						Snapshot: c.String(thanosSnapshot),
 						Filter: thanos.Filter{

app/vmctl/mimir/lazyreader.go

@@ -0,0 +1,195 @@
+package mimir
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/oklog/ulid/v2"
+	"github.com/prometheus/prometheus/tsdb"
+	"github.com/prometheus/prometheus/tsdb/tombstones"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+var _ tsdb.BlockReader = (*lazyBlockReader)(nil)
+
+// lazyBlockReader is stores block id and segment num information.
+// It is used to lazily fetch and parse block data.
+// It implements tsdb.BlockReader interface.
+type lazyBlockReader struct {
+	// Block ID.
+	ID ulid.ULID
+	// SegmentsNum stores the number of chunks segments in the block.
+	SegmentsNum int
+
+	mu          sync.Mutex
+	reader      *tsdb.Block
+	tempDirPath string
+	fs          common.RemoteFS
+	err         error
+}
+
+// newLazyBlockReader returns a new LazyBlockReader for the given block.
+func newLazyBlockReader(block *Block, fs common.RemoteFS) (*lazyBlockReader, error) {
+	if block.SegmentsFormat != "1b6d" {
+		return nil, fmt.Errorf("unsupported segments format: %s", block.SegmentsFormat)
+	}
+
+	return &lazyBlockReader{
+		ID:          block.ID,
+		SegmentsNum: block.SegmentsNum,
+		fs:          fs,
+	}, nil
+}
+
+func (lbr *lazyBlockReader) initialize() error {
+	lbr.mu.Lock()
+	defer lbr.mu.Unlock()
+	if lbr.reader != nil {
+		return nil
+	}
+	// fetching block and parse it and store it in lbr.reader
+	temp, err := lbr.mkTempDir()
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %s", err)
+	}
+
+	lbr.tempDirPath = temp
+
+	// TODO: replace fetchFile and writeFile with buffered IO if needed
+	meta, err := lbr.fetchFile(metaFilename)
+	if err != nil {
+		return err
+	}
+	if err := lbr.writeFile(temp, metaFilename, meta); err != nil {
+		return fmt.Errorf("failed to write meta file: %w", err)
+	}
+	idx, err := lbr.fetchFile(indexFilename)
+	if err != nil {
+		return fmt.Errorf("failed to fetch index file %q: %w", indexFilename, err)
+	}
+	if err := lbr.writeFile(temp, indexFilename, idx); err != nil {
+		return err
+	}
+
+	for i := 1; i <= lbr.SegmentsNum; i++ {
+		// segments formats has format 1b06d
+		// https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L32
+		chunkName := fmt.Sprintf("%06d", i)
+		blockChunkPath := filepath.Join("chunks", chunkName)
+		chunk, err := lbr.fetchFile(blockChunkPath)
+		if err != nil {
+			return fmt.Errorf("failed to fetch chunk file: %q: %w", chunkName, err)
+		}
+		if err := lbr.writeFile(temp, blockChunkPath, chunk); err != nil {
+			return fmt.Errorf("failed to write chunk file: %q: %s", chunkName, err)
+		}
+	}
+
+	// Set postingDecoder to nil because
+	// If it is nil then a default decoder is used, compatible with Prometheus v2.
+	pb, err := tsdb.OpenBlock(nil, temp, nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to open block %q: %w", lbr.ID, err)
+	}
+	lbr.reader = pb
+	return nil
+}
+
+// Index returns an IndexReader over the block's data.
+func (lbr *lazyBlockReader) Index() (tsdb.IndexReader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Index()
+}
+
+// Chunks returns a ChunkReader over the block's data.
+func (lbr *lazyBlockReader) Chunks() (tsdb.ChunkReader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Chunks()
+}
+
+// Tombstones returns a tombstones.Reader over the block's deleted data.
+func (lbr *lazyBlockReader) Tombstones() (tombstones.Reader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Tombstones()
+}
+
+// Meta provides meta information about the block reader.
+func (lbr *lazyBlockReader) Meta() tsdb.BlockMeta {
+	if err := lbr.initialize(); err != nil {
+		lbr.err = fmt.Errorf("cannot get BlockMeta: %w", err)
+		return tsdb.BlockMeta{}
+	}
+	return lbr.reader.Meta()
+}
+
+// Size returns the number of bytes that the block takes up on disk.
+func (lbr *lazyBlockReader) Size() int64 {
+	if err := lbr.initialize(); err != nil {
+		lbr.err = fmt.Errorf("error get Size of the block: %s, return zero size", err)
+		return 0
+	}
+	return lbr.reader.Size()
+}
+
+// Err returns the last error that occurred on the block reader.
+func (lbr *lazyBlockReader) Err() error {
+	return lbr.err
+}
+
+// Close closes block and releases all resources
+func (lbr *lazyBlockReader) Close() error {
+	lbr.mu.Lock()
+	defer lbr.mu.Unlock()
+	if lbr.reader == nil {
+		return nil
+	}
+
+	err := lbr.reader.Close()
+	if err := os.RemoveAll(lbr.tempDirPath); err != nil {
+		log.Printf("failed to remove temp dir: %s", err)
+	}
+	lbr.reader = nil
+	lbr.tempDirPath = ""
+
+	return err
+}
+
+func (lbr *lazyBlockReader) mkTempDir() (string, error) {
+	temp, err := os.MkdirTemp("", lbr.ID.String())
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp dir: %s", err)
+	}
+	err = os.Mkdir(filepath.Join(temp, "chunks"), os.ModePerm)
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp dir: %s", err)
+	}
+	return temp, nil
+}
+
+func (lbr *lazyBlockReader) fetchFile(filePath string) ([]byte, error) {
+	blockID := lbr.ID.String()
+	blockPath := filepath.Join(blockID, filePath)
+	has, err := lbr.fs.HasFile(blockPath)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, fmt.Errorf("block meta %s not found", blockID)
+	}
+	return lbr.fs.ReadFile(blockPath)
+}
+
+func (lbr *lazyBlockReader) writeFile(folder string, filename string, file []byte) error {
+	fileName := filepath.Join(folder, filename)
+	return os.WriteFile(fileName, file, os.ModePerm)
+}

app/vmctl/mimir/mimir.go

@@ -0,0 +1,238 @@
+package mimir
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/oklog/ulid/v2"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/tsdb"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
+	utils "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+const (
+	bucketIndex                   = "bucket-index.json"
+	bucketIndexCompressedFilename = bucketIndex + ".gz"
+	metaFilename                  = "meta.json"
+	indexFilename                 = "index"
+)
+
+// BlockDeletionMark holds the information about a block's deletion mark in the index.
+// This type was copied from the mimir repository https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L234.
+type BlockDeletionMark struct {
+	// Block ID.
+	ID ulid.ULID `json:"block_id"`
+
+	// DeletionTime is a unix timestamp (seconds precision) of when the block was marked to be deleted.
+	DeletionTime int64 `json:"deletion_time"`
+}
+
+// Block holds the information about a block in the index.
+// This is a partial implementation of the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L73
+type Block struct {
+	// Block ID.
+	ID ulid.ULID `json:"block_id"`
+
+	// MinTime and MaxTime specify the time range all samples in the block are in (millis precision).
+	MinTime int64 `json:"min_time"`
+	MaxTime int64 `json:"max_time"`
+
+	// SegmentsFormat and SegmentsNum stores the format and number of chunks segments
+	// in the block.
+	SegmentsFormat string `json:"segments_format,omitempty"`
+	SegmentsNum    int    `json:"segments_num,omitempty"`
+}
+
+// Index contains all known blocks and markers of a tenant.
+// This is a partial implementation pof the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L36
+type Index struct {
+	// Version of the index format.
+	Version int `json:"version"`
+
+	// List of complete blocks (partial blocks are excluded from the index).
+	Blocks []*Block `json:"blocks"`
+}
+
+// Config contains a list of params needed
+// for reading mimir snapshots
+type Config struct {
+	// Path to remote storage bucket
+	Path string
+	// TenantID is the tenant id for the storage
+	TenantID string
+
+	Filter Filter
+
+	CredsFilePath           string
+	ConfigFilePath          string
+	ConfigProfile           string
+	CustomS3Endpoint        string
+	S3ForcePathStyle        bool
+	S3TLSInsecureSkipVerify bool
+
+	SSEKMSKeyID  string
+	SSEAlgorithm string
+}
+
+// Filter contains configuration for filtering
+// the timeseries
+type Filter struct {
+	TimeMin    string
+	TimeMax    string
+	Label      string
+	LabelValue string
+}
+
+// Client is a wrapper over Prometheus tsdb.DBReader
+type Client struct {
+	common.RemoteFS
+	filter filter
+}
+
+type filter struct {
+	min, max   int64
+	label      string
+	labelValue string
+}
+
+func (f filter) inRange(minTime, maxTime int64) bool {
+	fmin, fmax := f.min, f.max
+	if minTime == 0 {
+		fmin = minTime
+	}
+	if fmax == 0 {
+		fmax = maxTime
+	}
+	return minTime <= fmax && fmin <= maxTime
+}
+
+// NewClient creates and validates new Client
+// with given Config
+func NewClient(ctx context.Context, cfg Config) (*Client, error) {
+	if cfg.Path == "" {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+
+	if cfg.TenantID != "" {
+		cfg.Path = fmt.Sprintf("%s/%s", cfg.Path, cfg.TenantID)
+	}
+
+	var c Client
+	rfs, err := newRemoteFS(ctx, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-src`=%q: %w", cfg.Path, err)
+	}
+
+	c.RemoteFS = rfs
+	timeMin, err := utils.ParseTime(cfg.Filter.TimeMin)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse min time in filter: %s", err)
+	}
+	timeMax, err := utils.ParseTime(cfg.Filter.TimeMax)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse max time in filter: %s", err)
+	}
+	c.filter = filter{
+		min:        timeMin.UnixMilli(),
+		max:        timeMax.UnixMilli(),
+		label:      cfg.Filter.Label,
+		labelValue: cfg.Filter.LabelValue,
+	}
+	return &c, nil
+}
+
+// Explore a fetches bucket-index.json file from a remote storage or local filesystem
+// and filter blocks via the defined time range, but does not take into account label filters.
+func (c *Client) Explore() ([]tsdb.BlockReader, error) {
+
+	log.Printf("Fetching blocks from remote storage")
+
+	indexFile, err := c.fetchIndexFile()
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch index file: %s", err)
+	}
+
+	var blocksToImport []tsdb.BlockReader
+	for _, block := range indexFile.Blocks {
+		if !c.filter.inRange(block.MinTime, block.MaxTime) {
+			// Skipping block outside of time range
+			continue
+		}
+
+		if block.ID.String() == "" {
+			continue
+		}
+
+		lazyBlockReader, err := newLazyBlockReader(block, c.RemoteFS)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create lazy block reader: %s", err)
+		}
+		blocksToImport = append(blocksToImport, lazyBlockReader)
+	}
+
+	return blocksToImport, nil
+}
+
+// Read reads the given BlockReader according to configured
+// time and label filters.
+func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error) {
+	meta := block.Meta()
+	if b, ok := block.(*lazyBlockReader); ok && b.Err() != nil {
+		return nil, fmt.Errorf("failed to read block: %s", b.Err())
+	}
+
+	if meta.ULID.String() == "" {
+		return nil, fmt.Errorf("unexpected block without id")
+	}
+
+	minTime, maxTime := meta.MinTime, meta.MaxTime
+	if c.filter.min != 0 {
+		minTime = c.filter.min
+	}
+	if c.filter.max != 0 {
+		maxTime = c.filter.max
+	}
+	q, err := tsdb.NewBlockQuerier(block, minTime, maxTime)
+	if err != nil {
+		return nil, err
+	}
+	ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
+	return &prometheus.CloseableSeriesSet{SeriesSet: ss, Close: q.Close}, nil
+}
+
+func (c *Client) fetchIndexFile() (*Index, error) {
+	has, err := c.HasFile(bucketIndexCompressedFilename)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, fmt.Errorf("bucket-index.json.gz not found")
+	}
+
+	file, err := c.ReadFile(bucketIndexCompressedFilename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read bucket index: %s", err)
+	}
+
+	r := bytes.NewReader(file)
+	// Read all the content.
+	gzipReader, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create gzip reader: %s", err)
+	}
+
+	var indexFile Index
+	err = json.NewDecoder(gzipReader).Decode(&indexFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode bucket index: %s", err)
+	}
+
+	return &indexFile, nil
+}

app/vmctl/mimir/remotefs.go

@@ -0,0 +1,93 @@
+package mimir
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/azremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
+)
+
+// newRemoteFS returns new remote fs from the given Config.
+func newRemoteFS(ctx context.Context, cfg Config) (common.RemoteFS, error) {
+	if len(cfg.Path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	n := strings.Index(cfg.Path, "://")
+	if n < 0 {
+		return nil, fmt.Errorf("missing scheme in path %q. Supported schemes: `gs://`, `s3://`, `azblob://`, `fs://`", cfg.Path)
+	}
+	scheme := cfg.Path[:n]
+	dir := cfg.Path[n+len("://"):]
+	switch scheme {
+	case "fs":
+		if !filepath.IsAbs(dir) {
+			return nil, fmt.Errorf("dir must be absolute; got %q", dir)
+		}
+		fsr := &fsremote.FS{
+			Dir: filepath.Clean(dir),
+		}
+		return fsr, nil
+	case "gcs", "gs":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &gcsremote.FS{
+			CredsFilePath: cfg.CredsFilePath,
+			Bucket:        bucket,
+			Dir:           dir,
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to gcs: %w", err)
+		}
+		return fsr, nil
+	case "azblob":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the AZBlob container %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &azremote.FS{
+			Container: bucket,
+			Dir:       dir,
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to AZBlob: %w", err)
+		}
+		return fsr, nil
+	case "s3":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the s3 bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &s3remote.FS{
+			CredsFilePath:         cfg.CredsFilePath,
+			ConfigFilePath:        cfg.ConfigFilePath,
+			CustomEndpoint:        cfg.CustomS3Endpoint,
+			TLSInsecureSkipVerify: cfg.S3TLSInsecureSkipVerify,
+			S3ForcePathStyle:      cfg.S3ForcePathStyle,
+			ProfileName:           cfg.ConfigProfile,
+			Bucket:                bucket,
+			Dir:                   dir,
+			SSEKMSKeyId:           cfg.SSEKMSKeyID,
+			SSEAlgorithm:          s3remote.StringToEncryptionAlgorithm(cfg.SSEAlgorithm),
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to s3: %w", err)
+		}
+		return fsr, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q", scheme)
+	}
+}

app/vmctl/prometheus.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
 	"strings"
 	"sync"
@@ -18,10 +19,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 )
 
+// Runner is an interface for fetching and reading
+// snapshot blocks
+type Runner interface {
+	Explore() ([]tsdb.BlockReader, error)
+	Read(context.Context, tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error)
+}
+
 type prometheusProcessor struct {
-	// prometheus client fetches and reads
+	// Runner fetches and reads
 	// snapshot blocks
-	cl *prometheus.Client
+	cl Runner
 	// importer performs import requests
 	// for timeseries data returned from
 	// snapshot blocks
@@ -48,7 +56,7 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
-	if err := pp.processBlocks(blocks); err != nil {
+	if err := pp.processBlocks(ctx, blocks); err != nil {
 		return fmt.Errorf("migration failed: %s", err)
 	}
 
@@ -57,11 +65,17 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
 	return nil
 }
 
-func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
-	ss, err := pp.cl.Read(b)
+func (pp *prometheusProcessor) do(ctx context.Context, b tsdb.BlockReader) error {
+	css, err := pp.cl.Read(ctx, b)
 	if err != nil {
 		return fmt.Errorf("failed to read block: %s", err)
 	}
+	defer func() {
+		if err := css.Close(); err != nil {
+			log.Printf("cannot close SeriesSet for block: %q : %s\n", b.Meta().ULID, err)
+		}
+	}()
+	ss := css.SeriesSet
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
@@ -114,7 +128,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	return ss.Err()
 }
 
-func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
+func (pp *prometheusProcessor) processBlocks(ctx context.Context, blocks []tsdb.BlockReader) error {
 	promBlocksTotal.Add(len(blocks))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
 	if err := barpool.Start(); err != nil {
@@ -130,11 +144,16 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	for range pp.cc {
 		wg.Go(func() {
 			for br := range blockReadersCh {
-				if err := pp.do(br); err != nil {
+				if err := pp.do(ctx, br); err != nil {
 					promErrorsTotal.Inc()
-					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
+					errCh <- fmt.Errorf("cannot read block %q: %s", br.Meta().ULID, err)
 					return
 				}
+				if cb, ok := br.(io.Closer); ok {
+					if err := cb.Close(); err != nil {
+						errCh <- fmt.Errorf("cannot close block: %q: %w", br.Meta().ULID, err)
+					}
+				}
 				promBlocksProcessed.Inc()
 				bar.Increment()
 			}

app/vmctl/prometheus/prometheus.go

@@ -8,6 +8,8 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
 )
 
 // Config contains a list of params needed
@@ -60,13 +62,13 @@ func NewClient(cfg Config) (*Client, error) {
 		return nil, fmt.Errorf("failed to open snapshot %q: %s", cfg.Snapshot, err)
 	}
 	c := &Client{DBReadOnly: db}
-	minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
+	timeMin, timeMax, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse time in filter: %s", err)
 	}
 	c.filter = filter{
-		min:        minTime,
-		max:        maxTime,
+		min:        timeMin,
+		max:        timeMax,
 		label:      cfg.Filter.Label,
 		labelValue: cfg.Filter.LabelValue,
 	}
@@ -83,7 +85,7 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch blocks: %s", err)
 	}
-	s := &Stats{
+	s := &vmctlutil.Stats{
 		Filtered: c.filter.min != 0 || c.filter.max != 0 || c.filter.label != "",
 		Blocks:   len(blocks),
 	}
@@ -108,9 +110,15 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
 	return blocksToImport, nil
 }
 
+// CloseableSeriesSet defines a SeriesSet with Close method
+type CloseableSeriesSet struct {
+	SeriesSet storage.SeriesSet
+	Close     func() error
+}
+
 // Read reads the given BlockReader according to configured
 // time and label filters.
-func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
+func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*CloseableSeriesSet, error) {
 	minTime, maxTime := block.Meta().MinTime, block.Meta().MaxTime
 	if c.filter.min != 0 {
 		minTime = c.filter.min
@@ -122,8 +130,8 @@ func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
 	if err != nil {
 		return nil, err
 	}
-	ss := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
-	return ss, nil
+	ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
+	return &CloseableSeriesSet{ss, q.Close}, nil
 }
 
 func parseTime(start, end string) (int64, int64, error) {

app/vmctl/vmctlutil/stats.go

@@ -1,4 +1,4 @@
-package prometheus
+package vmctlutil
 
 import (
 	"fmt"
@@ -18,7 +18,7 @@ type Stats struct {
 
 // String returns string representation for s.
 func (s Stats) String() string {
-	str := fmt.Sprintf("Prometheus snapshot stats:\n"+
+	str := fmt.Sprintf("Snapshot stats:\n"+
 		"  blocks found: %d;\n"+
 		"  blocks skipped by time filter: %d;\n"+
 		"  min time: %d (%v);\n"+

app/vmselect/prometheus/federate.qtpl

@@ -2,13 +2,16 @@
 	"math"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 ) %}
 
 {% stripspace %}
 
 // Federate writes rs in /federate format.
 // See https://prometheus.io/docs/prometheus/latest/federation/
-{% func Federate(rs *netstorage.Result) %}
+{% func Federate(rs *netstorage.Result, escapeScheme string) %}
 	{% code
 		values := rs.Values
 		timestamps := rs.Timestamps
@@ -24,10 +27,54 @@
 			See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185
 		{% endcomment %}
 		{% return %}
-	{% endif %}
-	{%= prometheusMetricName(&rs.MetricName) %}{% space %}
+    {% endif %}
+
+    {% switch escapeScheme %}
+       {% case federateEscapeSchemeUTF8 %}
+          {%= prometheusFederateMetricNameUTF8(&rs.MetricName) %}{% space %}
+
+       {% case federateEscapeSchemeUnderscore %}
+         {%= prometheusFederateMetricNameEscapeUnderscore(&rs.MetricName) %}{% space %}
+
+       {% case "" %}
+         {%= prometheusMetricName(&rs.MetricName) %}{% space %}
+    {% endswitch %}
+
 	{%f= lastValue %}{% space %}
 	{%dl= timestamps[len(timestamps)-1] %}{% newline %}
 {% endfunc %}
 
+{% func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) %}
+	{%s= promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)) %}
+	{% if len(mn.Tags) > 0 %}
+	{
+		{% code tags := mn.Tags %}
+		{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)) %}={%= escapePrometheusLabel(tags[0].Value) %}
+		{% code tags = tags[1:] %}
+		{% for i := range tags %}
+			{% code tag := &tags[i] %}
+			,{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)) %}={%= escapePrometheusLabel(tag.Value) %}
+		{% endfor %}
+	}
+	{% endif %}
+{% endfunc %}
+
+{% func prometheusFederateMetricNameUTF8(mn *storage.MetricName) %}
+	{
+        {%= escapePrometheusLabel(mn.MetricGroup) %}
+        {% if len(mn.Tags) > 0 %}
+        ,
+		{% code tags := mn.Tags %}
+		{%= escapePrometheusLabel(tags[0].Key) %}={%= escapePrometheusLabel(tags[0].Value) %}
+		{% code tags = tags[1:] %}
+		{% for i := range tags %}
+			{% code tag := &tags[i] %}
+			,{%= escapePrometheusLabel(tag.Key) %}={%= escapePrometheusLabel(tag.Value) %}
+		{% endfor %}
+        {% endif %}
+	}
+{% endfunc %}
+
+
+
 {% endstripspace %}

app/vmselect/prometheus/federate.qtpl.go

@@ -9,82 +9,241 @@ import (
 	"math"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 // Federate writes rs in /federate format.// See https://prometheus.io/docs/prometheus/latest/federation/
 
-//line app/vmselect/prometheus/federate.qtpl:11
+//line app/vmselect/prometheus/federate.qtpl:14
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/federate.qtpl:11
+//line app/vmselect/prometheus/federate.qtpl:14
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/federate.qtpl:11
-func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/federate.qtpl:13
+//line app/vmselect/prometheus/federate.qtpl:14
+func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result, escapeScheme string) {
+//line app/vmselect/prometheus/federate.qtpl:16
 	values := rs.Values
 	timestamps := rs.Timestamps
 
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 	if len(timestamps) == 0 || len(values) == 0 {
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 		return
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 	}
-//line app/vmselect/prometheus/federate.qtpl:18
+//line app/vmselect/prometheus/federate.qtpl:21
 	lastValue := values[len(values)-1]
 
-//line app/vmselect/prometheus/federate.qtpl:20
+//line app/vmselect/prometheus/federate.qtpl:23
 	if math.IsNaN(lastValue) {
-//line app/vmselect/prometheus/federate.qtpl:26
+//line app/vmselect/prometheus/federate.qtpl:29
 		return
-//line app/vmselect/prometheus/federate.qtpl:27
+//line app/vmselect/prometheus/federate.qtpl:30
 	}
-//line app/vmselect/prometheus/federate.qtpl:28
-	streamprometheusMetricName(qw422016, &rs.MetricName)
-//line app/vmselect/prometheus/federate.qtpl:28
-	qw422016.N().S(` `)
-//line app/vmselect/prometheus/federate.qtpl:29
+//line app/vmselect/prometheus/federate.qtpl:32
+	switch escapeScheme {
+//line app/vmselect/prometheus/federate.qtpl:33
+	case federateEscapeSchemeUTF8:
+//line app/vmselect/prometheus/federate.qtpl:34
+		streamprometheusFederateMetricNameUTF8(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:34
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:36
+	case federateEscapeSchemeUnderscore:
+//line app/vmselect/prometheus/federate.qtpl:37
+		streamprometheusFederateMetricNameEscapeUnderscore(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:37
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:39
+	case "":
+//line app/vmselect/prometheus/federate.qtpl:40
+		streamprometheusMetricName(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:40
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:41
+	}
+//line app/vmselect/prometheus/federate.qtpl:43
 	qw422016.N().F(lastValue)
-//line app/vmselect/prometheus/federate.qtpl:29
+//line app/vmselect/prometheus/federate.qtpl:43
 	qw422016.N().S(` `)
-//line app/vmselect/prometheus/federate.qtpl:30
+//line app/vmselect/prometheus/federate.qtpl:44
 	qw422016.N().DL(timestamps[len(timestamps)-1])
-//line app/vmselect/prometheus/federate.qtpl:30
+//line app/vmselect/prometheus/federate.qtpl:44
 	qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 }
 
-//line app/vmselect/prometheus/federate.qtpl:31
-func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result, escapeScheme string) {
+//line app/vmselect/prometheus/federate.qtpl:45
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/federate.qtpl:31
-	StreamFederate(qw422016, rs)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+	StreamFederate(qw422016, rs, escapeScheme)
+//line app/vmselect/prometheus/federate.qtpl:45
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 }
 
-//line app/vmselect/prometheus/federate.qtpl:31
-func Federate(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+func Federate(rs *netstorage.Result, escapeScheme string) string {
+//line app/vmselect/prometheus/federate.qtpl:45
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/federate.qtpl:31
-	WriteFederate(qb422016, rs)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+	WriteFederate(qb422016, rs, escapeScheme)
+//line app/vmselect/prometheus/federate.qtpl:45
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 	return qs422016
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+}
+
+//line app/vmselect/prometheus/federate.qtpl:47
+func streamprometheusFederateMetricNameEscapeUnderscore(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:48
+	qw422016.N().S(promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)))
+//line app/vmselect/prometheus/federate.qtpl:49
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/federate.qtpl:49
+		qw422016.N().S(`{`)
+//line app/vmselect/prometheus/federate.qtpl:51
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/federate.qtpl:52
+		qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)))
+//line app/vmselect/prometheus/federate.qtpl:52
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:52
+		streamescapePrometheusLabel(qw422016, tags[0].Value)
+//line app/vmselect/prometheus/federate.qtpl:53
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/federate.qtpl:54
+		for i := range tags {
+//line app/vmselect/prometheus/federate.qtpl:55
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/federate.qtpl:55
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:56
+			qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)))
+//line app/vmselect/prometheus/federate.qtpl:56
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:56
+			streamescapePrometheusLabel(qw422016, tag.Value)
+//line app/vmselect/prometheus/federate.qtpl:57
+		}
+//line app/vmselect/prometheus/federate.qtpl:57
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/federate.qtpl:59
+	}
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:60
+func writeprometheusFederateMetricNameEscapeUnderscore(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:60
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+	streamprometheusFederateMetricNameEscapeUnderscore(qw422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:60
+func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/federate.qtpl:60
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/federate.qtpl:60
+	writeprometheusFederateMetricNameEscapeUnderscore(qb422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+	return qs422016
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:62
+func streamprometheusFederateMetricNameUTF8(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:62
+	qw422016.N().S(`{`)
+//line app/vmselect/prometheus/federate.qtpl:64
+	streamescapePrometheusLabel(qw422016, mn.MetricGroup)
+//line app/vmselect/prometheus/federate.qtpl:65
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/federate.qtpl:65
+		qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:67
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/federate.qtpl:68
+		streamescapePrometheusLabel(qw422016, tags[0].Key)
+//line app/vmselect/prometheus/federate.qtpl:68
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:68
+		streamescapePrometheusLabel(qw422016, tags[0].Value)
+//line app/vmselect/prometheus/federate.qtpl:69
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/federate.qtpl:70
+		for i := range tags {
+//line app/vmselect/prometheus/federate.qtpl:71
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/federate.qtpl:71
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:72
+			streamescapePrometheusLabel(qw422016, tag.Key)
+//line app/vmselect/prometheus/federate.qtpl:72
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:72
+			streamescapePrometheusLabel(qw422016, tag.Value)
+//line app/vmselect/prometheus/federate.qtpl:73
+		}
+//line app/vmselect/prometheus/federate.qtpl:74
+	}
+//line app/vmselect/prometheus/federate.qtpl:74
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/federate.qtpl:76
+}
+
+//line app/vmselect/prometheus/federate.qtpl:76
+func writeprometheusFederateMetricNameUTF8(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:76
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+	streamprometheusFederateMetricNameUTF8(qw422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+}
+
+//line app/vmselect/prometheus/federate.qtpl:76
+func prometheusFederateMetricNameUTF8(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/federate.qtpl:76
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/federate.qtpl:76
+	writeprometheusFederateMetricNameUTF8(qb422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+	return qs422016
+//line app/vmselect/prometheus/federate.qtpl:76
 }

app/vmselect/prometheus/federate_test.go

@@ -8,15 +8,15 @@ import (
 )
 
 func TestFederate(t *testing.T) {
-	f := func(rs *netstorage.Result, expectedResult string) {
+	f := func(rs *netstorage.Result, escapeScheme string, expectedResult string) {
 		t.Helper()
-		result := Federate(rs)
+		result := Federate(rs, escapeScheme)
 		if result != expectedResult {
 			t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, expectedResult)
 		}
 	}
 
-	f(&netstorage.Result{}, ``)
+	f(&netstorage.Result{}, ``, ``)
 
 	f(&netstorage.Result{
 		MetricName: storage.MetricName{
@@ -39,5 +39,60 @@ func TestFederate(t *testing.T) {
 		},
 		Values:     []float64{1.23},
 		Timestamps: []int64{123},
-	}, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
+	}, ``, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
+
+	f(&netstorage.Result{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foo.bar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("some.!other"),
+					Value: []byte("value.unchanged!."),
+				},
+				{
+					Key:   []byte("qqq"),
+					Value: []byte("\\"),
+				},
+				{
+					Key:   []byte("!key"),
+					Value: []byte("value"),
+				},
+				{
+					Key: []byte("abc"),
+					// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
+					Value: []byte("a<b\"\\c"),
+				},
+			},
+		},
+		Values:     []float64{1.23},
+		Timestamps: []int64{123},
+	}, federateEscapeSchemeUnderscore, `foo_bar{some__other="value.unchanged!.",qqq="\\",_key="value",abc="a<b\"\\c"} 1.23 123`+"\n")
+
+	f(&netstorage.Result{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foo.bar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("some.!other"),
+					Value: []byte("value.unchanged!."),
+				},
+				{
+					Key:   []byte("qqq"),
+					Value: []byte("\\"),
+				},
+				{
+					Key:   []byte("!key"),
+					Value: []byte("value"),
+				},
+				{
+					Key: []byte(`ab"c`),
+					// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
+					Value: []byte("a<b\"\\c"),
+				},
+			},
+		},
+		Values:     []float64{1.23},
+		Timestamps: []int64{123},
+	}, federateEscapeSchemeUTF8, `{"foo.bar","some.!other"="value.unchanged!.","qqq"="\\","!key"="value","ab\"c"="a<b\"\\c"} 1.23 123`+"\n")
+
 }

app/vmselect/prometheus/federate_timing_test.go

@@ -9,16 +9,17 @@ import (
 )
 
 func BenchmarkFederate(b *testing.B) {
+
 	rs := &netstorage.Result{
 		MetricName: storage.MetricName{
-			MetricGroup: []byte("foo_bar_bazaaaa_total"),
+			MetricGroup: []byte("foo_bar_?_._bazaaaa_total"),
 			Tags: []storage.Tag{
 				{
-					Key:   []byte("instance"),
+					Key:   []byte("instance:job"),
 					Value: []byte("foobarbaz:2344"),
 				},
 				{
-					Key:   []byte("job"),
+					Key:   []byte("job.name"),
 					Value: []byte("aaabbbccc"),
 				},
 			},
@@ -27,12 +28,22 @@ func BenchmarkFederate(b *testing.B) {
 		Timestamps: []int64{1234567890},
 	}
 
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		var bb bytes.Buffer
-		for pb.Next() {
-			bb.Reset()
-			WriteFederate(&bb, rs)
-		}
-	})
+	f := func(name, escapeScheme string) {
+		b.Helper()
+
+		b.Run(name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.RunParallel(func(pb *testing.PB) {
+				var bb bytes.Buffer
+				for pb.Next() {
+					bb.Reset()
+					WriteFederate(&bb, rs, escapeScheme)
+				}
+			})
+		})
+	}
+
+	f("without escape", "")
+	f("allow-utf-8", federateEscapeSchemeUTF8)
+	f("legacy-underscore", federateEscapeSchemeUnderscore)
 }

app/vmselect/prometheus/prometheus.go

@@ -108,6 +108,11 @@ func PrettifyQuery(w http.ResponseWriter, r *http.Request) {
 	_ = bw.Flush()
 }
 
+const (
+	federateEscapeSchemeUnderscore = "underscore"
+	federateEscapeSchemeUTF8       = "utf-8"
+)
+
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
 func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	defer federateDuration.UpdateDuration(startTime)
@@ -132,6 +137,21 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 
+	// add best-effort format negotiation
+	// modern version of Prometheus always set allow-utf-8 in order to properly parse utf-8 names and labels
+	// prometheus below v3 uses underscore escaping by default and it's the most common standard
+	var escapeScheme string
+	accept := r.Header.Get("Accept")
+	if len(accept) > 0 && strings.Contains(accept, "allow-utf-8") {
+		escapeScheme = federateEscapeSchemeUTF8
+	}
+	// try fallback to legacy underscore escaping if needed for Prometheus only,
+	// it's not widely used after Prometheus v3.0 release
+	// most of the Prometheus scrapers already use allow-utf-8 header
+	isPrometheus := strings.HasPrefix(r.UserAgent(), "Prometheus")
+	if len(escapeScheme) == 0 && isPrometheus {
+		escapeScheme = federateEscapeSchemeUnderscore
+	}
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
@@ -141,7 +161,7 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 			return err
 		}
 		bb := sw.getBuffer(workerID)
-		WriteFederate(bb, rs)
+		WriteFederate(bb, rs, escapeScheme)
 		return sw.maybeFlushBuffer(bb)
 	})
 	if err == nil {

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-BjJ7fDL7.js"></script>
+  <script type="module" crossorigin src="./assets/index-CoGukb-x.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BL7jEFBa.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BBUnmLOr.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.3 AS build-web-stage
+FROM golang:1.26.4 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts

@@ -1,7 +1,7 @@
-export const seriesFetchedWarning = `No match! 
+export const seriesFetchedWarning = `No match!
 This query hasn't selected any time series from database.
 Either the requested metrics are missing in the database,
 or there is a typo in series selector.`;
 
 export const partialWarning = `The shown results are marked as PARTIAL.
-The result is marked as partial if one or more vmstorage nodes failed to respond to the query.`;
+The result is marked as partial if one or more storage nodes failed to respond to the query.`;

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/index.tsx

@@ -71,7 +71,7 @@ const RulesHeader = ({
           <TextField
             label="Search"
             value={search}
-            placeholder="Filter by rule, name or labels"
+            placeholder="Filter by group or rule name"
             startIcon={<SearchIcon />}
             onChange={onChangeSearch}
           />

apptest/model.go

@@ -79,24 +79,25 @@ type PrometheusWriteQuerier interface {
 
 // QueryOpts contains various params used for querying or ingesting data
 type QueryOpts struct {
-	Tenant         string
-	Timeout        string
-	Start          string
-	End            string
-	Time           string
-	Step           string
-	ExtraFilters   []string
-	ExtraLabels    []string
-	Trace          string
-	ReduceMemUsage string
-	MaxLookback    string
-	LatencyOffset  string
-	Format         string
-	NoCache        string
-	Headers        http.Header
-	From           string
-	Until          string
-	StorageStep    string
+	Tenant              string
+	Timeout             string
+	Start               string
+	End                 string
+	Time                string
+	Step                string
+	ExtraFilters        []string
+	ExtraLabels         []string
+	Trace               string
+	ReduceMemUsage      string
+	MaxLookback         string
+	LatencyOffset       string
+	Format              string
+	NoCache             string
+	Headers             http.Header
+	From                string
+	Until               string
+	StorageStep         string
+	DenyPartialResponse string
 }
 
 func (qos *QueryOpts) getHeaders() http.Header {
@@ -132,6 +133,7 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("from", qos.From)
 	addNonEmpty("until", qos.Until)
 	addNonEmpty("storage_step", qos.StorageStep)
+	addNonEmpty("deny_partial_response", qos.DenyPartialResponse)
 
 	return uv
 }

apptest/tests/replication_test.go

@@ -1015,35 +1015,42 @@ func testGroupSkipSlowReplicas(tc *apptest.TestCase, opts *testGroupReplicationO
 func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOpts) {
 	t := tc.T()
 
-	assertSeries := func(app *apptest.Vmselect, wantPartial bool) {
+	assertSeries := func(app *apptest.Vmselect, denyPartialResponse string, want *apptest.PrometheusAPIV1SeriesResponse) {
 		t.Helper()
 		tc.Assert(&apptest.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
 				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{
-					Start: "2024-01-01T00:00:00Z",
-					End:   "2024-01-31T00:00:00Z",
+					Start:               "2024-01-01T00:00:00Z",
+					End:                 "2024-01-31T00:00:00Z",
+					DenyPartialResponse: denyPartialResponse,
 				}).Sort()
 			},
-			Want: &apptest.PrometheusAPIV1SeriesResponse{
-				Status:    "success",
-				IsPartial: wantPartial,
-			},
+			Want: want,
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
+				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data", "Error"),
 			},
 		})
 	}
 
-	mustReturnPartialResponse := true
-	mustReturnFullResponse := false
+	allowPartialResponse := ""
+	denyPartialResponse := "1"
+
+	mustReturnPartialResponse := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: true,
+	}
+	mustReturnFullResponse := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: false,
+	}
 
 	// All vmstorage replicas are available so both vmselects must return full
 	// response.
-	assertSeries(opts.c.vmselect, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop groupRF-1 vmstorage nodes in first group.
 	//
@@ -1053,10 +1060,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	// about the replication factor and therefore they must still be able to
 	// return full dataset.
 	opts.c.storageGroups[0].stopNodes(tc, opts.groupRF-1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop groupRF-1 vmstorages in the remaining groups.
 	//
@@ -1066,10 +1073,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	for g := 1; g < len(opts.c.storageGroups); g++ {
 		opts.c.storageGroups[g].stopNodes(tc, opts.groupRF-1)
 	}
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstorage in the first group.
 	//
@@ -1077,10 +1084,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	// because it is unaware of replication across groups. vmselectGroupGlobalRF
 	// will continue retuning full dataset.
 	opts.c.storageGroups[0].stopNodes(tc, 1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstoarge in remaining globarRF-1 groups.
 	//
@@ -1089,19 +1096,56 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	for g := 1; g < opts.globalRF-1; g++ {
 		opts.c.storageGroups[g].stopNodes(tc, 1)
 	}
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstoarge in one more group.
 	//
 	// vmselectGroupGlobalRF must now return partial dataset.
 	opts.c.storageGroups[opts.globalRF].stopNodes(tc, 1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+
+	// Stop all the remaining vmstorage nodes except a single node.
+	//
+	// At this point vmselects still must be able to return partial response
+	// because at least one vmstorage node has successfully returned results.
+	n := len(opts.c.storageGroups[0].vmstorages)
+	opts.c.storageGroups[0].stopNodes(tc, n-1)
+	for g := 1; g < len(opts.c.storageGroups); g++ {
+		n := len(opts.c.storageGroups[g].vmstorages)
+		opts.c.storageGroups[g].stopNodes(tc, n)
+	}
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+
+	mustReturnUnavailableError := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "error",
+		ErrorType: "503",
+	}
+
+	// vmselects must return an error for the same request when partial
+	// responses are denied explicitly.
+	assertSeries(opts.c.vmselect, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupRF, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGlobalRF, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupGlobalRF, denyPartialResponse, mustReturnUnavailableError)
+
+	// Stop the last remaining vmstorage node.
+	//
+	// vmselects must return an error when there are no successful vmstorage
+	// responses.
+	opts.c.storageGroups[0].stopNodes(tc, 1)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnUnavailableError)
 }
 
 // TestClusterReplication_PartialResponseMultitenant checks how vmselect handles some

apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/meta.json

@@ -0,0 +1,51 @@
+{
+	"ulid": "01JFJBS3YP1SHZ3PJQ6HK76EC3",
+	"minTime": 1734709200000,
+	"maxTime": 1734709320000,
+	"stats": {
+		"numSamples": 400,
+		"numSeries": 100,
+		"numChunks": 100
+	},
+	"compaction": {
+		"level": 1,
+		"sources": [
+			"01JFJBS3YP1SHZ3PJQ6HK76EC3"
+		],
+		"parents": [
+			{
+				"ulid": "00000000000000000000000000",
+				"minTime": 0,
+				"maxTime": 0
+			}
+		],
+		"hints": [
+			"from-out-of-order"
+		]
+	},
+	"version": 1,
+	"out_of_order": false,
+	"thanos": {
+		"labels": {},
+		"downsample": {
+			"resolution": 0
+		},
+		"source": "receive",
+		"segment_files": [
+			"000001"
+		],
+		"files": [
+			{
+				"rel_path": "chunks/000001",
+				"size_bytes": 4808
+			},
+			{
+				"rel_path": "index",
+				"size_bytes": 55021
+			},
+			{
+				"rel_path": "meta.json"
+			}
+		]
+	}
+}

apptest/tests/vmctl_mimir_migration_test.go

@@ -0,0 +1,139 @@
+package tests
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+)
+
+const (
+	testMimirPath             = "testdata/mimir-tsdb"
+	expectedMimirResponseFile = "./testdata/mimir-tsdb/expected_response.json"
+)
+
+func TestSingleVmctlMimirProtocol(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("cannot get current working directory: %s", err)
+	}
+
+	path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
+	vmctlFlags := []string{
+		`mimir`,
+		`--mimir-tenant-id=anonymous`,
+		`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
+		`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
+		`--mimir-custom-s3-endpoint=http://localhost:9000`,
+		`--mimir-path=` + path,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+		`--vm-concurrency=6`,
+		`--mimir-concurrency=6`,
+	}
+
+	testMimirProtocol(tc, vmsingleDst, vmctlFlags)
+}
+
+func TestClusterVmctlMimirProtocol(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	cluster := tc.MustStartDefaultCluster()
+	vmAddr := fmt.Sprintf("http://%s/", cluster.Vminsert.HTTPAddr())
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("cannot get current working directory: %s", err)
+	}
+
+	path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
+
+	vmctlFlags := []string{
+		`mimir`,
+		`--mimir-tenant-id=anonymous`,
+		`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
+		`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
+		`--mimir-custom-s3-endpoint=http://localhost:9000`,
+		`--mimir-path=` + path,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+		`--vm-concurrency=6`,
+		`--mimir-concurrency=6`,
+	}
+
+	testMimirProtocol(tc, cluster, vmctlFlags)
+}
+
+func testMimirProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
+	t := tc.T()
+	t.Helper()
+
+	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+
+	// test for empty data request
+	got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
+		Step: "5m",
+		Time: "2025-06-02T17:14:00Z",
+	})
+
+	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	tc.MustStartVmctl("vmctl", vmctlFlags)
+
+	sut.ForceFlush(t)
+
+	// open the expected series response file
+	file, err := os.Open(expectedMimirResponseFile)
+	if err != nil {
+		t.Fatalf("cannot open expected series response file: %s", err)
+	}
+	defer file.Close()
+
+	bytes, err := io.ReadAll(file)
+	if err != nil {
+		t.Fatalf("cannot read expected series response file: %s", err)
+	}
+
+	var wantResponse apptest.PrometheusAPIV1QueryResponse
+	if err := json.Unmarshal(bytes, &wantResponse); err != nil {
+		t.Fatalf("cannot unmarshal expected series response file: %s", err)
+	}
+	wantResponse.Sort()
+
+	tc.Assert(&apptest.AssertOptions{
+		// For cluster version, we need to wait longer for the metrics to be stored
+		Retries: 300,
+		Msg:     `unexpected metrics stored on vmsingle via the prometheus protocol`,
+		Got: func() any {
+			expected := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
+				Start: "2024-12-01T15:31:10Z",
+				End:   "2024-12-31T15:32:20Z",
+			})
+			expected.Sort()
+			return expected.Data.Result
+		},
+		Want: wantResponse.Data.Result,
+		CmpOpts: []cmp.Option{
+			cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		},
+	})
+}

dashboards/operator.json

@@ -130,7 +130,7 @@
           "calcs": [
             "lastNotNull"
           ],
-          "fields": "/^short_version$/",
+          "fields": "/^version$/",
           "values": false
         },
         "showPercentChange": false,
@@ -146,11 +146,10 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "vm_app_version{job=~\"$job\",instance=~\"$instance\"}",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "interval": "",
-          "legendFormat": "{{short_version}}",
           "range": false,
           "refId": "A"
         }

dashboards/victoriametrics-cluster.json

@@ -791,7 +791,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/victoriametrics.json

@@ -789,7 +789,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/vm/operator.json

@@ -131,7 +131,7 @@
           "calcs": [
             "lastNotNull"
           ],
-          "fields": "/^short_version$/",
+          "fields": "/^version$/",
           "values": false
         },
         "showPercentChange": false,
@@ -147,11 +147,10 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "vm_app_version{job=~\"$job\",instance=~\"$instance\"}",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "interval": "",
-          "legendFormat": "{{short_version}}",
           "range": false,
           "refId": "A"
         }

dashboards/vm/victoriametrics-cluster.json

@@ -792,7 +792,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/vm/victoriametrics.json

@@ -790,7 +790,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/vm/vmagent.json

@@ -785,7 +785,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/vm/vmalert.json

@@ -566,7 +566,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2804,10 +2804,10 @@
             "overrides": []
           },
           "gridPos": {
-            "h": 8,
+            "h": 7,
             "w": 12,
             "x": 0,
-            "y": 352
+            "y": 11
           },
           "id": 63,
           "options": {
@@ -2843,7 +2843,113 @@
           ],
           "title": "Restarts ($job)",
           "type": "timeseries"
-        }
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Group iteration reset can be caused by irregular delays during evaluation or by the system wall clock being moved backward.\nIf it is caused by host clock changes, vmalert could generate duplicate results for the group rules, since some evaluations could be repeated.\nCheck the host clock time synchronization configuration if this happens frequently.\n",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "bars",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 11
+          },
+          "id": 70,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(increase(vmalert_iteration_reset_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
+              "interval": "1m",
+              "legendFormat": "({{job}}) {{group}}({{file}})",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Group Iteration Reset ($instance)",
+          "type": "timeseries"
+        }        
       ],
       "title": "Troubleshooting",
       "type": "row"

dashboards/vm/vmauth.json

@@ -492,14 +492,14 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by (job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,
           "refId": "A"
         }
       ],
-      "title": "Version",
+      "title": "",
       "type": "table"
     },
     {

dashboards/vmagent.json

@@ -784,7 +784,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,

dashboards/vmalert.json

@@ -565,7 +565,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,
@@ -2803,10 +2803,10 @@
             "overrides": []
           },
           "gridPos": {
-            "h": 8,
+            "h": 7,
             "w": 12,
             "x": 0,
-            "y": 352
+            "y": 11
           },
           "id": 63,
           "options": {
@@ -2842,7 +2842,113 @@
           ],
           "title": "Restarts ($job)",
           "type": "timeseries"
-        }
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Group iteration reset can be caused by irregular delays during evaluation or by the system wall clock being moved backward.\nIf it is caused by host clock changes, vmalert could generate duplicate results for the group rules, since some evaluations could be repeated.\nCheck the host clock time synchronization configuration if this happens frequently.\n",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "bars",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 11
+          },
+          "id": 70,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(increase(vmalert_iteration_reset_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
+              "interval": "1m",
+              "legendFormat": "({{job}}) {{group}}({{file}})",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Group Iteration Reset ($instance)",
+          "type": "timeseries"
+        }        
       ],
       "title": "Troubleshooting",
       "type": "row"

dashboards/vmauth.json

@@ -491,14 +491,14 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by (job, short_version)",
+          "expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
           "format": "table",
           "instant": true,
           "range": false,
           "refId": "A"
         }
       ],
-      "title": "Version",
+      "title": "",
       "type": "table"
     },
     {

deployment/docker/Makefile

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.23.4
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.23.4
 
-GO_BUILDER_IMAGE := golang:1.26.3
+GO_BUILDER_IMAGE := golang:1.26.4
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/rules/alerts-vmalert.yml

@@ -64,6 +64,18 @@ groups:
             group \"{{ $labels.group }}\". See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups. 
             If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries."
 
+      - alert: GroupIterationReset
+        expr: increase(vmalert_iteration_reset_total[5m]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Evaluation iteration for group {{ $labels.group }} in file {{ $labels.file }} is reset"
+          description: "Evaluation iteration for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\" is reset on vmalert instance {{ $labels.instance }}.
+            This can be caused by irregular delays during evaluation or by the system wall clock being moved backward. If it is caused by host clock changes, vmalert could
+            generate duplicate results for the group rules since some evaluations could be repeated. Check host clock time synchronization configurations if this happens frequently."
+
+
       - alert: RemoteWriteErrors
         expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
         for: 15m
@@ -108,4 +120,3 @@ groups:
           summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
           description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
             Check vmalert's logs for detailed error message."
-

docs/anomaly-detection/CHANGELOG.md

@@ -130,7 +130,7 @@ Released: 2025-11-05
 ## v1.27.0
 Released: 2025-10-31
 
-- FEATURE: Added runtime state compatibility guard for [stateful](https://docs.victoriametrics.com/anomaly-detection/components/settings/#restore-state) deployments. The service now persists normalized versions, evaluates an [upgrade/downgrade compatibility matrix](https://docs.victoriametrics.com/anomaly-detection/migration/#compatibility-matrix), and selectively drops or reuses DB records and on-disk artifacts to keep migrations safe and automatic. Please refer to the [migration page](https://docs.victoriametrics.com/anomaly-detection/migration/) for more details.
+- FEATURE: Added runtime state compatibility guard for [stateful](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) deployments. The service now persists normalized versions, evaluates an [upgrade/downgrade compatibility matrix](https://docs.victoriametrics.com/anomaly-detection/migration/#compatibility-matrix), and selectively drops or reuses DB records and on-disk artifacts to keep migrations safe and automatic. Please refer to the [migration page](https://docs.victoriametrics.com/anomaly-detection/migration/) for more details.
 
 - IMPROVEMENT: Parallelization now honours container cgroup CPU/RAM limits, so `settings.n_workers` in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#parallelization), internal routines and the `vmanomaly_available_memory_bytes`/`vmanomaly_cpu_cores_available` [startup metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) report or use container resources instead of host totals, keeping the [self-monitoring dashboard](https://docs.victoriametrics.com/anomaly-detection/self-monitoring/#grafana-dashboard) accurate.
 
@@ -190,7 +190,7 @@ Released: 2025-08-19
 ## v1.25.2
 Released: 2025-07-30
 
-- BUGFIX: Resolved inconsistent state between in-memory models and state database (if [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#stateful-mode) is enabled). This bug caused `Model instance not found` warnings during inference calls and prevented proper cleanup of stale models from disk. The fix also prevents state updates when operations are terminated mid-execution of scheduled fit/infer jobs.
+- BUGFIX: Resolved inconsistent state between in-memory models and state database (if [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) is enabled). This bug caused `Model instance not found` warnings during inference calls and prevented proper cleanup of stale models from disk. The fix also prevents state updates when operations are terminated mid-execution of scheduled fit/infer jobs.
 
 - BUGFIX: Added explicit handling for inference calls on models that were deleted from disk by the time of their usage, but still referenced in the state database, preventing `'NoneType' object has no attribute 'infer'` rows in logs. Now a warning is logged and the inference call is skipped, which is expected behavior for deleted models.
 
@@ -210,7 +210,7 @@ Released: 2025-07-24
 
 - BUGFIX: Prevented `OneOffScheduler` and `BacktestingScheduler` [schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) from receiving no data (when [state restoration](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) is enabled). Now a warning is logged and such scheduler types are implicitly used without state restoration, which is expected behavior for these one-time-job schedulers.
 
-- BUGFIX: Now the paths to artifact database (if [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#stateful-mode) is enabled) are properly resolved to absolute, preventing errors at initialization time (like `sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) unable to open database file`) or warnings (like `SAWarning: fully NULL primary key identity cannot load any object.`).
+- BUGFIX: Now the paths to artifact database (if [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) is enabled) are properly resolved to absolute, preventing errors at initialization time (like `sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) unable to open database file`) or warnings (like `SAWarning: fully NULL primary key identity cannot load any object.`).
 
 ## v1.25.0
 Released: 2025-07-17
@@ -559,7 +559,7 @@ Released: 2024-08-10
   - **Lowest anomaly scores** (=0) when the *model's predictions (`yhat`) fall outside the expected range*, signaling uncertain predictions.
   - For more details, please refer to the [documentation](https://docs.victoriametrics.com/anomaly-detection/components/reader/#per-query-parameters).
 
-- IMPROVEMENT: Added `latency_offset` argument to the [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) to override the default `-search.latencyOffset` [flag of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/#list-of-command-line-flags) (30s). The default value is set to 1ms, which should help in cases where `sampling_frequency` is low (10-60s) and `sampling_frequency` equals `infer_every` in the [PeriodicScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler). This prevents users from receiving `service - WARNING - [Scheduler [scheduler_alias]] No data available for inference.` warnings in logs and allows for consecutive `infer` calls without gaps. To restore the backward compatible behavior, set it equal to your `-search.latencyOffset` value in [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) config section.
+- IMPROVEMENT: Added `latency_offset` argument to the [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) to override the default `-search.latencyOffset` [flag of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/#list-of-command-line-flags) (30s). The default value is set to 1ms, which should help in cases where `sampling_period` is low (10-60s) and `sampling_period` equals `infer_every` in the [PeriodicScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler). This prevents users from receiving `service - WARNING - [Scheduler [scheduler_alias]] No data available for inference.` warnings in logs and allows for consecutive `infer` calls without gaps. To restore the backward compatible behavior, set it equal to your `-search.latencyOffset` value in [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) config section.
 
 - BUGFIX: Ensure the `use_transform` argument of the [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) functions as intended.
 - BUGFIX: Add a docstring for `query_from_last_seen_timestamp` arg of [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader).

docs/anomaly-detection/FAQ.md

@@ -281,7 +281,7 @@ reader:
   datasource_url: 'some_url_to_read_data_from'
   queries:
     query_alias1: 'some_metricsql_query'
-  sampling_frequency: '1m'  # change to whatever you need in data granularity
+  sampling_period: '1m'  # change to whatever you need in data granularity
   # other params if needed
   # https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader
 
@@ -294,7 +294,7 @@ writer:
 # https://docs.victoriametrics.com/anomaly-detection/components/monitoring/
 ```
 
-Configuration above will produce N intervals of full length (`fit_window`=14d + `fit_every`=1h) until `to_iso` timestamp is reached to run N consecutive `fit` calls to train models; Then these models will be used to produce `M = [fit_every / sampling_frequency]` infer datapoints for `fit_every` range at the end of each such interval, imitating M consecutive calls of `infer_every` in `PeriodicScheduler` [config](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler). These datapoints then will be written back to VictoriaMetrics TSDB, defined in `writer` [section](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) for further visualization (i.e. in VMUI or Grafana)
+Configuration above will produce N intervals of full length (`fit_window`=14d + `fit_every`=1h) until `to_iso` timestamp is reached to run N consecutive `fit` calls to train models; Then these models will be used to produce `M = [fit_every / sampling_period]` infer datapoints for `fit_every` range at the end of each such interval, imitating M consecutive calls of `infer_every` in `PeriodicScheduler` [config](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler). These datapoints then will be written back to VictoriaMetrics TSDB, defined in `writer` [section](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) for further visualization (i.e. in VMUI or Grafana)
 
 ## Forecasting
 
@@ -499,7 +499,7 @@ schedulers:
 models:
   zscore_example:
     class: 'zscore_online'
-    min_n_samples_seen: 120  # i.e. minimal relevant seasonality or (initial) fit_window / sampling_frequency
+    min_n_samples_seen: 120  # i.e. minimal relevant seasonality or (initial) fit_window / sampling_period
     decay: 0.999  # decay factor to control how fast the model adapts to new data, the lower, the faster it adapts
     schedulers: ['periodic']
     # other model params ...

docs/anomaly-detection/components/README.md

@@ -41,7 +41,7 @@ settings:
   restore_state: True  # restore state from previous run, if available
   retention:  # how long to keep stale models on disk/in memory
     ttl: "1d"  # time-to-live duration, if the model was not used for inference within this duration, it will be considered stale
-    check_every: "1h"  # how often to check for stale models and remove them
+    check_interval: "1h"  # how often to check for stale models and remove them
 
 # how and when to run the models is defined by schedulers
 # https://docs.victoriametrics.com/anomaly-detection/components/scheduler/
@@ -143,11 +143,11 @@ server:
 
 > This feature is better used in conjunction with [stateful service](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the state of the models and schedulers between restarts and reuse what can be reused, thus avoiding unnecessary re-training of models, re-initialization of schedulers and re-reading of data.
 
-{{% available_from "v1.25.0" anomaly %}} Service supports hot reload of configuration files, which allows for automatic reloading of configurations on config files change filesystem events without the need of explicit service restart. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). `vmanomaly_hot_reload_enabled` flag in [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) will be set to 1 (if enabled) or 0 (if disabled).
+{{% available_from "v1.25.0" anomaly %}} Service supports hot reload of configuration files, which allows for automatic reloading of configurations on config files change filesystem events without the need of explicit service restart. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). `vmanomaly_config_reload_enabled` flag in [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) will be set to 1 (if enabled) or 0 (if disabled).
 
 ### How it works
 
-It works by watching for file system events, such as modifications, creations, or deletions of `.yml|.yaml` files in the specified directories. When a change is detected, the service will attempt to reload the configuration files, rebuild the [global config](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-config) and reinitialize the components. If the reload is successful, the `vmanomaly_hot_reload_events_total` metric will be incremented for `status="success"` label, otherwise it will be incremented with `status="failure"` label and a respective error message on config validation failure(s) will be logged.
+It works by watching for file system events, such as modifications, creations, or deletions of `.yml|.yaml` files in the specified directories. When a change is detected, the service will attempt to reload the configuration files, rebuild the [global config](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-config) and reinitialize the components. If the reload is successful, the `vmanomaly_config_reloads_total` metric will be incremented for `status="success"` label, otherwise it will be incremented with `status="failure"` label and a respective error message on config validation failure(s) will be logged.
 
 > If the reload fails, the service will log an error message indicating the reason for the failure, and the **previous configuration will remain active until a successful reload occurs** to preserve the service's stability. This means that if there are errors in the new configuration, the service will continue to operate with the last valid configuration until the issues are resolved.
 
@@ -219,7 +219,7 @@ reader:
 # ... (rest of the config remains unchanged)
 ```
 
-After saving the changes, hot reload will automatically detect the changes in `config.yaml` and attempt to reload the configuration. As the changes are valid, the service will log a success message and increment the `vmanomaly_hot_reload_events_total` metric with `status="success"` label:
+After saving the changes, hot reload will automatically detect the changes in `config.yaml` and attempt to reload the configuration. As the changes are valid, the service will log a success message and increment the `vmanomaly_config_reloads_total` metric with `status="success"` label:
 
 - All the model instances of class `zscore_online`, that were trained on `host_network_receive_errors` can be reused as they are still valid and "fresh" for making inference on new datapoints until the next `fit_every` happens.
 - All the model instances of class `zscore_online`, that were trained on `cpu_seconds_total` will be re-trained with the new query expression and frequency, as old model instances are not valid anymore.

docs/anomaly-detection/components/reader.md

@@ -369,7 +369,7 @@ If True, then query will be performed from the last seen timestamp for a given s
 `1ms`
             </td>
             <td>
-It allows overriding the default `-search.latencyOffset`{{% available_from "v1.15.1" anomaly %}} [flag of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/#list-of-command-line-flags) (30s). The default value is set to 1ms, which should help in cases where `sampling_frequency` is low (10-60s) and `sampling_frequency` equals `infer_every` in the [PeriodicScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler). This prevents users from receiving `service - WARNING - [Scheduler [scheduler_alias]] No data available for inference.` warnings in logs and allows for consecutive `infer` calls without gaps. To restore the old behavior, set it equal to your `-search.latencyOffset` [flag value](https://docs.victoriametrics.com/victoriametrics/#list-of-command-line-flags).
+It allows overriding the default `-search.latencyOffset`{{% available_from "v1.15.1" anomaly %}} [flag of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/#list-of-command-line-flags) (30s). The default value is set to 1ms, which should help in cases where `sampling_period` is low (10-60s) and `sampling_period` equals `infer_every` in the [PeriodicScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler). This prevents users from receiving `service - WARNING - [Scheduler [scheduler_alias]] No data available for inference.` warnings in logs and allows for consecutive `infer` calls without gaps. To restore the old behavior, set it equal to your `-search.latencyOffset` [flag value](https://docs.victoriametrics.com/victoriametrics/#list-of-command-line-flags).
             </td>
         </tr>
         <tr>

docs/guides/guide-vmcluster-multiple-retention-setup/README.md

@@ -6,342 +6,45 @@ build:
 sitemap:
   disable: true
 ---
+**Objective**
 
-[VictoriaMetrics Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) supports specifying multiple retentions for distinct sets of time series and tenants. If you are an Enterprise user, [configure multiple retentions directly through retention filters](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#retention-filters) instead of following this guide.
+Setup Victoria Metrics Cluster with support of multiple retention periods within one installation.
 
-This guide explains how to set up multiple retentions using an [open-source VictoriaMetrics Cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
+**Enterprise Solution**
 
-## Overview
+[VictoriaMetrics Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) supports specifying multiple retentions
+for distinct sets of time series and [tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
+via [retention filters](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#retention-filters).
 
-VictoriaMetrics retains metrics by default for 1 month. You can change data retention with the [`-retentionPeriod` command-line flag](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention), but this value applies to all time series stored on a given `vmstorage` node and cannot be customized per tenant or per metric in the open source version. 
+**Open Source Solution**
 
-## Multi-Retention Architecture
+Community version of VictoriaMetrics supports only one retention period per `vmstorage` node via [-retentionPeriod](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention) command-line flag.
 
-To support multiple retentions with the open source version of VictoriaMetrics cluster, you can split the cluster into several logical groups of `vmstorage` nodes, where each group is configured with a different `-retentionPeriod` and receives only the data that must follow that retention. 
+A multi-retention setup can be implemented by dividing a [victoriametrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) into logical groups with different retentions.
 
-Each storage group is connected to a separate `vminsert`, while a shared `vmselect` layer queries across all storage groups so that dashboards and alerts continue to see a single logical VictoriaMetrics backend.
+Example:
+Setup should handle 3 different retention groups 3months, 1year and 3 years.
+Solution contains 3 groups of vmstorages + vminserts and one group of vmselects. Routing is done by [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/)
+by [splitting data streams](https://docs.victoriametrics.com/victoriametrics/vmagent/#splitting-data-streams-among-multiple-systems). 
+The [-retentionPeriod](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention) sets how long to keep the metrics.
+
+The diagram below shows a proposed solution
 
 ![Setup](setup.webp)
 
-In the example used throughout this guide, the cluster is divided into three groups: 
+**Implementation Details**
 
-- Group A: 3-month retention.
-- Group B: 1-year retention.
-- Group C: 3-year retention. 
+1. Groups of vminserts A know about only vmstorages A and this is explicitly specified via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup). 
+1. Groups of vminserts B know about only vmstorages B and this is explicitly specified via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup). 
+1. Groups of vminserts C know about only vmstorages C and this is explicitly specified via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup). 
+1. vmselect reads data from all vmstorage nodes via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup) 
+   with [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) setting equal to vmagent's scrape interval or minimum interval between collected samples. 
+1. vmagent routes incoming metrics to the given set of `vminsert` nodes using relabeling rules specified at `-remoteWrite.urlRelabelConfig` [configuration](https://docs.victoriametrics.com/victoriametrics/relabeling/).
 
-Metrics are routed to the appropriate `vminsert` group by [splitting data streams](https://docs.victoriametrics.com/victoriametrics/vmagent/#splitting-data-streams-among-multiple-systems) in `vmagent`. An optional [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) rule can be added on top to enforce per-tenant routing or API access policies.
+**Multi-Tenant Setup**
 
-## Implementing Multi-Retention on Kubernetes
-
-In this section, we'll install and configure the components for a multi-retention deployment of the VictoriaMetrics cluster. See [Kubernetes monitoring with VictoriaMetrics Cluster](https://docs.victoriametrics.com/guides/k8s-monitoring-via-vm-cluster/) for details on running VictoriaMetrics in Kubernetes.
-
-Run the following command to add the VictoriaMetrics Helm repository:
-
-```shell
-helm repo add vm https://victoriametrics.github.io/helm-charts/
-helm repo update
-```
-
-### Step 1: Deploying storage groups
-
-We'll create three retention groups. Each has a different retention period and disk size. Read [Understand Your Setup Size](https://docs.victoriametrics.com/guides/understand-your-setup-size/) to estimate how much space you will need for each group. The following table is shown as an example.
-
-| Group       | Retention Period | Disk Size |
-|-------------|------------------|-----------|
-| `vmcluster-a` | 3 months (`3M`)    | 80GB      |
-| `vmcluster-b` | 1 year (`1Y`)      | 300 GB    |
-| `vmcluster-c` | 3 years (`3Y`)     | 900 GB    |
-
-Create a Helm values file for Group A.
-
-```shell
-cat <<EOF > vmcluster-a.yaml
-vmstorage:
-  enabled: true
-  persistence:
-    size: 80Gi
-  extraArgs:
-    retentionPeriod: 3M
-    storageDataPath: /vmstorage-data
-    dedup.minScrapeInterval: 30s
-  podLabels:
-    retention-group: a
-    retention-period: 3M
-
-vminsert:
-  enabled: true
-  podLabels:
-    retention-group: a
-
-vmselect:
-  enabled: false
-EOF
-```
-
-The values file above creates `vminsert` and `vmstorage` services while turning off `vmselect`, which we'll deploy separately. It also defines a 30-second [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) window to handle possible duplicate metrics. The deduplication window must match the `vmagent` service scrape window (which we'll define later in the guide).
-
-Create the values files for Group B and Group C:
-
-```shell
-cat <<EOF > vmcluster-b.yaml
-vmstorage:
-  enabled: true
-  persistence:
-    size: 300Gi
-  extraArgs:
-    retentionPeriod: 1Y
-    storageDataPath: /vmstorage-data
-    dedup.minScrapeInterval: 30s
-  podLabels:
-    retention-group: b
-    retention-period: 1Y
-
-vminsert:
-  enabled: true
-  podLabels:
-    retention-group: b
-
-vmselect:
-  enabled: false
-EOF
-
-
-cat <<EOF > vmcluster-c.yaml
-vmstorage:
-  enabled: true
-  persistence:
-    size: 900Gi
-  extraArgs:
-    retentionPeriod: 3Y
-    storageDataPath: /vmstorage-data
-    dedup.minScrapeInterval: 30s
-  podLabels:
-    retention-group: c
-    retention-period: 3Y
-
-vminsert:
-  enabled: true
-  podLabels:
-    retention-group: c
-
-vmselect:
-  enabled: false
-EOF
-```
-
-Deploy the three storage groups with:
-
-```shell
-helm upgrade --install vmcluster-a vm/victoria-metrics-cluster -f vmcluster-a.yaml
-helm upgrade --install vmcluster-b vm/victoria-metrics-cluster -f vmcluster-b.yaml
-helm upgrade --install vmcluster-c vm/victoria-metrics-cluster -f vmcluster-c.yaml
-
-# Wait for all storage pods to be ready
-kubectl rollout status statefulset -l app.kubernetes.io/instance=vmcluster-a
-kubectl rollout status statefulset -l app.kubernetes.io/instance=vmcluster-b
-kubectl rollout status statefulset -l app.kubernetes.io/instance=vmcluster-c
-```
-
-### Step 2: Deploying vmselect
-
-Next, we'll deploy a `vmselect` service to route queries to the storage groups.
-
-Create a Helm values file with:
-
-```shell
-cat <<EOF >vmselect.yaml
-vmstorage:
-  enabled: false
-
-vminsert:
-  enabled: false
-
-vmselect:
-  enabled: true
-  replicaCount: 2
-  suppressStorageFQDNsRender: true
-  extraArgs:
-    # Each list item is a single -storageNode flag with comma-separated hosts
-    # in the same group. The FQDN format is:
-    #   <pod>.<svc>.default.svc
-    # where pod = <release>-victoria-metrics-cluster-vmstorage-<N>
-    # and   svc  = <release>-victoria-metrics-cluster-vmstorage
-    storageNode:
-      - "a/vmcluster-a-victoria-metrics-cluster-vmstorage-0.vmcluster-a-victoria-metrics-cluster-vmstorage.default.svc:8401,a/vmcluster-a-victoria-metrics-cluster-vmstorage-1.vmcluster-a-victoria-metrics-cluster-vmstorage.default.svc:8401"
-      - "b/vmcluster-b-victoria-metrics-cluster-vmstorage-0.vmcluster-b-victoria-metrics-cluster-vmstorage.default.svc:8401,b/vmcluster-b-victoria-metrics-cluster-vmstorage-1.vmcluster-b-victoria-metrics-cluster-vmstorage.default.svc:8401"
-      - "c/vmcluster-c-victoria-metrics-cluster-vmstorage-0.vmcluster-c-victoria-metrics-cluster-vmstorage.default.svc:8401,c/vmcluster-c-victoria-metrics-cluster-vmstorage-1.vmcluster-c-victoria-metrics-cluster-vmstorage.default.svc:8401"
-    dedup.minScrapeInterval: 30s
-EOF
-```
-
-Let's break down the file above:
-
-- Deploys `vmselect` as a separate Helm release 
-- Disables `vminsert` and `vmstorage` as these services were already deployed in Step 1.
-- `supressStorageFQDNsRender: true` turns off automatic FQDN generation for storage nodes. By default, the Helm chart auto-generates `-storageNodes` flags, but since `vmstorage` has been disabled, we need to supply them manually in `extraArgs`.
-- In `extraArgs.storageNode:` we define the list of `vmstorage` services to reach for queries. The `storageNode` flags tell vmselect to query all 6 storage pods, which are organized into three groups: `a`, `b`, and `c`.
-
-Deploy the `vmselect` release with:
-
-```shell
-helm upgrade --install vmselect vm/victoria-metrics-cluster -f vmselect.yaml
-```
-
-### Step 3: Deploying vmagent
-
-We'll use `vmagent` to route incoming metrics to the correct retention group. For example, we can use a `retention` label for mapping metrics to storage groups in the following way:
-
-| `retention` label | Storage Group |
-| ----------------| --------------|
-| `"3mo"`         | `vmcluster-a`   | 
-| `"1yr"`         | `vmcluster-b`   | 
-| `"3yr"`         | `vmcluster-c`   | 
-
-
-Create the values file for vmagent:
-
-```shell
-cat <<EOF >vmagent.yaml
-service:
-  enabled: true
-remoteWrite:
-  # Group A: receives metrics with retention="3mo"
-  - url: http://vmcluster-a-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write
-    urlRelabelConfig:
-      - action: keep
-        source_labels: [retention]
-        regex: "3mo"
-  # Group B: receives metrics with retention="1yr"
-  - url: http://vmcluster-b-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write
-    urlRelabelConfig:
-      - action: keep
-        source_labels: [retention]
-        regex: "1yr"
-  # Group C: receives metrics with retention="3yr"
-  - url: http://vmcluster-c-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write
-    urlRelabelConfig:
-      - action: keep
-        source_labels: [retention]
-        regex: "3yr"
-EOF
-```
-
-> Two important notes on scraping:
-> - Metrics without a matching `retention` label are silently dropped by the `keep` rules. You must ensure that every metric receives a label, or use a different routing configuration.
-> - The [scrape interval](https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs) should match the `dedup.minScrapeInterval` defined in the vmstorage nodes.
-`
-
-Now deploy the vmagent release:
-
-```shell
-helm upgrade --install vmagent vm/victoria-metrics-agent -f vmagent.yaml
-```
-
-Wait for vmagent to become ready:
-
-```shell
-kubectl rollout status deploy/vmagent-victoria-metrics-agent
-```
-
-### Step 4: Verification
-
-We can send test data to verify that data is flowing to the correct storage group.
-
-First, port-forward vmagent and vmselect:
-
-```shell
-VMAGENT_SVC=$(kubectl get svc -l app.kubernetes.io/instance=vmagent -o jsonpath='{.items[0].metadata.name}')
-kubectl port-forward "svc/$VMAGENT_SVC" 8429 &
-
-VMSELECT_SVC=$(kubectl get svc -l app.kubernetes.io/instance=vmselect -o jsonpath='{.items[0].metadata.name}')
-kubectl port-forward "svc/$VMSELECT_SVC" 8481 &
-```
-
-Send test metrics directly to vmagent's HTTP endpoint to exercise all three retention labels:
-
-```shell
-POD=$(kubectl get pod -l app.kubernetes.io/instance=vmagent -o jsonpath='{.items[0].metadata.name}')
-
-for retention in 3mo 1yr 3yr; do
-  kubectl exec "$POD" -- wget -qO- --post-data="test_routing{retention=\"${retention}\"} 1.0" \
-    "http://127.0.0.1:8429/api/v1/import/prometheus"
-done
-```
-
-Query the data back from vmselect (it may take around 30-60 seconds for new data to be available for queries):
-
-```shell
-for retention in 3mo 1yr 3yr; do
-  echo "-> retention=${retention}"
-  curl -s "http://localhost:8481/select/0/prometheus/api/v1/query" \
-    --data-urlencode "query=test_routing{retention=\"${retention}\"}"
-  echo
-done
-```
-
-You can also check that vmagent is forwarding data to all three groups:
-
-```shell
-curl -s http://localhost:8429/metrics | grep vmagent_remotewrite_blocks_sent_total
-```
-
-Each `url="N:secret-url"` corresponds to one `remoteWrite` entry (N=1 for Group A, N=2 for Group B, N=3 for Group C). Non-zero values confirm data is flowing.
-
-## Alternative Routing by Existing Labels
-
-The example setup above relies on a synthetic `retention` label to exist in every incoming metric.
-
-If having a `retention` label in every metric isn't practical, you can, as an alternative, rely on existing labels to map data to the correct storage group.
-
-The following example configures `vmagent` to route metrics based on the `environment` and `team` labels:
-
-```yaml
-# vmagent.yaml
-remoteWrite:
-  # send dev and staging data to Group A
-  - url: "http://vmcluster-a-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write"
-    urlRelabelConfig:
-      - action: keep
-        source_labels: [environment]
-        regex: "dev|staging"
-  # send prod data to Group B
-  - url: "http://vmcluster-b-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write"
-    urlRelabelConfig:
-      - action: keep
-        source_labels: [environment]
-        regex: "prod|production"
-  # send data from Infra and SRE teams to Group B
-  - url: "http://vmcluster-b-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write"
-    urlRelabelConfig:
-      - action: keep
-        source_labels: [team]
-        regex: "infra|sre"
-```
-
-> Metrics that do not match any of the `keep` rules are dropped in the configuration above.
-
-## Alternative Multi-Tenant Routing
-
-VictoriaMetrics Cluster supports [multiple isolated tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) identified by `accountID` (and optionally `projectID`) in the URL path, e.g., `/insert/0/prometheus/...`. 
-
-In a standard deployment, a single `vminsert` handles all tenants and distributes data across a shared pool of `vmstorage` nodes. In our current setup, each retention group deploys its own `vminsert`. This means tenant IDs are not required for routing, since data is already isolated by the URL that receives the write. 
-
-You can safely use a single tenant (`/insert/0/prometheus`) for all groups and rely on the `retention` label or label-based routing to separate data at query time.
-
-If, however, you prefer tenant-level separation for the query layer, you can assign each group a distinct tenant ID, for instance:
-
-| Group | Insert URL | Query URL |
-|-------|------------|-----------|
-| A (3mo) | `/insert/0/prometheus` | `/select/0/prometheus` |
-| B (1yr) | `/insert/1/prometheus` | `/select/1/prometheus` |
-| C (3yr) | `/insert/2/prometheus` | `/select/2/prometheus` |
-
-This lets you query a single retention group directly, e.g., `/select/1/prometheus/api/v1/query?query=up` returns only data written to group B's `vminsert`. Queries to vmselect without a tenant prefix aggregate across all groups, preserving the unified view for dashboards.
-
-The tenant path does not affect where data is stored (routing is always determined by which `vminsert` receives the data). The tenant ID is purely a query-scoping convenience in this architecture.
-
-## Additional Enhancements
-
-You can set up [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) to route data to the specified vminsert group based on the required retention.
+Every group of vmstorages can handle one tenant or multiple one. Different groups can have overlapping tenants. As vmselect reads from all vmstorage nodes, the data is aggregated on its level.
 
+**Additional Enhancements**
 
+You can set up [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) for routing data to the given vminsert group depending on the needed retention.

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -205,13 +205,15 @@ curl 'http://vmselect:8481/select/multitenant/prometheus/api/v1/query' \
 
 The precedence for applying filters for tenants follows this order:
 
-1. Filter tenants by `extra_label` and `extra_filters` filters.
+1. Filter tenants by `extra_label`, `extra_filters` and `extra_filters[]` filters.
  These filters have the highest priority and are applied first when provided through the query arguments.
+ Filters use `OR` logic - a tenant is selected if it matches any of the filters.
 2. Filter tenants from labels selectors defined at metricsQL query expression.
 
 > **Security considerations**
 It is recommended restricting access to `multitenant` endpoints only to trusted sources,
 since untrusted source may break per-tenant data by writing unwanted samples or get access to data of arbitrary tenants.
+See also [vmauth security doc](https://docs.victoriametrics.com/victoriametrics/vmauth/#security).
 
 ## Binaries
 

docs/victoriametrics/README.md

@@ -1136,6 +1136,8 @@ By default, the last point on the interval `[now - max_lookback ... now]` is scr
 For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
 with scrape intervals exceeding `5m`.
 
+ VictoriaMetrics supports Prometheus v3.0 utf-8 content encoding with `Accept` header. If `Accept: allow-utf-8` HTTP header provided, `/federate` API response changes according to [Prometheus utf-8](https://prometheus.io/docs/guides/utf8/#querying) specification - `metric_name{tag="value"}` transforms into `{"metric_name","tag"="value"}`.
+
 ## Capacity planning
 
 VictoriaMetrics uses lower amounts of CPU, RAM and storage space on production workloads compared to competing solutions (Prometheus, Thanos, Cortex, TimescaleDB, InfluxDB, QuestDB, M3DB) according to [our case studies](https://docs.victoriametrics.com/victoriametrics/casestudies/).

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,10 +26,29 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-opentelemetry.promoteAllResourceAttributes` and `-opentelemetry.promoteScopeMetadata` command-line flags to allow managing label promotion for resource attributes and OTel scope metadata. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#10931](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10931).
+## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
 
+Release candidate
+
+* SECURITY: upgrade Go builder from Go1.26.3 to Go1.26.4. See [the list of issues addressed in Go1.26.4](https://github.com/golang/go/issues?q=milestone%3AGo1.26.4%20label%3ACherryPickApproved).
+
+* FEATURE: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add the new metrics `vm_downsampling_partitions_scheduled_rows` and `vm_retention_filters_partitions_scheduled_rows` for measuring background historical data merge completion time. See [#10960](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10960)
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support `match[]=<label_selector>` query parameters in `/api/v1/rules` and `/api/v1/alerts` APIs to return only the rules that have configured labels satisfying the provided label selectors. See [11020](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11020).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-opentelemetry.promoteAllResourceAttributes` and `-opentelemetry.promoteScopeMetadata` command-line flags to allow managing label promotion for resource attributes and OTel scope metadata. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#10931](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10931).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) : introduce `vmagent_remotewrite_kafka_outbuf_latency_seconds` and  `vmagent_remotewrite_kafka_rtt_seconds` metrics for [kafka integration](https://docs.victoriametrics.com/victoriametrics/integrations/kafka/). The metrics could help identify throughput bottlenecks. See [#10730](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10730).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly log user information when a missing route error occurs. See [#11052](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11052).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl/): add the ability to migrate data from [Mimir](https://docs.victoriametrics.com/victoriametrics/vmctl/mimir/#) object storage to VictoriaMetrics. See [#7717](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7717). 
+* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): show the full `version` label in the `Version` panel when `short_version` label is empty (e.g. custom builds from feature branch). Previously, the panel could appear empty. See [#11047](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11047).
+
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix the `Notifiers` page in web UI appearing blank despite the API returning notifier data correctly. See [#11035](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11035).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): reset the group evaluation timestamp if it exceeds the current host time. Previously, vmalert could use future timestamps for evaluations if the system clock was shifted backward. See [#10985](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10985).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly parse [Prometheus Native Histograms](https://prometheus.io/docs/specs/native_histograms/), previously Protobuf parser could produce unexpected `vmrange` labels. See [#11041](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11041).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly calculate number of loaded users to be printed in startup log. Previously, it was only accounting for static users and skipped JWT configuration entries. See [#11050](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11050/).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): `integrate()` no longer extrapolates the last sample's value past the end of the time series. Previously, querying `integrate(metric[1h])` at a timestamp where the series had already ended would keep accruing area as if the last value continued indefinitely, producing values much larger than the true integral. See [#9474](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474). Thanks to @wtfashwin for contribution.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid returning HTTP 503 for queries with partial results when a storage group is unavailable and `-search.denyPartialResponse` is disabled. See [#11009](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11009). Thanks to @fxrlv for the contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `utf-8` label names for [/federate](https://docs.victoriametrics.com/victoriametrics/#federation) API requests. See [#10968](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10968).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the `Disable deduplication` toggle under its own local storage key. Before this fix, the toggle state was lost after reload and could overwrite the `Compact view` table setting. See [#11004](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11004). Thanks to @immanuwell for the contribution.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
 
 ## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
 

docs/victoriametrics/vmagent_common_flags.md

@@ -502,7 +502,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Supports array of values separated by comma or specified via multiple flags.
      Empty values are set to default value.
   -remoteWrite.roundDigits array
-     Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics (default 100)
+     Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.significantFigures (default 100)
      Supports array of values separated by comma or specified via multiple flags.
      Empty values are set to default value.
   -remoteWrite.sendTimeout array

docs/victoriametrics/vmalert.md

@@ -801,17 +801,15 @@ Please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriam
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
 
 * `http://<vmalert-addr>` - UI;
-* `http://<vmalert-addr>/api/v1/rules` - list of all loaded groups and rules. Supports `search`, `group_limit`, and `page_num` parameters, as well as additional [filtering](https://prometheus.io/docs/prometheus/latest/querying/api/#rules);
-* `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
-* `http://<vmalert-addr>/api/v1/notifiers` - list all available notifiers;
-* `http://<vmalert-addr>/vmalert/api/v1/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in JSON format.
-* `http://<vmalert-addr>/vmalert/api/v1/rule?group_id=<group_id>&rule_id=<rule_id>` - get rule status in JSON format.
-* `http://<vmalert-addr>/vmalert/api/v1/group?group_id=<group_id>` - get group status in JSON format.
-Used as alert source in AlertManager.
-* `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in web UI.
-* `http://<vmalert-addr>/vmalert/rule?group_id=<group_id>&rule_id=<rule_id>` - get rule status in web UI.
-* `http://<vmalert-addr>/vmalert/api/v1/rule?group_id=<group_id>&alert_id=<alert_id>` - get rule status in JSON format.
-* `http://<vmalert-addr>/metrics` - application metrics.
+* `http://<vmalert-addr>/api/v1/rules` - returns a list of all loaded groups and rules. Supports the `datasource_type`, `search`, `group_limit`, and `page_num` parameters, as well as additional [filtering](https://prometheus.io/docs/prometheus/latest/querying/api/#rules);
+* `http://<vmalert-addr>/api/v1/alerts` - returns a list of all active alerts. Supports the `datasource_type`, `rule_group[]`, `file[]` and `match[]`(applied on templated alert labels) query parameters;
+* `http://<vmalert-addr>/api/v1/notifiers` - returns a list of all available notifiers;
+* `http://<vmalert-addr>/vmalert/api/v1/alert?group_id=<group_id>&alert_id=<alert_id>` - returns the alert status in JSON format;
+* `http://<vmalert-addr>/vmalert/api/v1/rule?group_id=<group_id>&rule_id=<rule_id>` - returns the rule status in JSON format;
+* `http://<vmalert-addr>/vmalert/api/v1/group?group_id=<group_id>` - returns the group status in JSON format. Used as the alert source in AlertManager;
+* `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>` - displays the alert status in the web UI;
+* `http://<vmalert-addr>/vmalert/rule?group_id=<group_id>&rule_id=<rule_id>` - displays the rule status in the web UI;
+* `http://<vmalert-addr>/metrics` - application metrics endpoint;
 * `http://<vmalert-addr>/-/reload` - hot configuration reload.
 
 `vmalert` web UI can be accessed from [single-node version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)

docs/victoriametrics/vmauth.md

@@ -1536,6 +1536,16 @@ To enable TLS on the public listener while keeping the internal listener non-TLS
 
 `vmauth` also supports restricting access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
 
+
+When `vmauth` performs tenant routing for [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenant-reads) requests, it is crucial to explicitly set `extra_label`, `extra_filters` and `extra_filters[]` in the url_prefix configuration:
+
+```yaml
+unauthorized_user:
+    url_prefix: http://vmselect/select/multitenant?extra_filters[]=&extra_filters=&extra_label=vm_account_id=10&extra_label=vm_project_id=100
+```
+
+This is required because `vmselect` uses `OR` logic for tenant filtering. If a client sets `extra_filters[]` or `extra_filters`, it could bypass the tenant restriction configured via `extra_label`.
+
 ## Automatic issuing of TLS certificates
 
 `vmauth` [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) supports automatic issuing of TLS certificates via [Let's Encrypt service](https://letsencrypt.org/).

docs/victoriametrics/vmctl/mimir.md

@@ -73,4 +73,66 @@ ou can define it via the flag `--remote-read-headers=X-Scope-OrgID:demo`.
 
 See [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) for more details.
 
-See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
+See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
+
+### Read data from the remote storage like S3, GCS, Azure etc.
+
+If you have data stored in remote storage like S3, GCS, Azure etc. you can use `vmctl` in `mimir` mode to read data from 
+the remote storage and import it into VictoriaMetrics. In this mode `vmctl` reads data from the remote storage or file system
+and checks index file, define needed blocks to be processed. After it downloads blocks by defined filters and 
+use Prometheus converter to read and sent data to VictoriaMetrics.
+
+The following example shows how to read data from the file system and import it into VictoriaMetrics:
+
+```sh
+./vmctl mimir --mimir-path="fs:///mimir/test_data/mimir-tsdb" \                                               ? ? orbstack 
+  --mimir-tenant-id=anonymous \
+  --mimir-filter-time-start=2024-12-01T00:00:00 \
+  --mimir-filter-time-end=2024-12-18T23:59:59 \
+  --mimir-creds-file-path=creads \
+  --vm-concurrency=6 \
+  --mimir-concurrency=6 \
+  --vm-addr=http://localhost:8428/
+```
+This approach is useful when you have data stored on the local file system or you have a mounted volume, 
+download the data from the remote storage etc.
+
+The following example shows how to read data from the remote storage and import it into VictoriaMetrics:
+
+```sh
+./vmctl mimir --mimir-path="s3:///mimir-tsdb/anonymous" \
+  --mimir-filter-time-start=2024-12-01T00:00:00 \
+  --mimir-filter-time-end=2024-12-17T23:59:59 \
+  --mimir-creds-file-path=creads \
+  --mimir-custom-s3-endpoint='http://localhost:9000' \
+  --vm-concurrency=6 \
+  --mimir-concurrency=6 \
+  --vm-addr=http://localhost:8428/
+```
+
+In the example above we are used `--mimir-custom-s3-endpoint` flag to specify the custom S3 endpoint if it is needed.
+
+When the process finishes, you will see the following:
+
+```sh
+2025/01/18 13:01:59 Fetching blocks from remote storage
+Found 204 blocks to import. Continue? [Y/n] y
+VM worker 0:? 1589405 samples/s                                                                                                                                                                                     
+VM worker 1:? 1911834 samples/s                                                                                                                                                                                     
+VM worker 2:? 1849187 samples/s                                                                                                                                                                                     
+VM worker 3:? 1648820 samples/s                                                                                                                                                                                     
+VM worker 4:? 1539212 samples/s                                                                                                                                                                                     
+VM worker 5:? 1411485 samples/s                                                                                                                                                                                     
+Processing blocks: 204 / 204 [?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????] 100.00%
+2025/01/18 13:02:18 Import finished!
+2025/01/18 13:02:18 VictoriaMetrics importer stats:
+  idle duration: 18.485875611s;
+  time spent while importing: 16.40543875s;
+  total samples: 177961995;
+  samples/s: 10847743.71;
+  total bytes: 4.1 GB;
+  bytes/s: 248.2 MB;
+  import requests: 893;
+  import requests retries: 0;
+2025/01/18 13:02:18 Total time: 18.867547083s
+```

docs/victoriametrics/vmctl/vmctl_influx_flags.md

@@ -56,7 +56,7 @@ OPTIONS:
    --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
    --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
    --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
-   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
    --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
    --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
       By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

docs/victoriametrics/vmctl/vmctl_opentsdb_flags.md

@@ -51,7 +51,7 @@ OPTIONS:
    --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
    --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
    --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
-   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
    --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
    --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
       By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

docs/victoriametrics/vmctl/vmctl_prometheus_flags.md

@@ -44,7 +44,7 @@ OPTIONS:
    --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
    --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
    --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
-   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
    --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
    --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
       By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

docs/victoriametrics/vmctl/vmctl_remote-read_flags.md

@@ -59,7 +59,7 @@ OPTIONS:
    --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
    --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
    --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
-   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
    --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
    --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
       By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

go.mod

@@ -1,6 +1,6 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
-go 1.26.3
+go 1.26.4
 
 require (
 	cloud.google.com/go/storage v1.62.1
@@ -25,6 +25,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.22.0
 	github.com/influxdata/influxdb v1.12.4
 	github.com/klauspost/compress v1.18.5
+	github.com/oklog/ulid/v2 v2.1.1
 	github.com/prometheus/prometheus v0.311.3
 	github.com/urfave/cli/v2 v2.27.7
 	github.com/valyala/fastjson v1.6.10
@@ -109,7 +110,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/oklog/ulid/v2 v2.1.1 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.150.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.150.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.150.0 // indirect
@@ -155,7 +155,7 @@ require (
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/crypto v0.52.0 // indirect
 	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/term v0.43.0 // indirect

go.sum

@@ -509,8 +509,8 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
-golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
 golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
 golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

lib/prompb/write_request_unmarshaler.go

@@ -398,6 +398,10 @@ func (nhctx *nativeHistogramContext) appendTimeSeries(tss []TimeSeries, baseLabe
 	if baseName == "" {
 		return tss, labelsPool, samplesPool
 	}
+	originName := *nameValueP
+	defer func() {
+		*nameValueP = originName
+	}()
 
 	*nameValueP = fb.formatName(baseName, "_count")
 	tss, labelsPool, samplesPool = appendHistogramSeries(tss, labelsPool, samplesPool, baseLabels, "", tsMillis, count)
@@ -550,9 +554,11 @@ func (nhctx *nativeHistogramContext) reset() {
 	nhctx.zeroCountInt = 0
 	nhctx.zeroCountFloat = 0
 	nhctx.timestamp = 0
+	clear(nhctx.negativeSpans)
 	nhctx.negativeSpans = nhctx.negativeSpans[:0]
 	nhctx.negativeDeltas = nhctx.negativeDeltas[:0]
 	nhctx.negativeCounts = nhctx.negativeCounts[:0]
+	clear(nhctx.positiveSpans)
 	nhctx.positiveSpans = nhctx.positiveSpans[:0]
 	nhctx.positiveDeltas = nhctx.positiveDeltas[:0]
 	nhctx.positiveCounts = nhctx.positiveCounts[:0]

lib/prompb/write_request_unmarshaler_test.go

@@ -3,8 +3,9 @@ package prompb
 import (
 	"encoding/binary"
 	"math"
-	"reflect"
 	"testing"
+
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestUnmarshalTimeSeries(t *testing.T) {
@@ -18,8 +19,8 @@ func TestUnmarshalTimeSeries(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if !reflect.DeepEqual(tss, wantTSS) {
-			t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, wantTSS)
+		if diff := cmp.Diff(wantTSS, tss); len(diff) > 0 {
+			t.Fatalf("unexpected timeseries (-want, +got):\n%s", diff)
 		}
 	}
 
@@ -197,6 +198,93 @@ func TestUnmarshalTimeSeries(t *testing.T) {
 			},
 		})
 	}
+
+	{
+		// verify histogram fields are correctly reused
+		nativeHistogramC := nativeHistogramContext{
+			countInt:         0,
+			isCountFloat:     true,
+			countFloat:       2.5,
+			sum:              1.0,
+			schema:           1,
+			zeroThreshold:    0.00001,
+			isZeroCountFloat: true,
+			zeroCountFloat:   0.5,
+			timestamp:        3000,
+			positiveSpans:    []bucketSpan{{offset: 1, length: 2}},
+			positiveCounts:   []float64{1.5, 1.0},
+			negativeSpans:    []bucketSpan{{offset: 0, length: 1}},
+		}
+		nativeHistogramC2 := nativeHistogramContext{
+			countInt:         0,
+			isCountFloat:     true,
+			countFloat:       0,
+			sum:              1.0,
+			schema:           1,
+			zeroThreshold:    0.00001,
+			isZeroCountFloat: true,
+			zeroCountFloat:   0.5,
+			timestamp:        4000,
+			positiveSpans:    []bucketSpan{{offset: 0, length: 2}},
+			positiveCounts:   []float64{1.5, 1.0},
+			negativeSpans:    []bucketSpan{{offset: 0, length: 1}},
+			negativeCounts:   []float64{1.5, 0},
+		}
+		hd1 := encodeHistogram(nativeHistogramC)
+		hd2 := encodeHistogram(nativeHistogramC2)
+
+		src := encodeTimeSeries(
+			[]Label{{Name: "__name__", Value: "rpc_latency_seconds"}},
+			nil,
+			[][]byte{hd1, hd2},
+		)
+		f(src, []TimeSeries{
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_count"}},
+				Samples: []Sample{{Value: 2.5, Timestamp: 3000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_sum"}},
+				Samples: []Sample{{Value: 1.0, Timestamp: 3000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(-0.00001, 0.00001)}},
+				Samples: []Sample{{Value: 0.5, Timestamp: 3000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(1, 1.414)}},
+				Samples: []Sample{{Value: 1.5, Timestamp: 3000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(1.414, 2.0)}},
+				Samples: []Sample{{Value: 1.0, Timestamp: 3000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_count"}},
+				Samples: []Sample{{Value: 0, Timestamp: 4000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_sum"}},
+				Samples: []Sample{{Value: 1.0, Timestamp: 4000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(-0.00001, 0.00001)}},
+				Samples: []Sample{{Value: 0.5, Timestamp: 4000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(0.7071, 1)}},
+				Samples: []Sample{{Value: 1.5, Timestamp: 4000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(1, 1.414)}},
+				Samples: []Sample{{Value: 1.0, Timestamp: 4000}},
+			},
+			{
+				Labels:  []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(-1, -0.7071)}},
+				Samples: []Sample{{Value: 1.5, Timestamp: 4000}},
+			},
+		})
+	}
 }
 
 func encodeTimeSeries(labels []Label, samples []Sample, histograms [][]byte) []byte {

vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go

@@ -20,7 +20,7 @@ func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool
 func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte)
 
 var (
-	useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2
+	useAVX2 = cpu.X86.HasSSSE3 && cpu.X86.HasAVX2 && cpu.X86.HasBMI2
 )
 
 // setupState writes a ChaCha20 input matrix to state. See
@@ -47,7 +47,7 @@ func setupState(state *[16]uint32, key *[32]byte, nonce []byte) {
 }
 
 func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte {
-	if !cpu.X86.HasSSSE3 {
+	if !useAVX2 {
 		return c.sealGeneric(dst, nonce, plaintext, additionalData)
 	}
 
@@ -66,7 +66,7 @@ func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []
 }
 
 func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
-	if !cpu.X86.HasSSSE3 {
+	if !useAVX2 {
 		return c.openGeneric(dst, nonce, ciphertext, additionalData)
 	}
 

vendor/modules.txt

@@ -851,7 +851,7 @@ go.yaml.in/yaml/v2
 # go.yaml.in/yaml/v3 v3.0.4
 ## explicit; go 1.16
 go.yaml.in/yaml/v3
-# golang.org/x/crypto v0.51.0
+# golang.org/x/crypto v0.52.0
 ## explicit; go 1.25.0
 golang.org/x/crypto/chacha20
 golang.org/x/crypto/chacha20poly1305