fix code review remarks

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/build.yml

@@ -22,8 +22,7 @@ on:
       - '!app/vmui/**'
       - '.github/workflows/build.yml'
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   cancel-in-progress: true
@@ -32,6 +31,8 @@ concurrency:
 jobs:
   build:
     name: ${{ matrix.os }}-${{ matrix.arch }}
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

.github/workflows/changelog-linter.yml

@@ -5,8 +5,12 @@ on:
     paths:
       - "docs/victoriametrics/changelog/CHANGELOG.md"
 
+permissions: {}
+
 jobs:
   tip-lint:
+    permissions:
+      contents: read
     runs-on: 'ubuntu-latest'
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

.github/workflows/check-commit-signed.yml

@@ -3,8 +3,12 @@ name: check-commit-signed
 on:
   pull_request:
 
+permissions: {}
+
 jobs:
   check-commit-signed:
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

.github/workflows/check-licenses.yml

@@ -6,12 +6,14 @@ on:
   pull_request:
     paths:
       - 'vendor'
-permissions:
-  contents: read
+
+permissions: {}
 
 jobs:
   build:
     name: Build
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout

.github/workflows/codeql-analysis-go.yml

@@ -18,6 +18,8 @@ concurrency:
   cancel-in-progress: true
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
 
+permissions: {}
+
 jobs:
   analyze:
     name: Analyze

.github/workflows/docs.yaml

@@ -7,12 +7,14 @@ on:
       - 'docs/**'
       - '.github/workflows/docs.yaml'
   workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
+
+permissions: {}
+
 jobs:
   build:
     name: Build
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout

.github/workflows/test.yml

@@ -18,8 +18,7 @@ on:
       - 'go.*'
       - '.github/workflows/main.yml'
 
-permissions:
-  contents: read
+permissions: {}
 
 concurrency:
   cancel-in-progress: true
@@ -29,6 +28,8 @@ concurrency:
 jobs:
   lint:
     name: lint
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
@@ -61,6 +62,8 @@ jobs:
 
   unit:
     name: unit
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
 
     strategy:
@@ -90,6 +93,8 @@ jobs:
 
   apptest:
     name: apptest
+    permissions:
+      contents: read
     runs-on: apptest
 
     steps:

.github/workflows/vmui.yml

@@ -16,11 +16,7 @@ on:
       - 'app/vmui/packages/vmui/**'
       - '.github/workflows/vmui.yml'
 
-permissions:
-  contents: read
-  packages: read
-  pull-requests: read
-  checks: write
+permissions: {}
 
 concurrency:
   cancel-in-progress: true
@@ -29,6 +25,10 @@ concurrency:
 jobs:
   vmui-checks:
     name: VMUI Checks (lint, test, typecheck)
+    permissions:
+      checks: write
+      contents: read
+      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout

Makefile

@@ -293,8 +293,8 @@ apptest-legacy: vminsert-race vmselect-race vmstorage-race vmbackup-race vmresto
 		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
 		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
 	); \
-	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
-	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
+	VMSINGLE_V1_132_0_PATH=$${DIR}/victoria-metrics-prod \
+	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacyCluster.*"
 
 benchmark:

app/vmagent/main.go

@@ -118,6 +118,7 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	opentelemetry.Init()
 	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 
 	if promscrape.IsDryRun() {

app/vmagent/opentelemetry/request_handler.go

@@ -25,6 +25,11 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
+// Init must be called after flag.Parse and before using the opentelemetry package.
+func Init() {
+	stream.InitDecodeOptions()
+}
+
 // InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {

app/vmagent/remotewrite/remotewrite.go

@@ -79,7 +79,8 @@ var (
 		"writing them to remote storage. "+
 		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
 		"By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
-		"This option may be used for improving data compression for the stored metrics")
+		"This option may be used for improving data compression for the stored metrics. "+
+		"See also -remoteWrite.significantFigures")
 	sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
 		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+

app/vmalert/config/config.go

@@ -113,15 +113,15 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
 			if err := exprValidator(r.Expr); err != nil {
-				return fmt.Errorf("invalid expression for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid expression for rule %q: %w", ruleName, err)
 			}
 		}
 		if validateTplFn != nil {
 			if err := validateTplFn(r.Annotations); err != nil {
-				return fmt.Errorf("invalid annotations for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid annotations for rule %q: %w", ruleName, err)
 			}
 			if err := validateTplFn(r.Labels); err != nil {
-				return fmt.Errorf("invalid labels for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid labels for rule %q: %w", ruleName, err)
 			}
 		}
 	}

app/vmalert/config/config_test.go

@@ -121,7 +121,7 @@ func TestParse_Failure(t *testing.T) {
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
 	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
+	f([]string{"testdata/rules/rules1-bad.rules"}, "bad GraphiteQL expr")
 	f([]string{"testdata/rules/vlog-rules0-bad.rules"}, "bad LogsQL expr")
 	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
 	f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields")
@@ -283,7 +283,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "up | 0",
 			},
 		},
-	}, true, "bad prometheus expr")
+	}, true, "bad MetricsQL expr")
 
 	f(&Group{
 		Name: "test graphite expr",
@@ -293,7 +293,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				"description": "some-description",
 			}},
 		},
-	}, true, "bad graphite expr")
+	}, true, "bad GraphiteQL expr")
 
 	f(&Group{
 		Name: "test vlogs expr",
@@ -327,7 +327,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "sum(up == 0 ) by (host)",
 			},
 		},
-	}, true, "bad graphite expr")
+	}, true, "bad GraphiteQL expr")
 
 	f(&Group{
 		Name: "test vlogs with prometheus exp",
@@ -351,7 +351,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				For:    promutil.NewDuration(10 * time.Millisecond),
 			},
 		},
-	}, true, "bad prometheus expr")
+	}, true, "bad MetricsQL expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/config/types.go

@@ -66,11 +66,11 @@ func (t *Type) ValidateExpr(expr string) error {
 	switch t.String() {
 	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
-			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad GraphiteQL expr: %q, err: %w", expr, err)
 		}
 	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
-			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad MetricsQL expr: %q, err: %w", expr, err)
 		}
 	case "vlogs":
 		q, err := logstorage.ParseStatsQuery(expr, 0)

app/vmalert/rule/group.go

@@ -95,6 +95,7 @@ type groupMetrics struct {
 	iterationTotal    *metrics.Counter
 	iterationDuration *metrics.Summary
 	iterationMissed   *metrics.Counter
+	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
 }
 
@@ -330,6 +331,7 @@ func (g *Group) Init() {
 	g.metrics.iterationTotal = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
 	g.metrics.iterationDuration = g.metrics.set.NewSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
 	g.metrics.iterationMissed = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	g.metrics.iterationReset = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_reset_total{%s}`, labels))
 	g.metrics.iterationInterval = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
 		i := g.Interval.Seconds()
 		return i
@@ -474,14 +476,16 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 			if missed < 0 {
 				// missed can become < 0 due to irregular delays during evaluation
 				// which can result in time.Since(evalTS) < g.Interval;
-				// or the system wall clock was changed backward
-				missed = 0
+				// or the system wall clock was changed backward,
+				// Reset the evalTS to the current time.
 				evalTS = time.Now()
+				g.metrics.iterationReset.Inc()
+			} else {
+				evalTS = evalTS.Add((missed + 1) * g.Interval)
 			}
 			if missed > 0 {
 				g.metrics.iterationMissed.Inc()
 			}
-			evalTS = evalTS.Add((missed + 1) * g.Interval)
 
 			eval(evalCtx, evalTS)
 		}

app/vmalert/web.go

@@ -11,6 +11,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
@@ -160,12 +162,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
-		gf, err := newGroupsFilter(r)
+		af, err := newAlertsFilter(r)
 		if err != nil {
 			errJson(w, r, err)
 			return true
 		}
-		data, err := rh.listAlerts(gf)
+		data, err := rh.listAlerts(af)
 		if err != nil {
 			errJson(w, r, err)
 			return true
@@ -325,6 +327,48 @@ func (gf *groupsFilter) matches(group *rule.Group) bool {
 	return true
 }
 
+type alertsFilter struct {
+	gf    *groupsFilter
+	match [][]metricsql.LabelFilter
+}
+
+func getMatchFilters(matches []string) ([][]metricsql.LabelFilter, *httpserver.ErrorWithStatusCode) {
+	if len(matches) == 0 {
+		return nil, nil
+	}
+	tfss := make([][]metricsql.LabelFilter, 0, len(matches))
+	for _, s := range matches {
+		expr, err := metricsql.Parse(s)
+		if err != nil {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": failed to parse %q: %w`, s, err), http.StatusBadRequest)
+		}
+		me, ok := expr.(*metricsql.MetricExpr)
+		if !ok {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": expecting metricSelector; got %q`, expr.AppendString(nil)), http.StatusBadRequest)
+		}
+		if len(me.LabelFilterss) == 0 {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": labelFilterss cannot be empty`), http.StatusBadRequest)
+		}
+		tfss = append(tfss, me.LabelFilterss...)
+	}
+	return tfss, nil
+}
+
+func newAlertsFilter(r *http.Request) (*alertsFilter, *httpserver.ErrorWithStatusCode) {
+	gf, err := newGroupsFilter(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var af alertsFilter
+	af.gf = gf
+	af.match, err = getMatchFilters(r.Form["match[]"])
+	if err != nil {
+		return nil, err
+	}
+	return &af, nil
+}
+
 // see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
 type rulesFilter struct {
 	gf             *groupsFilter
@@ -335,6 +379,7 @@ type rulesFilter struct {
 	maxGroups      int
 	pageNum        int
 	search         string
+	match          [][]metricsql.LabelFilter
 	extendedStates bool
 }
 
@@ -355,7 +400,10 @@ func newRulesFilter(r *http.Request) (*rulesFilter, *httpserver.ErrorWithStatusC
 			return nil, errResponse(fmt.Errorf(`invalid parameter "type": not supported value %q`, ruleTypeParam), http.StatusBadRequest)
 		}
 	}
-
+	rf.match, err = getMatchFilters(r.Form["match[]"])
+	if err != nil {
+		return nil, err
+	}
 	states := vs["state"]
 	if len(states) == 0 {
 		states = vs["filter"]
@@ -416,12 +464,47 @@ func (rf *rulesFilter) matchesRule(r *rule.ApiRule) bool {
 	if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
 		return false
 	}
+	if !areLabelsMatch(r.Labels, rf.match) {
+		return false
+	}
 	if len(rf.states) == 0 {
 		return true
 	}
 	return slices.Contains(rf.states, r.State)
 }
 
+func areLabelsMatch(labels map[string]string, matches [][]metricsql.LabelFilter) bool {
+	if len(matches) == 0 {
+		return true
+	}
+	// labels need to match at least one of the provided match[] arg
+	return slices.ContainsFunc(matches, func(filters []metricsql.LabelFilter) bool {
+		for _, mf := range filters {
+			if !isLabelFilterMatch(labels[mf.Label], mf) {
+				return false
+			}
+		}
+		return true
+	})
+}
+
+func isLabelFilterMatch(s string, match metricsql.LabelFilter) bool {
+	if !match.IsRegexp {
+		if match.IsNegative {
+			return s != match.Value
+		}
+		return s == match.Value
+	}
+	re, err := metricsql.CompileRegexpAnchored(match.Value)
+	if err != nil {
+		return false
+	}
+	if match.IsNegative {
+		return !re.MatchString(s)
+	}
+	return re.MatchString(s)
+}
+
 func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
@@ -543,14 +626,14 @@ func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	return gAlerts
 }
 
-func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
+func (rh *requestHandler) listAlerts(af *alertsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
 	lr := listAlertsResponse{Status: "success"}
 	lr.Data.Alerts = make([]*rule.ApiAlert, 0)
 	for _, group := range rh.m.groups {
-		if !gf.matches(group) {
+		if !af.gf.matches(group) {
 			continue
 		}
 		g := group.ToAPI()
@@ -558,7 +641,11 @@ func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.Erro
 			if r.Type != rule.TypeAlerting {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, r.Alerts...)
+			for _, alert := range r.Alerts {
+				if areLabelsMatch(alert.Labels, af.match) {
+					lr.Data.Alerts = append(lr.Data.Alerts, alert)
+				}
+			}
 		}
 	}
 

app/vmalert/web.qtpl

@@ -348,7 +348,7 @@
                 typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
                 count := len(ns)
             %}
-            <div class="w-100 flex-column vm-group">
+            <div class="w-100 flex-column">
                 <span class="d-flex justify-content-between" id="group-{%s typeK %}">
                     <a href="#group-{%s typeK %}">{%s typeK %} ({%d count %})</a>
                     <span
@@ -361,7 +361,7 @@
                 <div id="item-{%s typeK %}" class="collapse show">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
-                            <tr class="vm-item">
+                            <tr>
                                 <th scope="col">Labels</th>
                                 <th scope="col">Address</th>
                             </tr>

app/vmalert/web.qtpl.go

@@ -1115,7 +1115,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
 
 //line app/vmalert/web.qtpl:350
 			qw422016.N().S(`
-            <div class="w-100 flex-column vm-group">
+            <div class="w-100 flex-column">
                 <span class="d-flex justify-content-between" id="group-`)
 //line app/vmalert/web.qtpl:352
 			qw422016.E().S(typeK)
@@ -1152,7 +1152,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
 			qw422016.N().S(`" class="collapse show">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
-                            <tr class="vm-item">
+                            <tr>
                                 <th scope="col">Labels</th>
                                 <th scope="col">Address</th>
                             </tr>

app/vmalert/web_test.go

@@ -10,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -37,12 +39,14 @@ func TestHandler(t *testing.T) {
 			Concurrency: 1,
 			Rules: []config.Rule{
 				{
-					ID:    0,
-					Alert: "alert",
+					ID:     0,
+					Alert:  "alert",
+					Labels: map[string]string{"job": "foo"},
 				},
 				{
 					ID:     1,
 					Record: "record",
+					Labels: map[string]string{"job": "bar"},
 				},
 			},
 		}, fq, 1*time.Minute, nil)
@@ -128,6 +132,18 @@ func TestHandler(t *testing.T) {
 		if length := len(lr.Data.Alerts); length != 2 {
 			t.Fatalf("expected 2 alert got %d", length)
 		}
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+`/api/v1/alerts?match[]={job="foo"}`, &lr, 200)
+		if length := len(lr.Data.Alerts); length != 3 {
+			t.Fatalf("expected 3 alerts got %d", length)
+		}
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+`/api/v1/alerts?match[]={job="bar"}`, &lr, 200)
+		if length := len(lr.Data.Alerts); length != 0 {
+			t.Fatalf("expected 0 alerts got %d", length)
+		}
 	})
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
 		expAlert := rule.NewAlertAPI(ar, ar.GetAlerts()[0])
@@ -242,6 +258,13 @@ func TestHandler(t *testing.T) {
 		check("/vmalert/api/v1/rules?datasource_type=graphite", 200, 1, 2)
 		check("/vmalert/api/v1/rules?datasource_type=graphiti", 400, 0, 0)
 
+		// invalid match[] params
+		check(`/vmalert/api/v1/rules?match[]={job=!"foo"}`, 400, 0, 0)
+		check(`/vmalert/api/v1/rules?match[]={job="foo"}`, 200, 3, 3)
+		check(`/vmalert/api/v1/rules?match[]={job="bar"}`, 200, 3, 3)
+		check(`/vmalert/api/v1/rules?match[]={job="bar"}&match[]={job="foo"}`, 200, 3, 6)
+		check(`/vmalert/api/v1/rules?match[]={job="barzz"}`, 200, 0, 0)
+
 		// no filtering expected due to bad params
 		check("/api/v1/rules?type=badParam", 400, 0, 0)
 		check("/api/v1/rules?foo=bar", 200, 3, 6)
@@ -367,3 +390,116 @@ func TestEmptyResponse(t *testing.T) {
 		}
 	})
 }
+
+func TestMatchesRule(t *testing.T) {
+	parseMatch := func(t *testing.T, selectors []string) [][]metricsql.LabelFilter {
+		t.Helper()
+		var match [][]metricsql.LabelFilter
+		for _, s := range selectors {
+			expr, err := metricsql.Parse(s)
+			if err != nil {
+				t.Fatalf("failed to parse selector %q: %v", s, err)
+			}
+			me, ok := expr.(*metricsql.MetricExpr)
+			if !ok {
+				t.Fatalf("expected MetricExpr for %q, got %T", s, expr)
+			}
+			match = append(match, me.LabelFilterss...)
+		}
+		return match
+	}
+
+	f := func(t *testing.T, selectors []string, labels map[string]string, wantMatch bool) {
+		t.Helper()
+		rf := &rulesFilter{
+			gf:    &groupsFilter{},
+			match: parseMatch(t, selectors),
+		}
+		r := &rule.ApiRule{Labels: labels}
+		got := rf.matchesRule(r)
+		if got != wantMatch {
+			t.Fatalf("matchesRule(%v) with selectors %v: got %v, want %v",
+				labels, selectors, got, wantMatch)
+		}
+	}
+
+	f(t, nil, map[string]string{"foo": "bar"}, true)
+
+	f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "bar"}, true)
+	f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "baz"}, false)
+
+	f(t, []string{`{foo="bar"}`}, map[string]string{"bar": "baz"}, false)
+	f(t, []string{`{foo=""}`}, map[string]string{"bar": "baz"}, true)
+
+	f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "baz"}, true)
+	f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "bar"}, false)
+
+	f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "bar"}, true)
+	f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "baz"}, false)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "baz"}, true)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "bar"}, true)
+	f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "foo"}, false)
+
+	f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "baz"}, true)
+	f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "bar"}, false)
+
+	// single match[] with multiple filters
+	f(t,
+		[]string{`{job="foo",instance="bar"}`},
+		map[string]string{"job": "foo", "instance": "bar"},
+		true,
+	)
+	f(t,
+		[]string{`{job="foo",instance="bar"}`},
+		map[string]string{"job": "other", "instance": "bar"},
+		false,
+	)
+
+	f(t,
+		[]string{`{foo="bar",baz=~"b.*"}`},
+		map[string]string{"foo": "bar", "baz": "bazinga"},
+		true,
+	)
+	f(t,
+		[]string{`{foo="bar",baz=~"b.*"}`},
+		map[string]string{"foo": "other", "baz": "bazinga"},
+		false,
+	)
+
+	// multiple matches[]
+	f(t,
+		[]string{`{foo="bar"}`, `{foo="baz"}`},
+		map[string]string{"foo": "baz"},
+		true,
+	)
+	f(t,
+		[]string{`{foo="bar"}`, `{foo="baz"}`},
+		map[string]string{"foo": "unknown"},
+		false,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"bar": "bazinga"},
+		true,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"foo": "bartender"},
+		true,
+	)
+	f(t,
+		[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
+		map[string]string{"foo": "other", "bar": "other"},
+		false,
+	)
+	f(t,
+		[]string{`{job="foo",instance="bar"}`, `{foo="bar"}`},
+		map[string]string{"foo": "bar"},
+		true,
+	)
+	f(t,
+		[]string{`{job="foo", instance="bar"}`, `{foo="bar"}`},
+		map[string]string{"instance": "barr", "job": "foo"},
+		false,
+	)
+}

app/vmauth/auth_config.go

@@ -889,7 +889,8 @@ func reloadAuthConfig() (bool, error) {
 	}
 
 	mp := authUsers.Load()
-	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
+	jwtc := jwtAuthCache.Load()
+	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp)+len(jwtc.users), *authConfigPath)
 	return true, nil
 }
 

app/vmauth/main.go

@@ -317,7 +317,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tk
 	defer ui.endConcurrencyLimit()
 
 	// Process the request.
-	processRequest(w, r, ui, tkn)
+	processRequest(w, r, ui, tkn, userName)
 }
 
 func beginConcurrencyLimit(ctx context.Context) error {
@@ -391,7 +391,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 	return bb, nil
 }
 
-func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
+func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token, userName string) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
 	isDefault := false
@@ -409,7 +409,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 			if ui.DumpRequestOnErrors {
 				di = debugInfo(u, r)
 			}
-			httpserver.Errorf(w, r, "missing route for %q%s", u.String(), di)
+			httpserver.Errorf(w, r, "user %s missing route for %q%s", userName, u.String(), di)
 			return
 		}
 		up, hc = ui.DefaultURL, ui.HeadersConf
@@ -455,7 +455,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
 		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), ui.name()),
+		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), userName),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)

app/vmauth/main_test.go

@@ -307,6 +307,24 @@ statusCode=200
 requested_url={BACKEND}/bar/a/b`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
+	// correct authorization but unexisted path, hence missing route error.
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_map:
+  - src_paths:
+    - "/api/v1/write"
+    url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:secret@some-host.com/a/b"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=400
+user foo missing route for "http://foo:secret@some-host.com/a/b"`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
 	// verify how path cleanup works
 	cfgStr = `
 unauthorized_user:
@@ -403,7 +421,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=400
-missing route for "http://some-host.com/abc?de=fg"`
+user unauthorized missing route for "http://some-host.com/abc?de=fg"`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
 	// missing default_url and default url_prefix for unauthorized user with dump_request_on_errors enabled
@@ -419,7 +437,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=400
-missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
+user unauthorized missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
 Pass-Header: abc
 Some-Header: foobar
 X-Forwarded-For: 12.34.56.78
@@ -461,7 +479,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
 	// all the backend_urls are unavailable for authorized user
@@ -501,7 +519,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 0 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 0 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	netutil.Resolver = origResolver
 
@@ -518,7 +536,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=502
-all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
+all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	if n := retries.Load(); n != 2 {
 		t.Fatalf("unexpected number of retries; got %d; want 2", n)

app/vmctl/flags.go

@@ -146,7 +146,8 @@ var (
 			Name:  vmRoundDigits,
 			Value: 100,
 			Usage: "Round metric values to the given number of decimal digits after the point. " +
-				"This option may be used for increasing on-disk compression level for the stored metrics",
+				"This option may be used for increasing on-disk compression level for the stored metrics. " +
+				"See also --vm-significant-figures option",
 		},
 		&cli.StringSliceFlag{
 			Name:  vmExtraLabel,
@@ -500,6 +501,96 @@ var (
 	}
 )
 
+const (
+	mimirPath             = "mimir-path"
+	mimirTenantID         = "mimir-tenant-id"
+	mimirConcurrency      = "mimir-concurrency"
+	mimirFilterTimeStart  = "mimir-filter-time-start"
+	mimirFilterTimeEnd    = "mimir-filter-time-end"
+	mimirFilterLabel      = "mimir-filter-label"
+	mimirFilterLabelValue = "mimir-filter-label-value"
+
+	mimirCredsFilePath           = "mimir-creds-file-path"
+	mimirConfigFilePath          = "mimir-config-file-path"
+	mimirConfigProfile           = "mimir-config-profile"
+	mimirCustomS3Endpoint        = "mimir-custom-s3-endpoint"
+	mimirS3ForcePathStyle        = "mimir-s3-force-path-style"
+	mimirS3TLSInsecureSkipVerify = "mimir-s3-tls-insecure-skip-verify"
+	mimirSSEKMSKeyID             = "mimir-s3-sse-kms-key-id"
+	mimirSSEAlgorithm            = "mimir-s3-sse-algorithm"
+)
+
+var (
+	mimirFlags = []cli.Flag{
+		&cli.StringFlag{
+			Name:     mimirPath,
+			Usage:    "Path to Mimir storage bucket or local folder.",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:  mimirTenantID,
+			Usage: "Tenant ID for Mimir storage",
+		},
+		&cli.IntFlag{
+			Name:  mimirConcurrency,
+			Usage: "Number of concurrently running block readers",
+			Value: 1,
+		},
+		&cli.StringFlag{
+			Name:     mimirFilterTimeStart,
+			Usage:    "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     mimirFilterTimeEnd,
+			Usage:    "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:  mimirFilterLabel,
+			Usage: "Mimir label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
+		},
+		&cli.StringFlag{
+			Name:  mimirFilterLabelValue,
+			Usage: fmt.Sprintf("Regular expression to filter label from %q flag.", mimirFilterLabel),
+			Value: ".*",
+		},
+		&cli.StringFlag{
+			Name:  mimirCredsFilePath,
+			Usage: "Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
+		},
+		&cli.StringFlag{
+			Name:  mimirConfigFilePath,
+			Usage: "Path to file with S3 configs. Configs are loaded from default location if not set. See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
+		},
+		&cli.StringFlag{
+			Name:  mimirConfigProfile,
+			Usage: "Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used",
+		},
+		&cli.StringFlag{
+			Name:  mimirCustomS3Endpoint,
+			Usage: "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set",
+		},
+		&cli.BoolFlag{
+			Name:  mimirS3ForcePathStyle,
+			Usage: "Prefixing endpoint with bucket name when set false, true by default.",
+			Value: true,
+		},
+		&cli.BoolFlag{
+			Name:  mimirS3TLSInsecureSkipVerify,
+			Usage: "Whether to skip TLS verification when connecting to the S3 endpoint.",
+		},
+		&cli.StringFlag{
+			Name:  mimirSSEKMSKeyID,
+			Usage: "SSE KMS Key ID for use with S3-compatible storages.",
+		},
+		&cli.StringFlag{
+			Name:  mimirSSEAlgorithm,
+			Usage: "SSE algorithm for use with S3-compatible storages.",
+		},
+	}
+)
+
 const (
 	vmNativeFilterMatch       = "vm-native-filter-match"
 	vmNativeFilterTimeStart   = "vm-native-filter-time-start"

app/vmctl/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/mimir"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -297,12 +298,12 @@ func main() {
 				},
 			},
 			{
-				Name:   "thanos",
-				Usage:  "Migrate time series from Thanos blocks (supports raw and downsampled data)",
-				Flags:  mergeFlags(globalFlags, thanosFlags, vmFlags),
+				Name:   "mimir",
+				Usage:  "Migrate time series from Mimir object storage or local filesystem",
+				Flags:  mergeFlags(globalFlags, mimirFlags, vmFlags),
 				Before: beforeFn,
 				Action: func(c *cli.Context) error {
-					fmt.Println("Thanos import mode")
+					fmt.Println("Mimir import mode")
 
 					vmCfg, err := initConfigVM(c)
 					if err != nil {
@@ -314,6 +315,54 @@ func main() {
 						return fmt.Errorf("failed to create VM importer: %s", err)
 					}
 
+					mCfg := mimir.Config{
+						Filter: mimir.Filter{
+							TimeMin:    c.String(mimirFilterTimeStart),
+							TimeMax:    c.String(mimirFilterTimeEnd),
+							Label:      c.String(mimirFilterLabel),
+							LabelValue: c.String(mimirFilterLabelValue),
+						},
+						Path:                    c.String(mimirPath),
+						TenantID:                c.String(mimirTenantID),
+						CredsFilePath:           c.String(mimirCredsFilePath),
+						ConfigFilePath:          c.String(mimirConfigFilePath),
+						ConfigProfile:           c.String(mimirConfigProfile),
+						CustomS3Endpoint:        c.String(mimirCustomS3Endpoint),
+						S3ForcePathStyle:        c.Bool(mimirS3ForcePathStyle),
+						S3TLSInsecureSkipVerify: c.Bool(mimirS3TLSInsecureSkipVerify),
+						SSEKMSKeyID:             c.String(mimirSSEKMSKeyID),
+						SSEAlgorithm:            c.String(mimirSSEAlgorithm),
+					}
+					cl, err := mimir.NewClient(ctx, mCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create mimir client: %s", err)
+					}
+
+					pp := prometheusProcessor{
+						cl:        cl,
+						im:        importer,
+						cc:        c.Int(mimirConcurrency),
+						isVerbose: c.Bool(globalVerbose),
+					}
+					return pp.run(ctx)
+				},
+			},
+			{
+				Name:   "thanos",
+				Usage:  "Migrate time series from Thanos blocks (supports raw and downsampled data)",
+				Flags:  mergeFlags(globalFlags, thanosFlags, vmFlags),
+				Before: beforeFn,
+				Action: func(c *cli.Context) error {
+					fmt.Println("Thanos import mode")
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
+
+					importer, err = vm.NewImporter(ctx, vmCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create VM importer: %s", err)
+					}
 					thanosCfg := thanos.Config{
 						Snapshot: c.String(thanosSnapshot),
 						Filter: thanos.Filter{

app/vmctl/mimir/lazyreader.go

@@ -0,0 +1,195 @@
+package mimir
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/oklog/ulid/v2"
+	"github.com/prometheus/prometheus/tsdb"
+	"github.com/prometheus/prometheus/tsdb/tombstones"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+var _ tsdb.BlockReader = (*lazyBlockReader)(nil)
+
+// lazyBlockReader is stores block id and segment num information.
+// It is used to lazily fetch and parse block data.
+// It implements tsdb.BlockReader interface.
+type lazyBlockReader struct {
+	// Block ID.
+	ID ulid.ULID
+	// SegmentsNum stores the number of chunks segments in the block.
+	SegmentsNum int
+
+	mu          sync.Mutex
+	reader      *tsdb.Block
+	tempDirPath string
+	fs          common.RemoteFS
+	err         error
+}
+
+// newLazyBlockReader returns a new LazyBlockReader for the given block.
+func newLazyBlockReader(block *Block, fs common.RemoteFS) (*lazyBlockReader, error) {
+	if block.SegmentsFormat != "1b6d" {
+		return nil, fmt.Errorf("unsupported segments format: %s", block.SegmentsFormat)
+	}
+
+	return &lazyBlockReader{
+		ID:          block.ID,
+		SegmentsNum: block.SegmentsNum,
+		fs:          fs,
+	}, nil
+}
+
+func (lbr *lazyBlockReader) initialize() error {
+	lbr.mu.Lock()
+	defer lbr.mu.Unlock()
+	if lbr.reader != nil {
+		return nil
+	}
+	// fetching block and parse it and store it in lbr.reader
+	temp, err := lbr.mkTempDir()
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %s", err)
+	}
+
+	lbr.tempDirPath = temp
+
+	// TODO: replace fetchFile and writeFile with buffered IO if needed
+	meta, err := lbr.fetchFile(metaFilename)
+	if err != nil {
+		return err
+	}
+	if err := lbr.writeFile(temp, metaFilename, meta); err != nil {
+		return fmt.Errorf("failed to write meta file: %w", err)
+	}
+	idx, err := lbr.fetchFile(indexFilename)
+	if err != nil {
+		return fmt.Errorf("failed to fetch index file %q: %w", indexFilename, err)
+	}
+	if err := lbr.writeFile(temp, indexFilename, idx); err != nil {
+		return err
+	}
+
+	for i := 1; i <= lbr.SegmentsNum; i++ {
+		// segments formats has format 1b06d
+		// https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L32
+		chunkName := fmt.Sprintf("%06d", i)
+		blockChunkPath := filepath.Join("chunks", chunkName)
+		chunk, err := lbr.fetchFile(blockChunkPath)
+		if err != nil {
+			return fmt.Errorf("failed to fetch chunk file: %q: %w", chunkName, err)
+		}
+		if err := lbr.writeFile(temp, blockChunkPath, chunk); err != nil {
+			return fmt.Errorf("failed to write chunk file: %q: %s", chunkName, err)
+		}
+	}
+
+	// Set postingDecoder to nil because
+	// If it is nil then a default decoder is used, compatible with Prometheus v2.
+	pb, err := tsdb.OpenBlock(nil, temp, nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to open block %q: %w", lbr.ID, err)
+	}
+	lbr.reader = pb
+	return nil
+}
+
+// Index returns an IndexReader over the block's data.
+func (lbr *lazyBlockReader) Index() (tsdb.IndexReader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Index()
+}
+
+// Chunks returns a ChunkReader over the block's data.
+func (lbr *lazyBlockReader) Chunks() (tsdb.ChunkReader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Chunks()
+}
+
+// Tombstones returns a tombstones.Reader over the block's deleted data.
+func (lbr *lazyBlockReader) Tombstones() (tombstones.Reader, error) {
+	if err := lbr.initialize(); err != nil {
+		return nil, err
+	}
+	return lbr.reader.Tombstones()
+}
+
+// Meta provides meta information about the block reader.
+func (lbr *lazyBlockReader) Meta() tsdb.BlockMeta {
+	if err := lbr.initialize(); err != nil {
+		lbr.err = fmt.Errorf("cannot get BlockMeta: %w", err)
+		return tsdb.BlockMeta{}
+	}
+	return lbr.reader.Meta()
+}
+
+// Size returns the number of bytes that the block takes up on disk.
+func (lbr *lazyBlockReader) Size() int64 {
+	if err := lbr.initialize(); err != nil {
+		lbr.err = fmt.Errorf("error get Size of the block: %s, return zero size", err)
+		return 0
+	}
+	return lbr.reader.Size()
+}
+
+// Err returns the last error that occurred on the block reader.
+func (lbr *lazyBlockReader) Err() error {
+	return lbr.err
+}
+
+// Close closes block and releases all resources
+func (lbr *lazyBlockReader) Close() error {
+	lbr.mu.Lock()
+	defer lbr.mu.Unlock()
+	if lbr.reader == nil {
+		return nil
+	}
+
+	err := lbr.reader.Close()
+	if err := os.RemoveAll(lbr.tempDirPath); err != nil {
+		log.Printf("failed to remove temp dir: %s", err)
+	}
+	lbr.reader = nil
+	lbr.tempDirPath = ""
+
+	return err
+}
+
+func (lbr *lazyBlockReader) mkTempDir() (string, error) {
+	temp, err := os.MkdirTemp("", lbr.ID.String())
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp dir: %s", err)
+	}
+	err = os.Mkdir(filepath.Join(temp, "chunks"), os.ModePerm)
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp dir: %s", err)
+	}
+	return temp, nil
+}
+
+func (lbr *lazyBlockReader) fetchFile(filePath string) ([]byte, error) {
+	blockID := lbr.ID.String()
+	blockPath := filepath.Join(blockID, filePath)
+	has, err := lbr.fs.HasFile(blockPath)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, fmt.Errorf("block meta %s not found", blockID)
+	}
+	return lbr.fs.ReadFile(blockPath)
+}
+
+func (lbr *lazyBlockReader) writeFile(folder string, filename string, file []byte) error {
+	fileName := filepath.Join(folder, filename)
+	return os.WriteFile(fileName, file, os.ModePerm)
+}

app/vmctl/mimir/mimir.go

@@ -0,0 +1,238 @@
+package mimir
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/oklog/ulid/v2"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/tsdb"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
+	utils "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+)
+
+const (
+	bucketIndex                   = "bucket-index.json"
+	bucketIndexCompressedFilename = bucketIndex + ".gz"
+	metaFilename                  = "meta.json"
+	indexFilename                 = "index"
+)
+
+// BlockDeletionMark holds the information about a block's deletion mark in the index.
+// This type was copied from the mimir repository https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L234.
+type BlockDeletionMark struct {
+	// Block ID.
+	ID ulid.ULID `json:"block_id"`
+
+	// DeletionTime is a unix timestamp (seconds precision) of when the block was marked to be deleted.
+	DeletionTime int64 `json:"deletion_time"`
+}
+
+// Block holds the information about a block in the index.
+// This is a partial implementation of the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L73
+type Block struct {
+	// Block ID.
+	ID ulid.ULID `json:"block_id"`
+
+	// MinTime and MaxTime specify the time range all samples in the block are in (millis precision).
+	MinTime int64 `json:"min_time"`
+	MaxTime int64 `json:"max_time"`
+
+	// SegmentsFormat and SegmentsNum stores the format and number of chunks segments
+	// in the block.
+	SegmentsFormat string `json:"segments_format,omitempty"`
+	SegmentsNum    int    `json:"segments_num,omitempty"`
+}
+
+// Index contains all known blocks and markers of a tenant.
+// This is a partial implementation pof the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L36
+type Index struct {
+	// Version of the index format.
+	Version int `json:"version"`
+
+	// List of complete blocks (partial blocks are excluded from the index).
+	Blocks []*Block `json:"blocks"`
+}
+
+// Config contains a list of params needed
+// for reading mimir snapshots
+type Config struct {
+	// Path to remote storage bucket
+	Path string
+	// TenantID is the tenant id for the storage
+	TenantID string
+
+	Filter Filter
+
+	CredsFilePath           string
+	ConfigFilePath          string
+	ConfigProfile           string
+	CustomS3Endpoint        string
+	S3ForcePathStyle        bool
+	S3TLSInsecureSkipVerify bool
+
+	SSEKMSKeyID  string
+	SSEAlgorithm string
+}
+
+// Filter contains configuration for filtering
+// the timeseries
+type Filter struct {
+	TimeMin    string
+	TimeMax    string
+	Label      string
+	LabelValue string
+}
+
+// Client is a wrapper over Prometheus tsdb.DBReader
+type Client struct {
+	common.RemoteFS
+	filter filter
+}
+
+type filter struct {
+	min, max   int64
+	label      string
+	labelValue string
+}
+
+func (f filter) inRange(minTime, maxTime int64) bool {
+	fmin, fmax := f.min, f.max
+	if minTime == 0 {
+		fmin = minTime
+	}
+	if fmax == 0 {
+		fmax = maxTime
+	}
+	return minTime <= fmax && fmin <= maxTime
+}
+
+// NewClient creates and validates new Client
+// with given Config
+func NewClient(ctx context.Context, cfg Config) (*Client, error) {
+	if cfg.Path == "" {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+
+	if cfg.TenantID != "" {
+		cfg.Path = fmt.Sprintf("%s/%s", cfg.Path, cfg.TenantID)
+	}
+
+	var c Client
+	rfs, err := newRemoteFS(ctx, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-src`=%q: %w", cfg.Path, err)
+	}
+
+	c.RemoteFS = rfs
+	timeMin, err := utils.ParseTime(cfg.Filter.TimeMin)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse min time in filter: %s", err)
+	}
+	timeMax, err := utils.ParseTime(cfg.Filter.TimeMax)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse max time in filter: %s", err)
+	}
+	c.filter = filter{
+		min:        timeMin.UnixMilli(),
+		max:        timeMax.UnixMilli(),
+		label:      cfg.Filter.Label,
+		labelValue: cfg.Filter.LabelValue,
+	}
+	return &c, nil
+}
+
+// Explore a fetches bucket-index.json file from a remote storage or local filesystem
+// and filter blocks via the defined time range, but does not take into account label filters.
+func (c *Client) Explore() ([]tsdb.BlockReader, error) {
+
+	log.Printf("Fetching blocks from remote storage")
+
+	indexFile, err := c.fetchIndexFile()
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch index file: %s", err)
+	}
+
+	var blocksToImport []tsdb.BlockReader
+	for _, block := range indexFile.Blocks {
+		if !c.filter.inRange(block.MinTime, block.MaxTime) {
+			// Skipping block outside of time range
+			continue
+		}
+
+		if block.ID.String() == "" {
+			continue
+		}
+
+		lazyBlockReader, err := newLazyBlockReader(block, c.RemoteFS)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create lazy block reader: %s", err)
+		}
+		blocksToImport = append(blocksToImport, lazyBlockReader)
+	}
+
+	return blocksToImport, nil
+}
+
+// Read reads the given BlockReader according to configured
+// time and label filters.
+func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error) {
+	meta := block.Meta()
+	if b, ok := block.(*lazyBlockReader); ok && b.Err() != nil {
+		return nil, fmt.Errorf("failed to read block: %s", b.Err())
+	}
+
+	if meta.ULID.String() == "" {
+		return nil, fmt.Errorf("unexpected block without id")
+	}
+
+	minTime, maxTime := meta.MinTime, meta.MaxTime
+	if c.filter.min != 0 {
+		minTime = c.filter.min
+	}
+	if c.filter.max != 0 {
+		maxTime = c.filter.max
+	}
+	q, err := tsdb.NewBlockQuerier(block, minTime, maxTime)
+	if err != nil {
+		return nil, err
+	}
+	ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
+	return &prometheus.CloseableSeriesSet{SeriesSet: ss, Close: q.Close}, nil
+}
+
+func (c *Client) fetchIndexFile() (*Index, error) {
+	has, err := c.HasFile(bucketIndexCompressedFilename)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, fmt.Errorf("bucket-index.json.gz not found")
+	}
+
+	file, err := c.ReadFile(bucketIndexCompressedFilename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read bucket index: %s", err)
+	}
+
+	r := bytes.NewReader(file)
+	// Read all the content.
+	gzipReader, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create gzip reader: %s", err)
+	}
+
+	var indexFile Index
+	err = json.NewDecoder(gzipReader).Decode(&indexFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode bucket index: %s", err)
+	}
+
+	return &indexFile, nil
+}

app/vmctl/mimir/remotefs.go

@@ -0,0 +1,93 @@
+package mimir
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/azremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
+)
+
+// newRemoteFS returns new remote fs from the given Config.
+func newRemoteFS(ctx context.Context, cfg Config) (common.RemoteFS, error) {
+	if len(cfg.Path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	n := strings.Index(cfg.Path, "://")
+	if n < 0 {
+		return nil, fmt.Errorf("missing scheme in path %q. Supported schemes: `gs://`, `s3://`, `azblob://`, `fs://`", cfg.Path)
+	}
+	scheme := cfg.Path[:n]
+	dir := cfg.Path[n+len("://"):]
+	switch scheme {
+	case "fs":
+		if !filepath.IsAbs(dir) {
+			return nil, fmt.Errorf("dir must be absolute; got %q", dir)
+		}
+		fsr := &fsremote.FS{
+			Dir: filepath.Clean(dir),
+		}
+		return fsr, nil
+	case "gcs", "gs":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &gcsremote.FS{
+			CredsFilePath: cfg.CredsFilePath,
+			Bucket:        bucket,
+			Dir:           dir,
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to gcs: %w", err)
+		}
+		return fsr, nil
+	case "azblob":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the AZBlob container %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &azremote.FS{
+			Container: bucket,
+			Dir:       dir,
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to AZBlob: %w", err)
+		}
+		return fsr, nil
+	case "s3":
+		n := strings.Index(dir, "/")
+		if n < 0 {
+			return nil, fmt.Errorf("missing directory on the s3 bucket %q", dir)
+		}
+		bucket := dir[:n]
+		dir = dir[n:]
+		fsr := &s3remote.FS{
+			CredsFilePath:         cfg.CredsFilePath,
+			ConfigFilePath:        cfg.ConfigFilePath,
+			CustomEndpoint:        cfg.CustomS3Endpoint,
+			TLSInsecureSkipVerify: cfg.S3TLSInsecureSkipVerify,
+			S3ForcePathStyle:      cfg.S3ForcePathStyle,
+			ProfileName:           cfg.ConfigProfile,
+			Bucket:                bucket,
+			Dir:                   dir,
+			SSEKMSKeyId:           cfg.SSEKMSKeyID,
+			SSEAlgorithm:          s3remote.StringToEncryptionAlgorithm(cfg.SSEAlgorithm),
+		}
+		if err := fsr.Init(ctx); err != nil {
+			return nil, fmt.Errorf("cannot initialize connection to s3: %w", err)
+		}
+		return fsr, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q", scheme)
+	}
+}

app/vmctl/prometheus.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
 	"strings"
 	"sync"
@@ -18,10 +19,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 )
 
+// Runner is an interface for fetching and reading
+// snapshot blocks
+type Runner interface {
+	Explore() ([]tsdb.BlockReader, error)
+	Read(context.Context, tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error)
+}
+
 type prometheusProcessor struct {
-	// prometheus client fetches and reads
+	// Runner fetches and reads
 	// snapshot blocks
-	cl *prometheus.Client
+	cl Runner
 	// importer performs import requests
 	// for timeseries data returned from
 	// snapshot blocks
@@ -48,7 +56,7 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
-	if err := pp.processBlocks(blocks); err != nil {
+	if err := pp.processBlocks(ctx, blocks); err != nil {
 		return fmt.Errorf("migration failed: %s", err)
 	}
 
@@ -57,11 +65,17 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
 	return nil
 }
 
-func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
-	ss, err := pp.cl.Read(b)
+func (pp *prometheusProcessor) do(ctx context.Context, b tsdb.BlockReader) error {
+	css, err := pp.cl.Read(ctx, b)
 	if err != nil {
 		return fmt.Errorf("failed to read block: %s", err)
 	}
+	defer func() {
+		if err := css.Close(); err != nil {
+			log.Printf("cannot close SeriesSet for block: %q : %s\n", b.Meta().ULID, err)
+		}
+	}()
+	ss := css.SeriesSet
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
@@ -114,7 +128,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	return ss.Err()
 }
 
-func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
+func (pp *prometheusProcessor) processBlocks(ctx context.Context, blocks []tsdb.BlockReader) error {
 	promBlocksTotal.Add(len(blocks))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
 	if err := barpool.Start(); err != nil {
@@ -130,11 +144,16 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	for range pp.cc {
 		wg.Go(func() {
 			for br := range blockReadersCh {
-				if err := pp.do(br); err != nil {
+				if err := pp.do(ctx, br); err != nil {
 					promErrorsTotal.Inc()
-					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
+					errCh <- fmt.Errorf("cannot read block %q: %s", br.Meta().ULID, err)
 					return
 				}
+				if cb, ok := br.(io.Closer); ok {
+					if err := cb.Close(); err != nil {
+						errCh <- fmt.Errorf("cannot close block: %q: %w", br.Meta().ULID, err)
+					}
+				}
 				promBlocksProcessed.Inc()
 				bar.Increment()
 			}

app/vmctl/prometheus/prometheus.go

@@ -8,6 +8,8 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/tsdb"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
 )
 
 // Config contains a list of params needed
@@ -60,13 +62,13 @@ func NewClient(cfg Config) (*Client, error) {
 		return nil, fmt.Errorf("failed to open snapshot %q: %s", cfg.Snapshot, err)
 	}
 	c := &Client{DBReadOnly: db}
-	minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
+	timeMin, timeMax, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse time in filter: %s", err)
 	}
 	c.filter = filter{
-		min:        minTime,
-		max:        maxTime,
+		min:        timeMin,
+		max:        timeMax,
 		label:      cfg.Filter.Label,
 		labelValue: cfg.Filter.LabelValue,
 	}
@@ -83,7 +85,7 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch blocks: %s", err)
 	}
-	s := &Stats{
+	s := &vmctlutil.Stats{
 		Filtered: c.filter.min != 0 || c.filter.max != 0 || c.filter.label != "",
 		Blocks:   len(blocks),
 	}
@@ -108,9 +110,15 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
 	return blocksToImport, nil
 }
 
+// CloseableSeriesSet defines a SeriesSet with Close method
+type CloseableSeriesSet struct {
+	SeriesSet storage.SeriesSet
+	Close     func() error
+}
+
 // Read reads the given BlockReader according to configured
 // time and label filters.
-func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
+func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*CloseableSeriesSet, error) {
 	minTime, maxTime := block.Meta().MinTime, block.Meta().MaxTime
 	if c.filter.min != 0 {
 		minTime = c.filter.min
@@ -122,8 +130,8 @@ func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
 	if err != nil {
 		return nil, err
 	}
-	ss := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
-	return ss, nil
+	ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
+	return &CloseableSeriesSet{ss, q.Close}, nil
 }
 
 func parseTime(start, end string) (int64, int64, error) {

app/vmctl/vmctlutil/stats.go

@@ -1,4 +1,4 @@
-package prometheus
+package vmctlutil
 
 import (
 	"fmt"
@@ -18,7 +18,7 @@ type Stats struct {
 
 // String returns string representation for s.
 func (s Stats) String() string {
-	str := fmt.Sprintf("Prometheus snapshot stats:\n"+
+	str := fmt.Sprintf("Snapshot stats:\n"+
 		"  blocks found: %d;\n"+
 		"  blocks skipped by time filter: %d;\n"+
 		"  min time: %d (%v);\n"+

app/vminsert/clusternative/server.go

@@ -26,7 +26,7 @@ var (
 // NewVMinsertServer creates and start vminsert server at the given addr
 func NewVMinsertServer(addr string, tc *tls.Config) (*vminsertapi.VMInsertServer, error) {
 	api := &vminsertAPI{}
-	return vminsertapi.NewVMInsertServer(addr, *vminsertConnsShutdownDuration, "clusternative", api, tc)
+	return vminsertapi.NewServer(addr, *vminsertConnsShutdownDuration, "clusternative", api, tc)
 }
 
 type vminsertAPI struct {

app/vminsert/main.go

@@ -125,6 +125,7 @@ func main() {
 
 	relabel.Init()
 	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
+	opentelemetry.Init()
 	protoparserutil.StartUnmarshalWorkers()
 	if len(*clusternativeListenAddr) > 0 {
 		s, err := clusternative.NewVMinsertServer(*clusternativeListenAddr, nil)

app/vminsert/opentelemetry/request_handler.go

@@ -24,6 +24,11 @@ var (
 	metadataInserted   = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="opentelemetry"}`)
 )
 
+// Init must be called after flag.Parse and before using the opentelemetry package.
+func Init() {
+	stream.InitDecodeOptions()
+}
+
 // InsertHandler processes opentelemetry metrics.
 func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := protoparserutil.GetExtraLabels(req)

app/vmselect/clusternative/vmselect.go

@@ -36,9 +36,6 @@ var (
 func NewVMSelectServer(addr string) (*vmselectapi.Server, error) {
 	api := &vmstorageAPI{}
 	limits := vmselectapi.Limits{
-		MaxLabelNames:                 *maxTagKeys,
-		MaxLabelValues:                *maxTagValues,
-		MaxTagValueSuffixes:           *maxTagValueSuffixesPerSearch,
 		MaxConcurrentRequests:         *maxConcurrentRequests,
 		MaxConcurrentRequestsFlagName: "clusternative.maxConcurrentRequests",
 		MaxQueueDuration:              *maxQueueDuration,
@@ -69,6 +66,9 @@ func (api *vmstorageAPI) SearchMetricNames(qt *querytracer.Tracer, sq *storage.S
 }
 
 func (api *vmstorageAPI) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	if maxLabelValues <= 0 || maxLabelValues > *maxTagValues {
+		maxLabelValues = *maxTagValues
+	}
 	dl := searchutil.DeadlineFromTimestamp(deadline)
 	labelValues, _, err := netstorage.LabelValues(qt, true, labelName, sq, maxLabelValues, dl)
 	return labelValues, wrapClusterNativeError(err)
@@ -76,12 +76,18 @@ func (api *vmstorageAPI) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQ
 
 func (api *vmstorageAPI) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte,
 	maxSuffixes int, deadline uint64) ([]string, error) {
+	if maxSuffixes <= 0 || maxSuffixes > *maxTagValueSuffixesPerSearch {
+		maxSuffixes = *maxTagValueSuffixesPerSearch
+	}
 	dl := searchutil.DeadlineFromTimestamp(deadline)
 	suffixes, _, err := netstorage.TagValueSuffixes(qt, accountID, projectID, true, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, dl)
 	return suffixes, wrapClusterNativeError(err)
 }
 
 func (api *vmstorageAPI) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	if maxLabelNames <= 0 || maxLabelNames > *maxTagKeys {
+		maxLabelNames = *maxTagKeys
+	}
 	dl := searchutil.DeadlineFromTimestamp(deadline)
 	labelNames, _, err := netstorage.LabelNames(qt, true, sq, maxLabelNames, dl)
 	return labelNames, wrapClusterNativeError(err)

app/vmselect/netstorage/netstorage.go

@@ -2171,23 +2171,16 @@ func (snr *storageNodesRequest) collectResults(partialResultsCounter *metrics.Co
 		return false, nil
 	}
 
-	// Verify whether at least a single node per each group successfully returned result in order to be able returning partial result.
-	missingGroups := 0
 	var firstErr error
 	for g, errsPartial := range errsPartialPerGroup {
-		if len(errsPartial) == g.nodesCount {
-			missingGroups++
-			if firstErr == nil {
-				// Return only the first error, since it has no sense in returning all errors.
-				firstErr = errsPartial[0]
-			}
-		}
-		if len(errsPartial) > 0 {
-			partialErrorsLogger.Warnf("%d out of %d vmstorage nodes at group %q were unavailable during the query; a sample error: %s", len(errsPartial), len(sns), g.name, errsPartial[0])
+		if firstErr == nil {
+			// Return only the first error, since it has no sense in returning all errors.
+			firstErr = errsPartial[0]
 		}
+		partialErrorsLogger.Warnf("%d out of %d vmstorage nodes at group %q were unavailable during the query; a sample error: %s", len(errsPartial), g.nodesCount, g.name, errsPartial[0])
 	}
-	if missingGroups >= *globalReplicationFactor {
-		// Too many groups contain all the non-working vmstorage nodes.
+	if snr.denyPartialResponse || len(resultsCollectedPerGroup) == 0 {
+		// The response cannot be returned as partial.
 		// Returns 503 status code, so the caller could retry it if needed.
 		err := &httpserver.ErrorWithStatusCode{
 			Err:        firstErr,

app/vmselect/prometheus/federate.qtpl

@@ -2,13 +2,16 @@
 	"math"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 ) %}
 
 {% stripspace %}
 
 // Federate writes rs in /federate format.
 // See https://prometheus.io/docs/prometheus/latest/federation/
-{% func Federate(rs *netstorage.Result) %}
+{% func Federate(rs *netstorage.Result, escapeScheme string) %}
 	{% code
 		values := rs.Values
 		timestamps := rs.Timestamps
@@ -24,10 +27,54 @@
 			See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185
 		{% endcomment %}
 		{% return %}
-	{% endif %}
-	{%= prometheusMetricName(&rs.MetricName) %}{% space %}
+    {% endif %}
+
+    {% switch escapeScheme %}
+       {% case federateEscapeSchemeUTF8 %}
+          {%= prometheusFederateMetricNameUTF8(&rs.MetricName) %}{% space %}
+
+       {% case federateEscapeSchemeUnderscore %}
+         {%= prometheusFederateMetricNameEscapeUnderscore(&rs.MetricName) %}{% space %}
+
+       {% case "" %}
+         {%= prometheusMetricName(&rs.MetricName) %}{% space %}
+    {% endswitch %}
+
 	{%f= lastValue %}{% space %}
 	{%dl= timestamps[len(timestamps)-1] %}{% newline %}
 {% endfunc %}
 
+{% func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) %}
+	{%s= promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)) %}
+	{% if len(mn.Tags) > 0 %}
+	{
+		{% code tags := mn.Tags %}
+		{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)) %}={%= escapePrometheusLabel(tags[0].Value) %}
+		{% code tags = tags[1:] %}
+		{% for i := range tags %}
+			{% code tag := &tags[i] %}
+			,{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)) %}={%= escapePrometheusLabel(tag.Value) %}
+		{% endfor %}
+	}
+	{% endif %}
+{% endfunc %}
+
+{% func prometheusFederateMetricNameUTF8(mn *storage.MetricName) %}
+	{
+        {%= escapePrometheusLabel(mn.MetricGroup) %}
+        {% if len(mn.Tags) > 0 %}
+        ,
+		{% code tags := mn.Tags %}
+		{%= escapePrometheusLabel(tags[0].Key) %}={%= escapePrometheusLabel(tags[0].Value) %}
+		{% code tags = tags[1:] %}
+		{% for i := range tags %}
+			{% code tag := &tags[i] %}
+			,{%= escapePrometheusLabel(tag.Key) %}={%= escapePrometheusLabel(tag.Value) %}
+		{% endfor %}
+        {% endif %}
+	}
+{% endfunc %}
+
+
+
 {% endstripspace %}

app/vmselect/prometheus/federate.qtpl.go

@@ -9,82 +9,241 @@ import (
 	"math"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 // Federate writes rs in /federate format.// See https://prometheus.io/docs/prometheus/latest/federation/
 
-//line app/vmselect/prometheus/federate.qtpl:11
+//line app/vmselect/prometheus/federate.qtpl:14
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/federate.qtpl:11
+//line app/vmselect/prometheus/federate.qtpl:14
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/federate.qtpl:11
-func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/federate.qtpl:13
+//line app/vmselect/prometheus/federate.qtpl:14
+func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result, escapeScheme string) {
+//line app/vmselect/prometheus/federate.qtpl:16
 	values := rs.Values
 	timestamps := rs.Timestamps
 
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 	if len(timestamps) == 0 || len(values) == 0 {
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 		return
-//line app/vmselect/prometheus/federate.qtpl:16
+//line app/vmselect/prometheus/federate.qtpl:19
 	}
-//line app/vmselect/prometheus/federate.qtpl:18
+//line app/vmselect/prometheus/federate.qtpl:21
 	lastValue := values[len(values)-1]
 
-//line app/vmselect/prometheus/federate.qtpl:20
+//line app/vmselect/prometheus/federate.qtpl:23
 	if math.IsNaN(lastValue) {
-//line app/vmselect/prometheus/federate.qtpl:26
+//line app/vmselect/prometheus/federate.qtpl:29
 		return
-//line app/vmselect/prometheus/federate.qtpl:27
+//line app/vmselect/prometheus/federate.qtpl:30
 	}
-//line app/vmselect/prometheus/federate.qtpl:28
-	streamprometheusMetricName(qw422016, &rs.MetricName)
-//line app/vmselect/prometheus/federate.qtpl:28
-	qw422016.N().S(` `)
-//line app/vmselect/prometheus/federate.qtpl:29
+//line app/vmselect/prometheus/federate.qtpl:32
+	switch escapeScheme {
+//line app/vmselect/prometheus/federate.qtpl:33
+	case federateEscapeSchemeUTF8:
+//line app/vmselect/prometheus/federate.qtpl:34
+		streamprometheusFederateMetricNameUTF8(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:34
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:36
+	case federateEscapeSchemeUnderscore:
+//line app/vmselect/prometheus/federate.qtpl:37
+		streamprometheusFederateMetricNameEscapeUnderscore(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:37
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:39
+	case "":
+//line app/vmselect/prometheus/federate.qtpl:40
+		streamprometheusMetricName(qw422016, &rs.MetricName)
+//line app/vmselect/prometheus/federate.qtpl:40
+		qw422016.N().S(` `)
+//line app/vmselect/prometheus/federate.qtpl:41
+	}
+//line app/vmselect/prometheus/federate.qtpl:43
 	qw422016.N().F(lastValue)
-//line app/vmselect/prometheus/federate.qtpl:29
+//line app/vmselect/prometheus/federate.qtpl:43
 	qw422016.N().S(` `)
-//line app/vmselect/prometheus/federate.qtpl:30
+//line app/vmselect/prometheus/federate.qtpl:44
 	qw422016.N().DL(timestamps[len(timestamps)-1])
-//line app/vmselect/prometheus/federate.qtpl:30
+//line app/vmselect/prometheus/federate.qtpl:44
 	qw422016.N().S(`
 `)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 }
 
-//line app/vmselect/prometheus/federate.qtpl:31
-func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result) {
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result, escapeScheme string) {
+//line app/vmselect/prometheus/federate.qtpl:45
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/federate.qtpl:31
-	StreamFederate(qw422016, rs)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+	StreamFederate(qw422016, rs, escapeScheme)
+//line app/vmselect/prometheus/federate.qtpl:45
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 }
 
-//line app/vmselect/prometheus/federate.qtpl:31
-func Federate(rs *netstorage.Result) string {
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+func Federate(rs *netstorage.Result, escapeScheme string) string {
+//line app/vmselect/prometheus/federate.qtpl:45
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/federate.qtpl:31
-	WriteFederate(qb422016, rs)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+	WriteFederate(qb422016, rs, escapeScheme)
+//line app/vmselect/prometheus/federate.qtpl:45
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
 	return qs422016
-//line app/vmselect/prometheus/federate.qtpl:31
+//line app/vmselect/prometheus/federate.qtpl:45
+}
+
+//line app/vmselect/prometheus/federate.qtpl:47
+func streamprometheusFederateMetricNameEscapeUnderscore(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:48
+	qw422016.N().S(promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)))
+//line app/vmselect/prometheus/federate.qtpl:49
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/federate.qtpl:49
+		qw422016.N().S(`{`)
+//line app/vmselect/prometheus/federate.qtpl:51
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/federate.qtpl:52
+		qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)))
+//line app/vmselect/prometheus/federate.qtpl:52
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:52
+		streamescapePrometheusLabel(qw422016, tags[0].Value)
+//line app/vmselect/prometheus/federate.qtpl:53
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/federate.qtpl:54
+		for i := range tags {
+//line app/vmselect/prometheus/federate.qtpl:55
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/federate.qtpl:55
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:56
+			qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)))
+//line app/vmselect/prometheus/federate.qtpl:56
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:56
+			streamescapePrometheusLabel(qw422016, tag.Value)
+//line app/vmselect/prometheus/federate.qtpl:57
+		}
+//line app/vmselect/prometheus/federate.qtpl:57
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/federate.qtpl:59
+	}
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:60
+func writeprometheusFederateMetricNameEscapeUnderscore(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:60
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+	streamprometheusFederateMetricNameEscapeUnderscore(qw422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:60
+func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/federate.qtpl:60
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/federate.qtpl:60
+	writeprometheusFederateMetricNameEscapeUnderscore(qb422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/federate.qtpl:60
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/federate.qtpl:60
+	return qs422016
+//line app/vmselect/prometheus/federate.qtpl:60
+}
+
+//line app/vmselect/prometheus/federate.qtpl:62
+func streamprometheusFederateMetricNameUTF8(qw422016 *qt422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:62
+	qw422016.N().S(`{`)
+//line app/vmselect/prometheus/federate.qtpl:64
+	streamescapePrometheusLabel(qw422016, mn.MetricGroup)
+//line app/vmselect/prometheus/federate.qtpl:65
+	if len(mn.Tags) > 0 {
+//line app/vmselect/prometheus/federate.qtpl:65
+		qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:67
+		tags := mn.Tags
+
+//line app/vmselect/prometheus/federate.qtpl:68
+		streamescapePrometheusLabel(qw422016, tags[0].Key)
+//line app/vmselect/prometheus/federate.qtpl:68
+		qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:68
+		streamescapePrometheusLabel(qw422016, tags[0].Value)
+//line app/vmselect/prometheus/federate.qtpl:69
+		tags = tags[1:]
+
+//line app/vmselect/prometheus/federate.qtpl:70
+		for i := range tags {
+//line app/vmselect/prometheus/federate.qtpl:71
+			tag := &tags[i]
+
+//line app/vmselect/prometheus/federate.qtpl:71
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/federate.qtpl:72
+			streamescapePrometheusLabel(qw422016, tag.Key)
+//line app/vmselect/prometheus/federate.qtpl:72
+			qw422016.N().S(`=`)
+//line app/vmselect/prometheus/federate.qtpl:72
+			streamescapePrometheusLabel(qw422016, tag.Value)
+//line app/vmselect/prometheus/federate.qtpl:73
+		}
+//line app/vmselect/prometheus/federate.qtpl:74
+	}
+//line app/vmselect/prometheus/federate.qtpl:74
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/federate.qtpl:76
+}
+
+//line app/vmselect/prometheus/federate.qtpl:76
+func writeprometheusFederateMetricNameUTF8(qq422016 qtio422016.Writer, mn *storage.MetricName) {
+//line app/vmselect/prometheus/federate.qtpl:76
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+	streamprometheusFederateMetricNameUTF8(qw422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+}
+
+//line app/vmselect/prometheus/federate.qtpl:76
+func prometheusFederateMetricNameUTF8(mn *storage.MetricName) string {
+//line app/vmselect/prometheus/federate.qtpl:76
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/federate.qtpl:76
+	writeprometheusFederateMetricNameUTF8(qb422016, mn)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/federate.qtpl:76
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/federate.qtpl:76
+	return qs422016
+//line app/vmselect/prometheus/federate.qtpl:76
 }

app/vmselect/prometheus/federate_test.go

@@ -8,15 +8,15 @@ import (
 )
 
 func TestFederate(t *testing.T) {
-	f := func(rs *netstorage.Result, expectedResult string) {
+	f := func(rs *netstorage.Result, escapeScheme string, expectedResult string) {
 		t.Helper()
-		result := Federate(rs)
+		result := Federate(rs, escapeScheme)
 		if result != expectedResult {
 			t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, expectedResult)
 		}
 	}
 
-	f(&netstorage.Result{}, ``)
+	f(&netstorage.Result{}, ``, ``)
 
 	f(&netstorage.Result{
 		MetricName: storage.MetricName{
@@ -39,5 +39,60 @@ func TestFederate(t *testing.T) {
 		},
 		Values:     []float64{1.23},
 		Timestamps: []int64{123},
-	}, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
+	}, ``, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
+
+	f(&netstorage.Result{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foo.bar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("some.!other"),
+					Value: []byte("value.unchanged!."),
+				},
+				{
+					Key:   []byte("qqq"),
+					Value: []byte("\\"),
+				},
+				{
+					Key:   []byte("!key"),
+					Value: []byte("value"),
+				},
+				{
+					Key: []byte("abc"),
+					// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
+					Value: []byte("a<b\"\\c"),
+				},
+			},
+		},
+		Values:     []float64{1.23},
+		Timestamps: []int64{123},
+	}, federateEscapeSchemeUnderscore, `foo_bar{some__other="value.unchanged!.",qqq="\\",_key="value",abc="a<b\"\\c"} 1.23 123`+"\n")
+
+	f(&netstorage.Result{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foo.bar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("some.!other"),
+					Value: []byte("value.unchanged!."),
+				},
+				{
+					Key:   []byte("qqq"),
+					Value: []byte("\\"),
+				},
+				{
+					Key:   []byte("!key"),
+					Value: []byte("value"),
+				},
+				{
+					Key: []byte(`ab"c`),
+					// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
+					Value: []byte("a<b\"\\c"),
+				},
+			},
+		},
+		Values:     []float64{1.23},
+		Timestamps: []int64{123},
+	}, federateEscapeSchemeUTF8, `{"foo.bar","some.!other"="value.unchanged!.","qqq"="\\","!key"="value","ab\"c"="a<b\"\\c"} 1.23 123`+"\n")
+
 }

app/vmselect/prometheus/federate_timing_test.go

@@ -9,16 +9,17 @@ import (
 )
 
 func BenchmarkFederate(b *testing.B) {
+
 	rs := &netstorage.Result{
 		MetricName: storage.MetricName{
-			MetricGroup: []byte("foo_bar_bazaaaa_total"),
+			MetricGroup: []byte("foo_bar_?_._bazaaaa_total"),
 			Tags: []storage.Tag{
 				{
-					Key:   []byte("instance"),
+					Key:   []byte("instance:job"),
 					Value: []byte("foobarbaz:2344"),
 				},
 				{
-					Key:   []byte("job"),
+					Key:   []byte("job.name"),
 					Value: []byte("aaabbbccc"),
 				},
 			},
@@ -27,12 +28,22 @@ func BenchmarkFederate(b *testing.B) {
 		Timestamps: []int64{1234567890},
 	}
 
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		var bb bytes.Buffer
-		for pb.Next() {
-			bb.Reset()
-			WriteFederate(&bb, rs)
-		}
-	})
+	f := func(name, escapeScheme string) {
+		b.Helper()
+
+		b.Run(name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.RunParallel(func(pb *testing.PB) {
+				var bb bytes.Buffer
+				for pb.Next() {
+					bb.Reset()
+					WriteFederate(&bb, rs, escapeScheme)
+				}
+			})
+		})
+	}
+
+	f("without escape", "")
+	f("allow-utf-8", federateEscapeSchemeUTF8)
+	f("legacy-underscore", federateEscapeSchemeUnderscore)
 }

app/vmselect/prometheus/prometheus.go

@@ -112,6 +112,11 @@ func PrettifyQuery(w http.ResponseWriter, r *http.Request) {
 	_ = bw.Flush()
 }
 
+const (
+	federateEscapeSchemeUnderscore = "underscore"
+	federateEscapeSchemeUTF8       = "utf-8"
+)
+
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
 func FederateHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
 	defer federateDuration.UpdateDuration(startTime)
@@ -144,6 +149,21 @@ func FederateHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter,
 		return fmt.Errorf("cannot export federated metrics, because some of vmstorage nodes are unavailable")
 	}
 
+	// add best-effort format negotiation
+	// modern version of Prometheus always set allow-utf-8 in order to properly parse utf-8 names and labels
+	// prometheus below v3 uses underscore escaping by default and it's the most common standard
+	var escapeScheme string
+	accept := r.Header.Get("Accept")
+	if len(accept) > 0 && strings.Contains(accept, "allow-utf-8") {
+		escapeScheme = federateEscapeSchemeUTF8
+	}
+	// try fallback to legacy underscore escaping if needed for Prometheus only,
+	// it's not widely used after Prometheus v3.0 release
+	// most of the Prometheus scrapers already use allow-utf-8 header
+	isPrometheus := strings.HasPrefix(r.UserAgent(), "Prometheus")
+	if len(escapeScheme) == 0 && isPrometheus {
+		escapeScheme = federateEscapeSchemeUnderscore
+	}
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
@@ -153,7 +173,7 @@ func FederateHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter,
 			return err
 		}
 		bb := sw.getBuffer(workerID)
-		WriteFederate(bb, rs)
+		WriteFederate(bb, rs, escapeScheme)
 		return sw.maybeFlushBuffer(bb)
 	})
 	if err == nil {

app/vmselect/promql/rollup.go

@@ -2444,8 +2444,15 @@ func rollupIntegrate(rfa *rollupFuncArg) float64 {
 		prevTimestamp = timestamp
 		prevValue = v
 	}
-	dt := float64(rfa.currTimestamp-prevTimestamp) / 1e3
-	sum += prevValue * dt
+	// Only extrapolate the last value through to currTimestamp when the time
+	// series has any sample after the lookbehind window. When realNextValue is
+	// NaN the series has effectively ended at prevTimestamp, so accruing area
+	// past it would overcount the integral.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474
+	if !math.IsNaN(rfa.realNextValue) {
+		dt := float64(rfa.currTimestamp-prevTimestamp) / 1e3
+		sum += prevValue * dt
+	}
 	return sum
 }
 

app/vmselect/promql/rollup_test.go

@@ -1385,10 +1385,65 @@ func TestRollupFuncsNoWindow(t *testing.T) {
 		if samplesScanned != 24 {
 			t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
-		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36}
+		// At tEnd=160 the series has no samples past the window (last sample is at
+		// ts=130), so integrate() must not extrapolate prevValue through tEnd.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474
+		valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 0.34}
 		timestampsExpected := []int64{0, 40, 80, 120, 160}
 		testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
+	t.Run("integrate_past_series_end", func(t *testing.T) {
+		// Constant series of value 1.0 from t=0..3600s (1h) at 60s step.
+		// Query integrate(metric[1h]) across t=0..10800s with 600s step.
+		// For t=0..3600s the window overlap with the data is [0,t], so the integral grows from 0 to 3600 (seconds).
+		// After the series ends, integrate must NOT keep accruing 3600 — it
+		// should taper to 0 once the lookbehind window is entirely past the
+		// last sample.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474
+		var testValues []int64
+		var testTimestamps []float64
+		for t := int64(0); t <= 3600_000; t += 60_000 {
+			testValues = append(testValues, t)
+			testTimestamps = append(testTimestamps, 1.0)
+		}
+		rc := rollupConfig{
+			Func:               rollupIntegrate,
+			Start:              0,
+			End:                10800_000,
+			Step:               600_000,
+			Window:             3600_000,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		values, _ := rc.Do(nil, testTimestamps, testValues)
+		for i, ti := range rc.Timestamps {
+			v := values[i]
+
+			// For t<=3600s: window overlap is [0,ti], integral equals ti in seconds.
+			if ti <= 3600_000 {
+				expV := float64(ti / 1e3)
+				if v != expV {
+					t.Fatalf("unexpected integrate result at t=%ds, want=%.3f got=%.3f", ti/1e3, expV, v)
+				}
+				continue
+			}
+			// For 3600s<t<7200s: data is partially outside the window, so the
+			// integral shrinks linearly from 3600 to 0 as t approaches 7200s.
+			if ti > 3600_000 && ti < 7200_000 {
+				expV := float64((7200_000 - ti) / 1e3)
+				if v != expV {
+					t.Fatalf("unexpected integrate result at t=%ds, want=%.3f got=%.3f", ti/1e3, expV, v)
+				}
+				continue
+			}
+			if ti >= 7200_000 {
+				// Window entirely past data end: must be NaN.
+				if !math.IsNaN(v) {
+					t.Fatalf("unexpected integrate result at t=%ds, want=NaN got=%.3f", ti/1e3, v)
+				}
+			}
+		}
+	})
 	t.Run("distinct_over_time_1", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:               rollupDistinct,

app/vmselect/vmui/index.html

@@ -37,7 +37,7 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-BjJ7fDL7.js"></script>
+  <script type="module" crossorigin src="./assets/index-U3iNn2Tx.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">

app/vmstorage/main.go

@@ -9,12 +9,10 @@ import (
 	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/servers"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -29,10 +27,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vminsertapi"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
 )
 
 var (
+	storageDataPath = flag.String("storageDataPath", "vmstorage-data", "Path to storage data")
 	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
@@ -41,13 +41,23 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	storageDataPath   = flag.String("storageDataPath", "vmstorage-data", "Path to storage data")
-	vminsertAddr      = flag.String("vminsertAddr", ":8400", "TCP address to accept connections from vminsert services")
-	vmselectAddr      = flag.String("vmselectAddr", ":8401", "TCP address to accept connections from vmselect services")
-	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages")
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages")
-	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages")
-	snapshotsMaxAge   = flagutil.NewRetentionDuration("snapshotsMaxAge", "3d", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
+	vminsertAddr                  = flag.String("vminsertAddr", ":8400", "TCP address to accept connections from vminsert services")
+	vminsertConnsShutdownDuration = flag.Duration("storage.vminsertConnsShutdownDuration", 10*time.Second, "The time needed for gradual closing of vminsert connections during "+
+		"graceful shutdown. Bigger duration reduces spikes in CPU, RAM and disk IO load on the remaining vmstorage nodes during rolling restart. "+
+		"Smaller duration reduces the time needed to close all the vminsert connections, thus reducing the time for graceful shutdown. "+
+		"Configured value must always be lower than the graceful shutdown period configured by the orchestration platform (terminationGracePeriodSeconds for Kubernetes). "+
+		"See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#improving-re-routing-performance-during-restart")
+	vmselectAddr                  = flag.String("vmselectAddr", ":8401", "TCP address to accept connections from vmselect services")
+	vmselectMaxConcurrentRequests = flag.Int("search.maxConcurrentRequests", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent vmselect requests "+
+		"the vmstorage can process at -vmselectAddr. It shouldn't be high, since a single request usually saturates a CPU core, and many concurrently executed requests "+
+		"may require high amounts of memory. See also -search.maxQueueDuration")
+	vmselectMaxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the incoming vmselect request waits for execution "+
+		"when -search.maxConcurrentRequests limit is reached")
+	vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
+		"This reduces CPU usage at the cost of higher network bandwidth usage")
+	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
+	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
+	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
 	_                 = flag.Duration("snapshotCreateTimeout", 0, "Deprecated: this flag does nothing")
 
 	_ = flag.Duration("finalMergeDelay", 0, "Deprecated: this flag does nothing")
@@ -58,7 +68,7 @@ var (
 		"If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+
 		"If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication for details")
+		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication")
 	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
 		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
 		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
@@ -119,7 +129,7 @@ var (
 )
 
 func main() {
-	// vmstoage is optimized for reduced memory allocations,
+	// vmstorage is optimized for reduced memory allocations,
 	// so it can run with the reduced GOGC in order to reduce the used memory,
 	// while keeping CPU usage spent in GC at low levels.
 	//
@@ -178,7 +188,7 @@ func main() {
 		LogNewSeries:                *logNewSeries,
 	}
 	strg := storage.MustOpenStorage(*storageDataPath, opts)
-	initStaleSnapshotsRemover(strg)
+	vmStorage := newVMStorage(strg, *vmselectMaxConcurrentRequests)
 
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
@@ -192,19 +202,23 @@ func main() {
 
 	// register storage metrics
 	storageMetrics := metrics.NewSet()
-	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, strg)
-	})
+	storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
 	metrics.RegisterSet(storageMetrics)
 
 	protoparserutil.StartUnmarshalWorkers()
 
-	servers.GetMaxUniqueTimeSeries() // for init and logging only.
-	vminsertSrv, err := servers.NewVMInsertServer(*vminsertAddr, strg)
+	vminsertSrv, err := vminsertapi.NewServer(*vminsertAddr, *vminsertConnsShutdownDuration, "vminsert", vmStorage, nil)
 	if err != nil {
 		logger.Fatalf("cannot create a server with -vminsertAddr=%s: %s", *vminsertAddr, err)
+
 	}
-	vmselectSrv, err := servers.NewVMSelectServer(*vmselectAddr, strg)
+	limits := vmselectapi.Limits{
+		MaxConcurrentRequests:         *vmselectMaxConcurrentRequests,
+		MaxConcurrentRequestsFlagName: "search.maxConcurrentRequests",
+		MaxQueueDuration:              *vmselectMaxQueueDuration,
+		MaxQueueDurationFlagName:      "search.maxQueueDuration",
+	}
+	vmselectSrv, err := vmselectapi.NewServer(*vmselectAddr, vmStorage, limits, *vmselectDisableRPCCompression)
 	if err != nil {
 		logger.Fatalf("cannot create a server with -vmselectAddr=%s: %s", *vmselectAddr, err)
 	}
@@ -213,8 +227,7 @@ func main() {
 	if len(listenAddrs) == 0 {
 		listenAddrs = []string{":8482"}
 	}
-	requestHandler := newRequestHandler(strg)
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{UseProxyProtocol: useProxyProtocol})
+	go httpserver.Serve(listenAddrs, vmStorage.requestHandler, httpserver.ServeOptions{UseProxyProtocol: useProxyProtocol})
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
@@ -234,7 +247,6 @@ func main() {
 	metrics.UnregisterSet(storageMetrics, true)
 	storageMetrics = nil
 
-	stopStaleSnapshotsRemover()
 	vmselectSrv.MustStop()
 	vminsertSrv.MustStop()
 	protoparserutil.StopUnmarshalWorkers()
@@ -242,31 +254,29 @@ func main() {
 
 	logger.Infof("gracefully closing the storage at %s", *storageDataPath)
 	startTime = time.Now()
-	strg.MustClose()
+	vmStorage.Stop()
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 
 	fs.MustStopDirRemover()
 	logger.Infof("the vmstorage has been stopped")
 }
 
-func newRequestHandler(strg *storage.Storage) httpserver.RequestHandler {
-	return func(w http.ResponseWriter, r *http.Request) bool {
-		if r.URL.Path == "/" {
-			if r.Method != http.MethodGet {
-				return false
-			}
-			w.Header().Add("Content-Type", "text/html; charset=utf-8")
-			fmt.Fprintf(w, `vmstorage - a component of VictoriaMetrics cluster<br/>
+// requestHandler is a storage request handler.
+// TODO(@rtm0): Move to a separate file, request_handler.go
+func (vms *VMStorage) requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := r.URL.Path
+
+	if path == "/" {
+		if r.Method != http.MethodGet {
+			return false
+		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprintf(w, `vmstorage - a component of VictoriaMetrics cluster<br/>
 			<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/">docs</a><br>
 `)
-			return true
-		}
-		return requestHandler(w, r, strg)
+		return true
 	}
-}
 
-func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storage) bool {
-	path := r.URL.Path
 	if path == "/internal/force_merge" {
 		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
 			return true
@@ -278,7 +288,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 			defer activeForceMerges.Dec()
 			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
 			startTime := time.Now()
-			if err := strg.ForceMergePartitions(partitionNamePrefix); err != nil {
+			if err := vms.s.ForceMergePartitions(partitionNamePrefix); err != nil {
 				logger.Errorf("error in forced merge for partition_prefix=%q: %s", partitionNamePrefix, err)
 				return
 			}
@@ -291,7 +301,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 			return true
 		}
 		logger.Infof("flushing storage to make pending data available for reading")
-		strg.DebugFlush()
+		vms.s.DebugFlush()
 		return true
 	}
 
@@ -311,7 +321,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 		}
 		logger.Infof("enabling logging of new series for the next %s. This may increase resource usage during this period.", time.Duration(dealine)*time.Second)
 		endTime := fasttime.UnixTimestamp() + uint64(dealine)
-		strg.SetLogNewSeriesUntil(endTime)
+		vms.s.SetLogNewSeriesUntil(endTime)
 		fmt.Fprintf(w, `{"status":"success","data":{"logEndTime":%q}}`, time.Unix(int64(endTime), 0))
 		return true
 	}
@@ -327,13 +337,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotName := strg.MustCreateSnapshot()
+		snapshotName := vms.s.MustCreateSnapshot()
 
 		// Verify whether the client already closed the connection.
 		// In this case it is better to drop the created snapshot, since the client isn't interested in it.
 		if err := r.Context().Err(); err != nil {
 			logger.Infof("deleting already created snapshot at %s because the client canceled the request", snapshotName)
-			if err := deleteSnapshot(strg, snapshotName); err != nil {
+			if err := vms.deleteSnapshot(snapshotName); err != nil {
 				logger.Infof("cannot delete just created snapshot: %s", err)
 				return true
 			}
@@ -345,7 +355,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 	case "/list":
 		snapshotsListTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshots := strg.MustListSnapshots()
+		snapshots := vms.s.MustListSnapshots()
 		fmt.Fprintf(w, `{"status":"ok","snapshots":[`)
 		if len(snapshots) > 0 {
 			for _, snapshot := range snapshots[:len(snapshots)-1] {
@@ -359,7 +369,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 		snapshotsDeleteTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
-		if err := deleteSnapshot(strg, snapshotName); err != nil {
+		if err := vms.deleteSnapshot(snapshotName); err != nil {
 			jsonResponseError(w, err)
 			snapshotsDeleteErrorsTotal.Inc()
 			return true
@@ -369,9 +379,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshots := strg.MustListSnapshots()
+		snapshots := vms.s.MustListSnapshots()
 		for _, snapshotName := range snapshots {
-			if err := strg.DeleteSnapshot(snapshotName); err != nil {
+			if err := vms.s.DeleteSnapshot(snapshotName); err != nil {
 				err = fmt.Errorf("cannot delete snapshot %q: %w", snapshotName, err)
 				jsonResponseError(w, err)
 				snapshotsDeleteAllErrorsTotal.Inc()
@@ -385,50 +395,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 	}
 }
 
-func deleteSnapshot(strg *storage.Storage, snapshotName string) error {
-	snapshots := strg.MustListSnapshots()
-	for _, snName := range snapshots {
-		if snName == snapshotName {
-			if err := strg.DeleteSnapshot(snName); err != nil {
-				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
-			}
-			return nil
-		}
-	}
-	return fmt.Errorf("cannot find snapshot %q", snapshotName)
-}
-
-func initStaleSnapshotsRemover(strg *storage.Storage) {
-	staleSnapshotsRemoverCh = make(chan struct{})
-	if snapshotsMaxAge.Duration() <= 0 {
-		return
-	}
-	snapshotsMaxAgeDur := snapshotsMaxAge.Duration()
-	staleSnapshotsRemoverWG.Go(func() {
-		d := timeutil.AddJitterToDuration(time.Second * 11)
-		t := time.NewTicker(d)
-		defer t.Stop()
-		for {
-			select {
-			case <-staleSnapshotsRemoverCh:
-				return
-			case <-t.C:
-			}
-			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
-		}
-	})
-}
-
-func stopStaleSnapshotsRemover() {
-	close(staleSnapshotsRemoverCh)
-	staleSnapshotsRemoverWG.Wait()
-}
-
-var (
-	staleSnapshotsRemoverCh chan struct{}
-	staleSnapshotsRemoverWG sync.WaitGroup
-)
-
 var (
 	activeForceMerges = metrics.NewCounter("vm_active_force_merges")
 
@@ -443,7 +409,9 @@ var (
 	snapshotsDeleteAllErrorsTotal = metrics.NewCounter(`vm_http_request_errors_total{path="/snapshot/delete_all"}`)
 )
 
-func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
+// TODO(@rtm0): Move to metrics.go.
+func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
+	strg := vms.s
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
 	tm := &m.TableMetrics
@@ -667,7 +635,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled`, tm.ScheduledDownsamplingPartitions)
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled_size_bytes`, tm.ScheduledDownsamplingPartitionsSize)
 
-	metrics.WriteGaugeUint64(w, `vm_search_max_unique_timeseries`, uint64(servers.GetMaxUniqueTimeSeries()))
+	metrics.WriteGaugeUint64(w, `vm_search_max_unique_timeseries`, uint64(vms.maxUniqueTimeSeriesCalculated))
 
 	metrics.WriteGaugeUint64(w, `vm_metrics_metadata_storage_items`, m.MetadataStorageItemsCurrent)
 	metrics.WriteCounterUint64(w, `vm_metrics_metadata_storage_size_bytes`, m.MetadataStorageCurrentSizeBytes)

app/vmstorage/servers/vminsert.go

@@ -1,55 +0,0 @@
-package servers
-
-import (
-	"flag"
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vminsertapi"
-)
-
-var (
-	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression "+
-		"at the cost of precision loss")
-	vminsertConnsShutdownDuration = flag.Duration("storage.vminsertConnsShutdownDuration", 10*time.Second, "The time needed for gradual closing of vminsert connections during "+
-		"graceful shutdown. Bigger duration reduces spikes in CPU, RAM and disk IO load on the remaining vmstorage nodes during rolling restart. "+
-		"Smaller duration reduces the time needed to close all the vminsert connections, thus reducing the time for graceful shutdown. "+
-		"Configured value must always be lower than the graceful shutdown period configured by the orchestration platform (terminationGracePeriodSeconds for Kubernetes). "+
-		"See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#improving-re-routing-performance-during-restart")
-)
-
-// NewVMInsertServer starts vminsertapi.VMInsertServer at the given addr serving the given storage.
-func NewVMInsertServer(addr string, storage *storage.Storage) (*vminsertapi.VMInsertServer, error) {
-	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
-		return nil, fmt.Errorf("invalid -precisionBits: %w", err)
-	}
-	api := &vminsertAPI{
-		storage: storage,
-	}
-
-	return vminsertapi.NewVMInsertServer(addr, *vminsertConnsShutdownDuration, "vminsert", api, nil)
-}
-
-type vminsertAPI struct {
-	storage *storage.Storage
-}
-
-// WriteRows implements lib/vminsertapi.API interface
-func (v *vminsertAPI) WriteRows(rows []storage.MetricRow) error {
-	v.storage.AddRows(rows, uint8(*precisionBits))
-	return nil
-}
-
-// WriteMetadata implements lib/vminsertapi.API interface
-func (v *vminsertAPI) WriteMetadata(rows []metricsmetadata.Row) error {
-	v.storage.AddMetadataRows(rows)
-	return nil
-}
-
-// IsReadOnly implements lib/vminsertapi.API interface
-func (v *vminsertAPI) IsReadOnly() bool {
-	return v.storage.IsReadOnly()
-}

app/vmstorage/servers/vmselect.go

@@ -1,312 +0,0 @@
-package servers
-
-import (
-	"flag"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
-)
-
-var (
-	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be scanned during every query. "+
-		"This allows protecting against heavy queries, which select unexpectedly high number of series. When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional). See also -search.max* command-line flags at vmselect")
-	maxTagKeys = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned per search. "+
-		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
-	maxTagValues = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned per search. "+
-		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
-	maxTagValueSuffixesPerSearch = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
-	maxConcurrentRequests        = flag.Int("search.maxConcurrentRequests", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent vmselect requests "+
-		"the vmstorage can process at -vmselectAddr. It shouldn't be high, since a single request usually saturates a CPU core, and many concurrently executed requests "+
-		"may require high amounts of memory. See also -search.maxQueueDuration")
-	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the incoming vmselect request waits for execution "+
-		"when -search.maxConcurrentRequests limit is reached")
-
-	disableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
-		"This reduces CPU usage at the cost of higher network bandwidth usage")
-)
-
-var (
-	maxUniqueTimeseriesValue     int
-	maxUniqueTimeseriesValueOnce sync.Once
-)
-
-// NewVMSelectServer starts new server at the given addr, which serves vmselect requests from the given s.
-func NewVMSelectServer(addr string, s *storage.Storage) (*vmselectapi.Server, error) {
-	api := &vmstorageAPI{
-		s: s,
-	}
-	limits := vmselectapi.Limits{
-		MaxLabelNames:                 *maxTagKeys,
-		MaxLabelValues:                *maxTagValues,
-		MaxTagValueSuffixes:           *maxTagValueSuffixesPerSearch,
-		MaxConcurrentRequests:         *maxConcurrentRequests,
-		MaxConcurrentRequestsFlagName: "search.maxConcurrentRequests",
-		MaxQueueDuration:              *maxQueueDuration,
-		MaxQueueDurationFlagName:      "search.maxQueueDuration",
-	}
-	return vmselectapi.NewServer(addr, api, limits, *disableRPCCompression)
-}
-
-// vmstorageAPI impelements vmselectapi.API
-type vmstorageAPI struct {
-	s *storage.Storage
-}
-
-func (api *vmstorageAPI) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
-	tr := sq.GetTimeRange()
-	maxMetrics := getMaxMetrics(sq.MaxMetrics)
-	tfss, err := api.setupTfss(qt, sq, tr, maxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	if len(tfss) == 0 {
-		return nil, fmt.Errorf("missing tag filters")
-	}
-	bi := getBlockIterator()
-	bi.sr.Init(qt, api.s, tfss, tr, maxMetrics, deadline)
-	if err := bi.sr.Error(); err != nil {
-		bi.MustClose()
-		return nil, err
-	}
-	return bi, nil
-}
-
-func (api *vmstorageAPI) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
-	tr := sq.GetTimeRange()
-	maxMetrics := sq.MaxMetrics
-	if maxMetrics <= 0 {
-		// fallback to maxUniqueTimeSeries if no limit is provided,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
-		maxMetrics = GetMaxUniqueTimeSeries()
-	}
-	tfss, err := api.setupTfss(qt, sq, tr, maxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	if len(tfss) == 0 {
-		return nil, fmt.Errorf("missing tag filters")
-	}
-	return api.s.SearchMetricNames(qt, tfss, tr, maxMetrics, deadline)
-}
-
-func (api *vmstorageAPI) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
-	tr := sq.GetTimeRange()
-	maxMetrics := sq.MaxMetrics
-	if maxMetrics <= 0 {
-		// fallback to maxUniqueTimeSeries if no limit is provided,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
-		maxMetrics = GetMaxUniqueTimeSeries()
-	}
-	tfss, err := api.setupTfss(qt, sq, tr, maxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	return api.s.SearchLabelValues(qt, sq.AccountID, sq.ProjectID, labelName, tfss, tr, maxLabelValues, maxMetrics, deadline)
-}
-
-func (api *vmstorageAPI) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte,
-	maxSuffixes int, deadline uint64) ([]string, error) {
-	suffixes, err := api.s.SearchTagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
-	if err != nil {
-		return nil, err
-	}
-	if len(suffixes) >= maxSuffixes {
-		return nil, fmt.Errorf("more than -search.maxTagValueSuffixesPerSearch=%d suffixes returned; "+
-			"either narrow down the search or increase -search.maxTagValueSuffixesPerSearch command-line flag value", maxSuffixes)
-	}
-	return suffixes, nil
-}
-
-func (api *vmstorageAPI) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
-	tr := sq.GetTimeRange()
-	maxMetrics := sq.MaxMetrics
-	if maxMetrics <= 0 {
-		// fallback to maxUniqueTimeSeries if no limit is provided,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
-		maxMetrics = GetMaxUniqueTimeSeries()
-	}
-	tfss, err := api.setupTfss(qt, sq, tr, maxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	return api.s.SearchLabelNames(qt, sq.AccountID, sq.ProjectID, tfss, tr, maxLabelNames, maxMetrics, deadline)
-}
-
-func (api *vmstorageAPI) SeriesCount(_ *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
-	return api.s.GetSeriesCount(accountID, projectID, deadline)
-}
-
-func (api *vmstorageAPI) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
-	return api.s.SearchTenants(qt, tr, deadline)
-}
-
-func (api *vmstorageAPI) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
-	tr := sq.GetTimeRange()
-	maxMetrics := sq.MaxMetrics
-	if maxMetrics <= 0 {
-		// fallback to maxUniqueTimeSeries if no limit is provided,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
-		maxMetrics = GetMaxUniqueTimeSeries()
-	}
-	tfss, err := api.setupTfss(qt, sq, tr, maxMetrics, deadline)
-	if err != nil {
-		return nil, err
-	}
-	date := uint64(sq.MinTimestamp) / (24 * 3600 * 1000)
-	return api.s.GetTSDBStatus(qt, sq.AccountID, sq.ProjectID, tfss, date, focusLabel, topN, maxMetrics, deadline)
-}
-
-func (api *vmstorageAPI) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
-	tr := sq.GetTimeRange()
-	maxMetrics := sq.MaxMetrics
-	if maxMetrics <= 0 {
-		// fallback to maxUniqueTimeSeries if no limit is provided,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
-		maxMetrics = GetMaxUniqueTimeSeries()
-	}
-	tfss, err := api.setupTfss(qt, sq, tr, maxMetrics, deadline)
-	if err != nil {
-		return 0, err
-	}
-	if len(tfss) == 0 {
-		return 0, fmt.Errorf("missing tag filters")
-	}
-	return api.s.DeleteSeries(qt, tfss, maxMetrics)
-}
-
-func (api *vmstorageAPI) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, _ uint64) error {
-	api.s.RegisterMetricNames(qt, mrs)
-	return nil
-}
-
-func (api *vmstorageAPI) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, _ uint64) (metricnamestats.StatsResult, error) {
-	return api.s.GetMetricNamesStats(qt, tt, limit, le, matchPattern), nil
-}
-
-func (api *vmstorageAPI) ResetMetricNamesUsageStats(qt *querytracer.Tracer, _ uint64) error {
-	api.s.ResetMetricNamesStats(qt)
-	return nil
-}
-
-func (api *vmstorageAPI) setupTfss(qt *querytracer.Tracer, sq *storage.SearchQuery, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]*storage.TagFilters, error) {
-	tfss := make([]*storage.TagFilters, 0, len(sq.TagFilterss))
-	accountID := sq.AccountID
-	projectID := sq.ProjectID
-	for _, tagFilters := range sq.TagFilterss {
-		tfs := storage.NewTagFilters(accountID, projectID)
-		for i := range tagFilters {
-			tf := &tagFilters[i]
-			if string(tf.Key) == "__graphite__" {
-				query := tf.Value
-				qtChild := qt.NewChild("searching for series matching __graphite__=%q", query)
-				paths, err := api.s.SearchGraphitePaths(qtChild, accountID, projectID, tr, query, maxMetrics, deadline)
-				qtChild.Donef("found %d series", len(paths))
-				if err != nil {
-					return nil, fmt.Errorf("error when searching for Graphite paths for query %q: %w", query, err)
-				}
-				if len(paths) >= maxMetrics {
-					return nil, fmt.Errorf("more than %d time series match Graphite query %q; "+
-						"either narrow down the query or increase the corresponding -search.max* command-line flag value at vmselect nodes; "+
-						"see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#resource-usage-limits", maxMetrics, query)
-				}
-				tfs.AddGraphiteQuery(query, paths, tf.IsNegative)
-				continue
-			}
-			if err := tfs.Add(tf.Key, tf.Value, tf.IsNegative, tf.IsRegexp); err != nil {
-				return nil, fmt.Errorf("cannot parse tag filter %s: %w", tf, err)
-			}
-		}
-		tfss = append(tfss, tfs)
-	}
-	return tfss, nil
-}
-
-func (api *vmstorageAPI) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
-	return api.s.GetMetadataRows(qt, tt, limit, metricName, deadline)
-}
-
-// blockIterator implements vmselectapi.BlockIterator
-type blockIterator struct {
-	sr storage.Search
-	mb storage.MetricBlock
-}
-
-var blockIteratorsPool sync.Pool
-
-func (bi *blockIterator) MustClose() {
-	bi.sr.MustClose()
-	bi.mb.MetricName = nil
-	bi.mb.Block.Reset()
-	blockIteratorsPool.Put(bi)
-}
-
-func getBlockIterator() *blockIterator {
-	v := blockIteratorsPool.Get()
-	if v == nil {
-		v = &blockIterator{}
-	}
-	return v.(*blockIterator)
-}
-
-func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
-	if !bi.sr.NextMetricBlock() {
-		return dst, false
-	}
-	mb := bi.mb
-	mb.MetricName = bi.sr.MetricBlockRef.MetricName
-	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
-	dst = mb.Marshal(dst[:0])
-	return dst, true
-}
-
-func (bi *blockIterator) Error() error {
-	return bi.sr.Error()
-}
-
-func getMaxMetrics(searchQueryLimit int) int {
-	if searchQueryLimit <= 0 {
-		return GetMaxUniqueTimeSeries()
-	}
-	// searchQueryLimit cannot exceed `-search.maxUniqueTimeseries`
-	if *maxUniqueTimeseries != 0 && searchQueryLimit > *maxUniqueTimeseries {
-		searchQueryLimit = *maxUniqueTimeseries
-	}
-	return searchQueryLimit
-}
-
-// GetMaxUniqueTimeSeries returns `-search.maxUniqueTimeseries` or the auto-calculated value based on available resources.
-// The calculation is split into calculateMaxUniqueTimeSeriesForResource for unit testing.
-func GetMaxUniqueTimeSeries() int {
-	maxUniqueTimeseriesValueOnce.Do(func() {
-		maxUniqueTimeseriesValue = *maxUniqueTimeseries
-		if maxUniqueTimeseriesValue <= 0 {
-			maxUniqueTimeseriesValue = calculateMaxUniqueTimeSeriesForResource(*maxConcurrentRequests, memory.Remaining())
-		}
-	})
-	return maxUniqueTimeseriesValue
-}
-
-// calculateMaxUniqueTimeSeriesForResource calculate the max metrics limit calculated by available resources.
-func calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, remainingMemory int) int {
-	if maxConcurrentRequests <= 0 {
-		// This line should NOT be reached unless the user has set an incorrect `search.maxConcurrentRequests`.
-		// In such cases, fallback to unlimited.
-		logger.Warnf("limiting -search.maxUniqueTimeseries to %v because -search.maxConcurrentRequests=%d.", 2e9, maxConcurrentRequests)
-		return 2e9
-	}
-
-	// Calculate the max metrics limit for a single request in the worst-case concurrent scenario.
-	// The approximate size of 1 unique series that could occupy in the vmstorage is 200 bytes.
-	mts := remainingMemory / 200 / maxConcurrentRequests
-	logger.Infof("limiting -search.maxUniqueTimeseries to %d according to -search.maxConcurrentRequests=%d and remaining memory=%d bytes. To increase the limit, reduce -search.maxConcurrentRequests or increase memory available to the process.", mts, maxConcurrentRequests, remainingMemory)
-	return mts
-}

app/vmstorage/vmstorage.go

@@ -0,0 +1,391 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+)
+
+var (
+	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression "+
+		"at the cost of precision loss")
+	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be scanned during every query. "+
+		"This allows protecting against heavy queries, which select unexpectedly high number of series. When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional). See also -search.max* command-line flags at vmselect")
+	maxTagKeys = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned per search. "+
+		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
+	maxTagValues = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned per search. "+
+		"See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration")
+	maxTagValueSuffixesPerSearch = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
+	snapshotsMaxAge              = flagutil.NewRetentionDuration("snapshotsMaxAge", "3d", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
+)
+
+// newVMStorage creates a new instance of of VMStorage.
+//
+// The created VMStorage instance takes ownership of s.
+func newVMStorage(s *storage.Storage, vmselectMaxConcurrentRequests int) *VMStorage {
+	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
+		logger.Fatalf("invalid -precisionBits=%d: %s", *precisionBits, err)
+	}
+
+	maxUniqueTimeseriesCalculated := *maxUniqueTimeseries
+	if maxUniqueTimeseriesCalculated <= 0 {
+		maxUniqueTimeseriesCalculated = calculateMaxUniqueTimeseries(vmselectMaxConcurrentRequests, memory.Remaining())
+	}
+
+	vms := &VMStorage{
+		s:                             s,
+		maxUniqueTimeseries:           *maxUniqueTimeseries,
+		maxUniqueTimeSeriesCalculated: maxUniqueTimeseriesCalculated,
+		staleSnapshotsRemoverCh:       make(chan struct{}),
+	}
+	vms.initStaleSnapshotsRemover()
+	return vms
+}
+
+// calculateMaxUniqueTimeseries calculates the maxUniqueTimeseries based on the
+// available system resources.
+func calculateMaxUniqueTimeseries(maxConcurrentRequests, remainingMemory int) int {
+	if maxConcurrentRequests <= 0 {
+		// This line should NOT be reached unless the user has set an incorrect `search.maxConcurrentRequests`.
+		// In such cases, fallback to unlimited.
+		logger.Warnf("limiting -search.maxUniqueTimeseries to %v because -search.maxConcurrentRequests=%d.", 2e9, maxConcurrentRequests)
+		return 2e9
+	}
+
+	// Calculate the max metrics limit for a single request in the worst-case concurrent scenario.
+	// The approximate size of 1 unique series that could occupy in the vmstorage is 200 bytes.
+	mts := remainingMemory / 200 / maxConcurrentRequests
+	logger.Infof("limiting -search.maxUniqueTimeseries to %d according to -search.maxConcurrentRequests=%d and remaining memory=%d bytes. To increase the limit, reduce -search.maxConcurrentRequests or increase memory available to the process.", mts, maxConcurrentRequests, remainingMemory)
+	return mts
+}
+
+// VMStorage impelements vmselectapi.API and vminsertapi.API.
+type VMStorage struct {
+	s                             *storage.Storage
+	maxUniqueTimeseries           int
+	maxUniqueTimeSeriesCalculated int
+	staleSnapshotsRemoverCh       chan struct{}
+	staleSnapshotsRemoverWG       sync.WaitGroup
+}
+
+func (vms *VMStorage) initStaleSnapshotsRemover() {
+	if snapshotsMaxAge.Duration() <= 0 {
+		return
+	}
+	snapshotsMaxAgeDuration := snapshotsMaxAge.Duration()
+	vms.staleSnapshotsRemoverWG.Go(func() {
+		d := timeutil.AddJitterToDuration(time.Second * 11)
+		t := time.NewTicker(d)
+		defer t.Stop()
+		for {
+			select {
+			case <-vms.staleSnapshotsRemoverCh:
+				return
+			case <-t.C:
+			}
+			vms.s.MustDeleteStaleSnapshots(snapshotsMaxAgeDuration)
+		}
+	})
+}
+
+func (vms *VMStorage) Stop() {
+	close(vms.staleSnapshotsRemoverCh)
+	vms.staleSnapshotsRemoverWG.Wait()
+	vms.s.MustClose()
+}
+
+// WriteRows writes metric rows to the storage.
+//
+// The caller should limit the number of concurrent calls to WriteRows() in
+// order to limit memory usage.
+func (vms *VMStorage) WriteRows(rows []storage.MetricRow) error {
+	vms.s.AddRows(rows, uint8(*precisionBits))
+	return nil
+}
+
+// WriteMetadata writes metrics metadata to storage.
+//
+// The caller should limit the number of concurrent calls to WriteMetadata() in
+// order to limit memory usage.
+func (vms *VMStorage) WriteMetadata(rows []metricsmetadata.Row) error {
+	vms.s.AddMetadataRows(rows)
+	return nil
+}
+
+// IsReadOnly returns true is the storage is in read-only mode.
+func (vms *VMStorage) IsReadOnly() bool {
+	return vms.s.IsReadOnly()
+}
+
+func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := vms.getMaxMetrics(sq.MaxMetrics)
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	if len(tfss) == 0 {
+		return nil, fmt.Errorf("missing tag filters")
+	}
+	bi := getBlockIterator()
+	bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
+	if err := bi.sr.Error(); err != nil {
+		bi.MustClose()
+		return nil, err
+	}
+	return bi, nil
+}
+
+func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
+	if searchQueryLimit <= 0 {
+		return vms.maxUniqueTimeSeriesCalculated
+	}
+	// searchQueryLimit cannot exceed `-search.maxUniqueTimeseries`
+	if vms.maxUniqueTimeseries != 0 && searchQueryLimit > vms.maxUniqueTimeseries {
+		searchQueryLimit = vms.maxUniqueTimeseries
+	}
+	return searchQueryLimit
+}
+
+// blockIterator implements vmselectapi.BlockIterator
+type blockIterator struct {
+	sr storage.Search
+	mb storage.MetricBlock
+}
+
+var blockIteratorsPool sync.Pool
+
+func (bi *blockIterator) MustClose() {
+	bi.sr.MustClose()
+	bi.mb.MetricName = nil
+	bi.mb.Block.Reset()
+	blockIteratorsPool.Put(bi)
+}
+
+func getBlockIterator() *blockIterator {
+	v := blockIteratorsPool.Get()
+	if v == nil {
+		v = &blockIterator{}
+	}
+	return v.(*blockIterator)
+}
+
+func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
+	if !bi.sr.NextMetricBlock() {
+		return dst, false
+	}
+	mb := bi.mb
+	mb.MetricName = bi.sr.MetricBlockRef.MetricName
+	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
+	dst = mb.Marshal(dst[:0])
+	return dst, true
+}
+
+func (bi *blockIterator) Error() error {
+	return bi.sr.Error()
+}
+
+// SearchMetricNames returns metric names for the given tfss on the given tr.
+func (vms *VMStorage) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	if len(tfss) == 0 {
+		return nil, fmt.Errorf("missing tag filters")
+	}
+	return vms.s.SearchMetricNames(qt, tfss, tr, maxMetrics, deadline)
+}
+
+// SearchLabelValues searches for label values for the given labelName, tfss and
+// tr.
+func (vms *VMStorage) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	tr := sq.GetTimeRange()
+	if maxLabelValues <= 0 || maxLabelValues > *maxTagValues {
+		maxLabelValues = *maxTagValues
+	}
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	return vms.s.SearchLabelValues(qt, sq.AccountID, sq.ProjectID, labelName, tfss, tr, maxLabelValues, maxMetrics, deadline)
+}
+
+// TagValueSuffixes returns all the tag value suffixes for the given tagKey and
+// tagValuePrefix on the given tr.
+//
+// This allows implementing
+// https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find or
+// similar APIs.
+func (vms *VMStorage) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte,
+	maxSuffixes int, deadline uint64) ([]string, error) {
+	if maxSuffixes <= 0 || maxSuffixes > *maxTagValueSuffixesPerSearch {
+		maxSuffixes = *maxTagValueSuffixesPerSearch
+	}
+	suffixes, err := vms.s.SearchTagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
+	if err != nil {
+		return nil, err
+	}
+	if len(suffixes) >= maxSuffixes {
+		return nil, fmt.Errorf("more than -search.maxTagValueSuffixesPerSearch=%d suffixes returned; "+
+			"either narrow down the search or increase -search.maxTagValueSuffixesPerSearch command-line flag value", maxSuffixes)
+	}
+	return suffixes, nil
+}
+
+// SearchLabelNames searches for tag keys matching the given tfss on tr.
+func (vms *VMStorage) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	tr := sq.GetTimeRange()
+	if maxLabelNames <= 0 || maxLabelNames > *maxTagKeys {
+		maxLabelNames = *maxTagKeys
+	}
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	return vms.s.SearchLabelNames(qt, sq.AccountID, sq.ProjectID, tfss, tr, maxLabelNames, maxMetrics, deadline)
+}
+
+func (vms *VMStorage) SeriesCount(_ *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
+	return vms.s.GetSeriesCount(accountID, projectID, deadline)
+}
+
+func (vms *VMStorage) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
+	return vms.s.SearchTenants(qt, tr, deadline)
+}
+
+// GetTSDBStatus returns TSDB status for given filters on the given date.
+func (vms *VMStorage) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return nil, err
+	}
+	date := uint64(sq.MinTimestamp) / (24 * 3600 * 1000)
+	return vms.s.GetTSDBStatus(qt, sq.AccountID, sq.ProjectID, tfss, date, focusLabel, topN, maxMetrics, deadline)
+}
+
+// DeleteSeries deletes series matching tfss.
+//
+// Returns the number of deleted series.
+func (vms *VMStorage) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	tr := sq.GetTimeRange()
+	maxMetrics := sq.MaxMetrics
+	if maxMetrics <= 0 {
+		// fallback to maxUniqueTimeSeries if no limit is provided,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857
+		maxMetrics = vms.maxUniqueTimeSeriesCalculated
+	}
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		return 0, err
+	}
+	if len(tfss) == 0 {
+		return 0, fmt.Errorf("missing tag filters")
+	}
+	return vms.s.DeleteSeries(qt, tfss, maxMetrics)
+}
+
+func (vms *VMStorage) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, _ uint64) error {
+	vms.s.RegisterMetricNames(qt, mrs)
+	return nil
+}
+
+// GetMetricNamesUsageStats returns metric name usage stats.
+func (vms *VMStorage) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, _ uint64) (metricnamestats.StatsResult, error) {
+	return vms.s.GetMetricNamesStats(qt, tt, limit, le, matchPattern), nil
+}
+
+// ResetMetricNamesStats resets state for metric names usage tracker
+func (vms *VMStorage) ResetMetricNamesUsageStats(qt *querytracer.Tracer, _ uint64) error {
+	vms.s.ResetMetricNamesStats(qt)
+	return nil
+}
+
+func (vms *VMStorage) setupTfss(qt *querytracer.Tracer, sq *storage.SearchQuery, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]*storage.TagFilters, error) {
+	tfss := make([]*storage.TagFilters, 0, len(sq.TagFilterss))
+	accountID := sq.AccountID
+	projectID := sq.ProjectID
+	for _, tagFilters := range sq.TagFilterss {
+		tfs := storage.NewTagFilters(accountID, projectID)
+		for i := range tagFilters {
+			tf := &tagFilters[i]
+			if string(tf.Key) == "__graphite__" {
+				query := tf.Value
+				qtChild := qt.NewChild("searching for series matching __graphite__=%q", query)
+				paths, err := vms.s.SearchGraphitePaths(qtChild, accountID, projectID, tr, query, maxMetrics, deadline)
+				qtChild.Donef("found %d series", len(paths))
+				if err != nil {
+					return nil, fmt.Errorf("error when searching for Graphite paths for query %q: %w", query, err)
+				}
+				if len(paths) >= maxMetrics {
+					return nil, fmt.Errorf("more than %d time series match Graphite query %q; "+
+						"either narrow down the query or increase the corresponding -search.max* command-line flag value at vmselect nodes; "+
+						"see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#resource-usage-limits", maxMetrics, query)
+				}
+				tfs.AddGraphiteQuery(query, paths, tf.IsNegative)
+				continue
+			}
+			if err := tfs.Add(tf.Key, tf.Value, tf.IsNegative, tf.IsRegexp); err != nil {
+				return nil, fmt.Errorf("cannot parse tag filter %s: %w", tf, err)
+			}
+		}
+		tfss = append(tfss, tfs)
+	}
+	return tfss, nil
+}
+
+func (vms *VMStorage) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
+	return vms.s.GetMetadataRows(qt, tt, limit, metricName, deadline)
+}
+
+// deleteSnapshot deletes a snapshot by its name.
+func (vms *VMStorage) deleteSnapshot(snapshotName string) error {
+	snapshots := vms.s.MustListSnapshots()
+	for _, snName := range snapshots {
+		if snName == snapshotName {
+			if err := vms.s.DeleteSnapshot(snName); err != nil {
+				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("cannot find snapshot %q", snapshotName)
+}

app/vmstorage/vmstorage_test.go

@@ -1,22 +1,26 @@
-package servers
+package main
 
 import (
 	"math"
-	"runtime"
+	"strconv"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 func TestCalculateMaxMetricsLimitByResource(t *testing.T) {
 	f := func(maxConcurrentRequest, remainingMemory, expect int) {
 		t.Helper()
-		maxMetricsLimit := calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequest, remainingMemory)
+		maxMetricsLimit := calculateMaxUniqueTimeseries(maxConcurrentRequest, remainingMemory)
 		if maxMetricsLimit != expect {
 			t.Fatalf("unexpected max metrics limit: got %d, want %d", maxMetricsLimit, expect)
 		}
 	}
 
-	// Skip when GOARCH=386
-	if runtime.GOARCH != "386" {
+	// 64-bit architectures support memory sizes > 4GB.
+	if strconv.IntSize == 64 {
 		// 8 CPU & 32 GiB
 		f(16, int(math.Round(32*1024*1024*1024*0.4)), 4294967)
 		// 4 CPU & 32 GiB
@@ -36,11 +40,17 @@ func TestGetMaxMetrics(t *testing.T) {
 	originalMaxUniqueTimeSeries := *maxUniqueTimeseries
 	defer func() {
 		*maxUniqueTimeseries = originalMaxUniqueTimeSeries
+		fs.MustRemoveDir(t.Name())
 	}()
+
+	maxConcurrentRequests := 2 * cgroup.AvailableCPUs()
 	f := func(searchQueryLimit, storageMaxUniqueTimeseries, expect int) {
 		t.Helper()
 		*maxUniqueTimeseries = storageMaxUniqueTimeseries
-		maxMetrics := getMaxMetrics(searchQueryLimit)
+		s := storage.MustOpenStorage(t.Name(), storage.OpenOptions{})
+		vms := newVMStorage(s, maxConcurrentRequests)
+		defer vms.Stop()
+		maxMetrics := vms.getMaxMetrics(searchQueryLimit)
 		if maxMetrics != expect {
 			t.Fatalf("unexpected max metrics: got %d, want %d", maxMetrics, expect)
 		}

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.3 AS build-web-stage
+FROM golang:1.26.4 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts

@@ -1,7 +1,7 @@
-export const seriesFetchedWarning = `No match! 
+export const seriesFetchedWarning = `No match!
 This query hasn't selected any time series from database.
 Either the requested metrics are missing in the database,
 or there is a typo in series selector.`;
 
 export const partialWarning = `The shown results are marked as PARTIAL.
-The result is marked as partial if one or more vmstorage nodes failed to respond to the query.`;
+The result is marked as partial if one or more storage nodes failed to respond to the query.`;

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/index.tsx

@@ -71,7 +71,7 @@ const RulesHeader = ({
           <TextField
             label="Search"
             value={search}
-            placeholder="Filter by rule, name or labels"
+            placeholder="Filter by group or rule name"
             startIcon={<SearchIcon />}
             onChange={onChangeSearch}
           />

app/vmui/packages/vmui/src/components/Main/Tabs/TabItem.tsx

@@ -1,6 +1,5 @@
 import { Component, FC, Ref } from "preact/compat";
 import classNames from "classnames";
-import { getCssVariable } from "../../../utils/theme";
 import { TabItemType } from "./Tabs";
 import TabItemWrapper from "./TabItemWrapper";
 import "./style.scss";
@@ -8,7 +7,6 @@ import "./style.scss";
 interface TabItemProps {
   activeItem: string
   item: TabItemType
-  color?: string
   onChange?: (value: string) => void
   activeNavRef: Ref<Component>
   isNavLink?: boolean
@@ -17,7 +15,6 @@ interface TabItemProps {
 const TabItem: FC<TabItemProps> = ({
   activeItem,
   item,
-  color = getCssVariable("color-primary"),
   activeNavRef,
   onChange,
   isNavLink
@@ -35,7 +32,6 @@ const TabItem: FC<TabItemProps> = ({
       })}
       isNavLink={isNavLink}
       to={item.value}
-      style={{ color: color }}
       onClick={createHandlerClickTab(item.value)}
       ref={activeItem === item.value ? activeNavRef : undefined}
     >

app/vmui/packages/vmui/src/components/Main/Tabs/TabItemWrapper.tsx

@@ -6,7 +6,6 @@ interface TabItemWrapperProps {
   to: string
   isNavLink?: boolean
   className: string
-  style: { color: string }
   children: ReactNode
   onClick: () => void
 }

app/vmui/packages/vmui/src/components/Main/Tabs/Tabs.tsx

@@ -1,6 +1,5 @@
 import { Component, FC, useRef, useState } from "preact/compat";
 import { ReactNode, useEffect } from "react";
-import { getCssVariable } from "../../../utils/theme";
 import TabItem from "./TabItem";
 import "./style.scss";
 import useWindowSize from "../../../hooks/useWindowSize";
@@ -15,7 +14,6 @@ export interface TabItemType {
 interface TabsProps {
   activeItem: string
   items: TabItemType[]
-  color?: string
   onChange?: (value: string) => void
   indicatorPlacement?: "bottom" | "top"
   isNavLink?: boolean
@@ -24,7 +22,6 @@ interface TabsProps {
 const Tabs: FC<TabsProps> = ({
   activeItem,
   items,
-  color = getCssVariable("color-primary"),
   onChange,
   indicatorPlacement = "bottom",
   isNavLink,
@@ -48,14 +45,13 @@ const Tabs: FC<TabsProps> = ({
         activeItem={activeItem}
         item={item}
         onChange={onChange}
-        color={color}
         activeNavRef={activeNavRef}
         isNavLink={isNavLink}
       />
     ))}
     <div
       className="vm-tabs__indicator"
-      style={{ ...indicatorPosition, borderColor: color }}
+      style={{ ...indicatorPosition }}
     />
   </div>;
 };

app/vmui/packages/vmui/src/components/Main/Tabs/style.scss

@@ -14,7 +14,7 @@
     align-items: center;
     justify-content: center;
     padding: $padding-global $padding-small;
-    color: inherit;
+    color: $color-primary;
     text-decoration: none;
     text-transform: capitalize;
     font-size: inherit;
@@ -46,5 +46,6 @@
     position: absolute;
     border-bottom: 2px solid;
     transition: width 200ms ease, left 300ms cubic-bezier(0.280, 0.840, 0.420, 1);
+    border-color: $color-primary;
   }
 }

app/vmui/packages/vmui/src/state/customPanel/reducer.test.ts

@@ -0,0 +1,35 @@
+import { afterEach, describe, expect, it, vi, type Mock } from "vitest";
+import { getFromStorage, saveToStorage } from "../../utils/storage";
+
+vi.mock("../../utils/storage", () => ({
+  getFromStorage: vi.fn(),
+  saveToStorage: vi.fn(),
+}));
+
+describe("customPanel reducer", () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+    vi.resetModules();
+  });
+
+  it("persists reduceMemUsage under its own storage key", async () => {
+    const { reducer, initialCustomPanelState } = await import("./reducer");
+
+    reducer(initialCustomPanelState, { type: "TOGGLE_REDUCE_MEM_USAGE" });
+
+    expect(saveToStorage).toHaveBeenCalledWith("REDUCE_MEM_USAGE", true);
+    expect(saveToStorage).not.toHaveBeenCalledWith("TABLE_COMPACT", true);
+  });
+
+  it("hydrates reduceMemUsage from storage", async () => {
+    const getFromStorageMock = getFromStorage as Mock;
+    getFromStorageMock.mockImplementation((key: string) => {
+      if (key === "REDUCE_MEM_USAGE") return true;
+      return undefined;
+    });
+
+    const { initialCustomPanelState } = await import("./reducer");
+
+    expect(initialCustomPanelState.reduceMemUsage).toBe(true);
+  });
+});

app/vmui/packages/vmui/src/state/customPanel/reducer.ts

@@ -35,7 +35,7 @@ export const initialCustomPanelState: CustomPanelState = {
   isTracingEnabled: false,
   seriesLimits: limitsStorage ? JSON.parse(limitsStorage) : DEFAULT_MAX_SERIES,
   tableCompact: getFromStorage("TABLE_COMPACT") as boolean || false,
-  reduceMemUsage: false
+  reduceMemUsage: getFromStorage("REDUCE_MEM_USAGE") as boolean || false
 };
 
 export function reducer(state: CustomPanelState, action: CustomPanelAction): CustomPanelState {
@@ -69,7 +69,7 @@ export function reducer(state: CustomPanelState, action: CustomPanelAction): Cus
         tableCompact: !state.tableCompact
       };
     case "TOGGLE_REDUCE_MEM_USAGE":
-      saveToStorage("TABLE_COMPACT", !state.reduceMemUsage);
+      saveToStorage("REDUCE_MEM_USAGE", !state.reduceMemUsage);
       return {
         ...state,
         reduceMemUsage: !state.reduceMemUsage

app/vmui/packages/vmui/src/utils/storage.ts

@@ -7,6 +7,7 @@ export const ALL_STORAGE_KEYS = [
   "SERIES_LIMITS",
   "LEGEND_AUTO_COLLAPSE",
   "TABLE_COMPACT",
+  "REDUCE_MEM_USAGE",
   "TIMEZONE",
   "DISABLED_DEFAULT_TIMEZONE",
   "THEME",

apptest/app.go

@@ -22,6 +22,7 @@ var (
 	vminsertAddrRE              = regexp.MustCompile(`accepting vminsert conns at (.*:\d{1,5})$`)
 	vminsertClusterNativeAddrRE = regexp.MustCompile(`started TCP clusternative server at "(.*:\d{1,5})"`)
 	vmselectAddrRE              = regexp.MustCompile(`accepting vmselect conns at (.*:\d{1,5})$`)
+	vmauthHttpListenAddrRE      = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
 )
 
 // app represents an instance of some VictoriaMetrics server (such as vmstorage,

apptest/model.go

@@ -79,24 +79,25 @@ type PrometheusWriteQuerier interface {
 
 // QueryOpts contains various params used for querying or ingesting data
 type QueryOpts struct {
-	Tenant         string
-	Timeout        string
-	Start          string
-	End            string
-	Time           string
-	Step           string
-	ExtraFilters   []string
-	ExtraLabels    []string
-	Trace          string
-	ReduceMemUsage string
-	MaxLookback    string
-	LatencyOffset  string
-	Format         string
-	NoCache        string
-	Headers        http.Header
-	From           string
-	Until          string
-	StorageStep    string
+	Tenant              string
+	Timeout             string
+	Start               string
+	End                 string
+	Time                string
+	Step                string
+	ExtraFilters        []string
+	ExtraLabels         []string
+	Trace               string
+	ReduceMemUsage      string
+	MaxLookback         string
+	LatencyOffset       string
+	Format              string
+	NoCache             string
+	Headers             http.Header
+	From                string
+	Until               string
+	StorageStep         string
+	DenyPartialResponse string
 }
 
 func (qos *QueryOpts) getHeaders() http.Header {
@@ -132,6 +133,7 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("from", qos.From)
 	addNonEmpty("until", qos.Until)
 	addNonEmpty("storage_step", qos.StorageStep)
+	addNonEmpty("deny_partial_response", qos.DenyPartialResponse)
 
 	return uv
 }

apptest/testcase.go

@@ -88,19 +88,11 @@ func (tc *TestCase) MustStartDefaultVmsingle() *Vmsingle {
 }
 
 // MustStartVmsingle is a test helper function that starts an instance of
-// vmsingle located at ../../bin/victoria-metrics-race and fails the test if the app
-// fails to start.
+// vmsingle (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
 	tc.t.Helper()
-	return tc.MustStartVmsingleAt(instance, "../../bin/victoria-metrics-race", flags)
-}
 
-// MustStartVmsingleAt is a test helper function that starts an instance of
-// vmsingle and fails the test if the app fails to start.
-func (tc *TestCase) MustStartVmsingleAt(instance, binary string, flags []string) *Vmsingle {
-	tc.t.Helper()
-
-	app, err := StartVmsingleAt(instance, binary, flags, tc.cli, tc.output)
+	app, err := StartVmsingle(instance, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -109,19 +101,11 @@ func (tc *TestCase) MustStartVmsingleAt(instance, binary string, flags []string)
 }
 
 // MustStartVmstorage is a test helper function that starts an instance of
-// vmstorage located at ../../bin/vmstorage-race and fails the test if the app fails
-// to start.
+// vmstorage (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmstorage(instance string, flags []string) *Vmstorage {
 	tc.t.Helper()
-	return tc.MustStartVmstorageAt(instance, "../../bin/vmstorage-race", flags)
-}
 
-// MustStartVmstorageAt is a test helper function that starts an instance of
-// vmstorage and fails the test if the app fails to start.
-func (tc *TestCase) MustStartVmstorageAt(instance string, binary string, flags []string) *Vmstorage {
-	tc.t.Helper()
-
-	app, err := StartVmstorageAt(instance, binary, flags, tc.cli, tc.output)
+	app, err := StartVmstorage(instance, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -130,7 +114,7 @@ func (tc *TestCase) MustStartVmstorageAt(instance string, binary string, flags [
 }
 
 // MustStartVmselect is a test helper function that starts an instance of
-// vmselect and fails the test if the app fails to start.
+// vmselect (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmselect(instance string, flags []string) *Vmselect {
 	tc.t.Helper()
 
@@ -290,10 +274,8 @@ func (tc *TestCase) MustStartDefaultCluster() *Vmcluster {
 // tests usually come paired with corresponding vmsingle tests.
 type ClusterOptions struct {
 	Vmstorage1Instance string
-	Vmstorage1Binary   string
 	Vmstorage1Flags    []string
 	Vmstorage2Instance string
-	Vmstorage2Binary   string
 	Vmstorage2Flags    []string
 	VminsertInstance   string
 	VminsertFlags      []string
@@ -305,15 +287,8 @@ type ClusterOptions struct {
 func (tc *TestCase) MustStartCluster(opts *ClusterOptions) *Vmcluster {
 	tc.t.Helper()
 
-	if opts.Vmstorage1Binary == "" {
-		opts.Vmstorage1Binary = "../../bin/vmstorage-race"
-	}
-	vmstorage1 := tc.MustStartVmstorageAt(opts.Vmstorage1Instance, opts.Vmstorage1Binary, opts.Vmstorage1Flags)
-
-	if opts.Vmstorage2Binary == "" {
-		opts.Vmstorage2Binary = "../../bin/vmstorage-race"
-	}
-	vmstorage2 := tc.MustStartVmstorageAt(opts.Vmstorage2Instance, opts.Vmstorage2Binary, opts.Vmstorage2Flags)
+	vmstorage1 := tc.MustStartVmstorage(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
+	vmstorage2 := tc.MustStartVmstorage(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
 
 	opts.VminsertFlags = append(opts.VminsertFlags, []string{
 		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),

apptest/testcase_legacy.go

@@ -0,0 +1,50 @@
+package apptest
+
+// MustStartVmsingle_v1_132_0 is a test helper function that starts an instance
+// of vmsingle-v1.132.0 (last version that uses legacy index) and fails the test
+// if the app fails to start.
+func (tc *TestCase) MustStartVmsingle_v1_132_0(instance string, flags []string) *Vmsingle {
+	tc.t.Helper()
+
+	app, err := StartVmsingle_v1_132_0(instance, flags, tc.cli, tc.output)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
+// MustStartVmstorage_v1_132_0 is a test helper function that starts an instance
+// of vmstorage-v1.132.0 (last version that uses legacy index)  and fails the
+// test if the app fails to start.
+func (tc *TestCase) MustStartVmstorage_v1_132_0(instance string, flags []string) *Vmstorage {
+	tc.t.Helper()
+
+	app, err := StartVmstorage_v1_132_0(instance, flags, tc.cli, tc.output)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
+// MustStartCluster_v1_132_0 starts a cluster with vmstorage-v1.132.0 with
+// custom flags.
+func (tc *TestCase) MustStartCluster_v1_132_0(opts *ClusterOptions) *Vmcluster {
+	tc.t.Helper()
+
+	vmstorage1 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
+	vmstorage2 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
+
+	opts.VminsertFlags = append(opts.VminsertFlags, []string{
+		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
+	}...)
+	vminsert := tc.MustStartVminsert(opts.VminsertInstance, opts.VminsertFlags)
+
+	opts.VmselectFlags = append(opts.VmselectFlags, []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	}...)
+	vmselect := tc.MustStartVmselect(opts.VmselectInstance, opts.VmselectFlags)
+
+	return &Vmcluster{vminsert, vmselect, []*Vmstorage{vmstorage1, vmstorage2}}
+}

apptest/tests/legacy_indexdb_test.go

@@ -2,7 +2,6 @@ package tests
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"slices"
 	"testing"
@@ -11,11 +10,6 @@ import (
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
-var (
-	legacyVmsinglePath  = os.Getenv("VM_LEGACY_VMSINGLE_PATH")
-	legacyVmstoragePath = os.Getenv("VM_LEGACY_VMSTORAGE_PATH")
-)
-
 type testLegacyDeleteSeriesOpts struct {
 	startLegacySUT func() at.PrometheusWriteQuerier
 	startNewSUT    func() at.PrometheusWriteQuerier
@@ -31,7 +25,7 @@ func TestLegacySingleDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.maxStalenessInterval=1m",
@@ -64,15 +58,13 @@ func TestLegacyClusterDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -255,7 +247,7 @@ func TestLegacySingleBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -298,15 +290,13 @@ func TestLegacyClusterBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -583,7 +573,7 @@ func TestLegacySingleDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -618,15 +608,13 @@ func TestLegacyClusterDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",

apptest/tests/replication_test.go

@@ -1015,35 +1015,42 @@ func testGroupSkipSlowReplicas(tc *apptest.TestCase, opts *testGroupReplicationO
 func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOpts) {
 	t := tc.T()
 
-	assertSeries := func(app *apptest.Vmselect, wantPartial bool) {
+	assertSeries := func(app *apptest.Vmselect, denyPartialResponse string, want *apptest.PrometheusAPIV1SeriesResponse) {
 		t.Helper()
 		tc.Assert(&apptest.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
 				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{
-					Start: "2024-01-01T00:00:00Z",
-					End:   "2024-01-31T00:00:00Z",
+					Start:               "2024-01-01T00:00:00Z",
+					End:                 "2024-01-31T00:00:00Z",
+					DenyPartialResponse: denyPartialResponse,
 				}).Sort()
 			},
-			Want: &apptest.PrometheusAPIV1SeriesResponse{
-				Status:    "success",
-				IsPartial: wantPartial,
-			},
+			Want: want,
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
+				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data", "Error"),
 			},
 		})
 	}
 
-	mustReturnPartialResponse := true
-	mustReturnFullResponse := false
+	allowPartialResponse := ""
+	denyPartialResponse := "1"
+
+	mustReturnPartialResponse := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: true,
+	}
+	mustReturnFullResponse := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "success",
+		IsPartial: false,
+	}
 
 	// All vmstorage replicas are available so both vmselects must return full
 	// response.
-	assertSeries(opts.c.vmselect, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop groupRF-1 vmstorage nodes in first group.
 	//
@@ -1053,10 +1060,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	// about the replication factor and therefore they must still be able to
 	// return full dataset.
 	opts.c.storageGroups[0].stopNodes(tc, opts.groupRF-1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop groupRF-1 vmstorages in the remaining groups.
 	//
@@ -1066,10 +1073,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	for g := 1; g < len(opts.c.storageGroups); g++ {
 		opts.c.storageGroups[g].stopNodes(tc, opts.groupRF-1)
 	}
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstorage in the first group.
 	//
@@ -1077,10 +1084,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	// because it is unaware of replication across groups. vmselectGroupGlobalRF
 	// will continue retuning full dataset.
 	opts.c.storageGroups[0].stopNodes(tc, 1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstoarge in remaining globarRF-1 groups.
 	//
@@ -1089,19 +1096,56 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
 	for g := 1; g < opts.globalRF-1; g++ {
 		opts.c.storageGroups[g].stopNodes(tc, 1)
 	}
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
 
 	// Stop one more vmstoarge in one more group.
 	//
 	// vmselectGroupGlobalRF must now return partial dataset.
 	opts.c.storageGroups[opts.globalRF].stopNodes(tc, 1)
-	assertSeries(opts.c.vmselect, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
-	assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+
+	// Stop all the remaining vmstorage nodes except a single node.
+	//
+	// At this point vmselects still must be able to return partial response
+	// because at least one vmstorage node has successfully returned results.
+	n := len(opts.c.storageGroups[0].vmstorages)
+	opts.c.storageGroups[0].stopNodes(tc, n-1)
+	for g := 1; g < len(opts.c.storageGroups); g++ {
+		n := len(opts.c.storageGroups[g].vmstorages)
+		opts.c.storageGroups[g].stopNodes(tc, n)
+	}
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
+
+	mustReturnUnavailableError := &apptest.PrometheusAPIV1SeriesResponse{
+		Status:    "error",
+		ErrorType: "503",
+	}
+
+	// vmselects must return an error for the same request when partial
+	// responses are denied explicitly.
+	assertSeries(opts.c.vmselect, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupRF, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGlobalRF, denyPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupGlobalRF, denyPartialResponse, mustReturnUnavailableError)
+
+	// Stop the last remaining vmstorage node.
+	//
+	// vmselects must return an error when there are no successful vmstorage
+	// responses.
+	opts.c.storageGroups[0].stopNodes(tc, 1)
+	assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnUnavailableError)
+	assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnUnavailableError)
 }
 
 // TestClusterReplication_PartialResponseMultitenant checks how vmselect handles some

apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/meta.json

@@ -0,0 +1,51 @@
+{
+	"ulid": "01JFJBS3YP1SHZ3PJQ6HK76EC3",
+	"minTime": 1734709200000,
+	"maxTime": 1734709320000,
+	"stats": {
+		"numSamples": 400,
+		"numSeries": 100,
+		"numChunks": 100
+	},
+	"compaction": {
+		"level": 1,
+		"sources": [
+			"01JFJBS3YP1SHZ3PJQ6HK76EC3"
+		],
+		"parents": [
+			{
+				"ulid": "00000000000000000000000000",
+				"minTime": 0,
+				"maxTime": 0
+			}
+		],
+		"hints": [
+			"from-out-of-order"
+		]
+	},
+	"version": 1,
+	"out_of_order": false,
+	"thanos": {
+		"labels": {},
+		"downsample": {
+			"resolution": 0
+		},
+		"source": "receive",
+		"segment_files": [
+			"000001"
+		],
+		"files": [
+			{
+				"rel_path": "chunks/000001",
+				"size_bytes": 4808
+			},
+			{
+				"rel_path": "index",
+				"size_bytes": 55021
+			},
+			{
+				"rel_path": "meta.json"
+			}
+		]
+	}
+}

apptest/tests/vmctl_mimir_migration_test.go

@@ -0,0 +1,140 @@
+package tests
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+)
+
+const (
+	testMimirPath             = "testdata/mimir-tsdb"
+	expectedMimirResponseFile = "./testdata/mimir-tsdb/expected_response.json"
+)
+
+func TestSingleVmctlMimirProtocol(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("cannot get current working directory: %s", err)
+	}
+
+	path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
+	vmctlFlags := []string{
+		`mimir`,
+		`--mimir-tenant-id=anonymous`,
+		`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
+		`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
+		`--mimir-custom-s3-endpoint=http://localhost:9000`,
+		`--mimir-path=` + path,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+		`--vm-concurrency=6`,
+		`--mimir-concurrency=6`,
+	}
+
+	testMimirProtocol(tc, vmsingleDst, vmctlFlags)
+}
+
+func TestClusterVmctlMimirProtocol(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	cluster := tc.MustStartDefaultCluster()
+	vmAddr := fmt.Sprintf("http://%s/", cluster.Vminsert.HTTPAddr())
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("cannot get current working directory: %s", err)
+	}
+
+	path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
+
+	vmctlFlags := []string{
+		`mimir`,
+		`--mimir-tenant-id=anonymous`,
+		`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
+		`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
+		`--mimir-custom-s3-endpoint=http://localhost:9000`,
+		`--mimir-path=` + path,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+		`--vm-concurrency=6`,
+		`--mimir-concurrency=6`,
+		`--vm-account-id=0`,
+	}
+
+	testMimirProtocol(tc, cluster, vmctlFlags)
+}
+
+func testMimirProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
+	t := tc.T()
+	t.Helper()
+
+	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+
+	// test for empty data request
+	got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
+		Step: "5m",
+		Time: "2025-06-02T17:14:00Z",
+	})
+
+	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	tc.MustStartVmctl("vmctl", vmctlFlags)
+
+	sut.ForceFlush(t)
+
+	// open the expected series response file
+	file, err := os.Open(expectedMimirResponseFile)
+	if err != nil {
+		t.Fatalf("cannot open expected series response file: %s", err)
+	}
+	defer file.Close()
+
+	bytes, err := io.ReadAll(file)
+	if err != nil {
+		t.Fatalf("cannot read expected series response file: %s", err)
+	}
+
+	var wantResponse apptest.PrometheusAPIV1QueryResponse
+	if err := json.Unmarshal(bytes, &wantResponse); err != nil {
+		t.Fatalf("cannot unmarshal expected series response file: %s", err)
+	}
+	wantResponse.Sort()
+
+	tc.Assert(&apptest.AssertOptions{
+		// For cluster version, we need to wait longer for the metrics to be stored
+		Retries: 300,
+		Msg:     `unexpected metrics stored on vmsingle via the prometheus protocol`,
+		Got: func() any {
+			expected := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
+				Start: "2024-12-01T15:31:10Z",
+				End:   "2024-12-31T15:32:20Z",
+			})
+			expected.Sort()
+			return expected.Data.Result
+		},
+		Want: wantResponse.Data.Result,
+		CmpOpts: []cmp.Option{
+			cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		},
+	})
+}

apptest/vmagent.go

@@ -16,43 +16,63 @@ import (
 	"github.com/golang/snappy"
 )
 
-// Vmagent holds the state of a vmagent app and provides vmagent-specific functions
-type Vmagent struct {
-	*app
-	*metricsClient
-
-	httpListenAddr string
-
-	cli *Client
-}
-
-// StartVmagent starts an instance of vmagent with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
+// StartVmagent starts the latest version of vmagent.
+//
+// The path to the binary can be provided via VMAGENT_PATH environment
+// variable. If the variable is not set, ../../bin/vmagent-race will be
+// used.
 func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfigFilePath string, output io.Writer) (*Vmagent, error) {
-	extractREs := []*regexp.Regexp{
-		httpListenAddrRE,
+	binary := os.Getenv("VMAGENT_PATH")
+	if binary == "" {
+		binary = "../../bin/vmagent-race"
 	}
-
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmagent-race", flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-promscrape.config":       promScrapeConfigFilePath,
 			"-remoteWrite.tmpDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
 		},
-		extractREs: extractREs,
-		output:     output,
+		extractREs: []*regexp.Regexp{
+			httpListenAddrRE,
+		},
+		output: output,
 	})
 	if err != nil {
 		return nil, err
 	}
 
+	return newVmagent(app, cli, vmagentRuntimeValues{
+		httpListenAddr: stderrExtracts[0],
+	}), nil
+}
+
+type vmagentRuntimeValues struct {
+	httpListenAddr string
+}
+
+func newVmagent(app *app, cli *Client, rt vmagentRuntimeValues) *Vmagent {
 	return &Vmagent{
 		app:            app,
-		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
-		httpListenAddr: stderrExtracts[0],
 		cli:            cli,
-	}, nil
+		metricsClient:  newMetricsClient(cli, rt.httpListenAddr),
+		httpListenAddr: rt.httpListenAddr,
+	}
+}
+
+// Vmagent holds the state of a vmagent app and provides vmagent-specific
+// functions.
+type Vmagent struct {
+	*app
+	*metricsClient
+
+	cli            *Client
+	httpListenAddr string
+}
+
+// HTTPAddr returns the address at which the vmagent process is listening
+// for http connections.
+func (app *Vmagent) HTTPAddr() string {
+	return app.httpListenAddr
 }
 
 // APIV1ImportPrometheus is a test helper function that inserts a
@@ -203,12 +223,6 @@ func (app *Vmagent) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, o
 	})
 }
 
-// HTTPAddr returns the address at which the vmagent process is listening
-// for http connections.
-func (app *Vmagent) HTTPAddr() string {
-	return app.httpListenAddr
-}
-
 // sendBlocking sends the data to vmstorage by executing `send` function and
 // waits until the data is actually sent.
 //

apptest/vmauth.go

@@ -2,6 +2,7 @@ package apptest
 
 import (
 	"io"
+	"os"
 	"regexp"
 	"syscall"
 	"testing"
@@ -10,7 +11,48 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 )
 
-var httpBuilitinListenAddrRE = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
+// StartVmauth starts the latest version of vmauth.
+//
+// The path to the binary can be provided via VMAUTH_PATH environment
+// variable. If the variable is not set, ../../bin/vmauth-race will be
+// used.
+func StartVmauth(instance string, flags []string, cli *Client, configFilePath string, output io.Writer) (*Vmauth, error) {
+	binary := os.Getenv("VMAUTH_PATH")
+	if binary == "" {
+		binary = "../../bin/vmauth-race"
+	}
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr": "127.0.0.1:0",
+			"-auth.config":    configFilePath,
+		},
+		extractREs: []*regexp.Regexp{
+			vmauthHttpListenAddrRE,
+		},
+		output: output,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return newVmauth(app, cli, configFilePath, vmauthRuntimeValues{
+		httpListenAddr: stderrExtracts[0],
+	}), nil
+}
+
+type vmauthRuntimeValues struct {
+	httpListenAddr string
+}
+
+func newVmauth(app *app, cli *Client, configFilePath string, rt vmauthRuntimeValues) *Vmauth {
+	return &Vmauth{
+		app:            app,
+		metricsClient:  newMetricsClient(cli, rt.httpListenAddr),
+		httpListenAddr: rt.httpListenAddr,
+		configFilePath: configFilePath,
+		cli:            cli,
+	}
+}
 
 // Vmauth holds the state of a vmauth app and provides vmauth-specific
 // functions.
@@ -18,38 +60,14 @@ type Vmauth struct {
 	*app
 	*metricsClient
 
+	cli            *Client
 	httpListenAddr string
 	configFilePath string
-	cli            *Client
 }
 
-// StartVmauth starts an instance of vmauth with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
-func StartVmauth(instance string, flags []string, cli *Client, configFilePath string, output io.Writer) (*Vmauth, error) {
-	extractREs := []*regexp.Regexp{
-		httpBuilitinListenAddrRE,
-	}
-
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmauth-race", flags, &appOptions{
-		defaultFlags: map[string]string{
-			"-httpListenAddr": "127.0.0.1:0",
-			"-auth.config":    configFilePath,
-		},
-		extractREs: extractREs,
-		output:     output,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return &Vmauth{
-		app:            app,
-		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
-		httpListenAddr: stderrExtracts[0],
-		configFilePath: configFilePath,
-		cli:            cli,
-	}, nil
+// GetHTTPListenAddr returns listen http addr
+func (app *Vmauth) GetHTTPListenAddr() string {
+	return app.httpListenAddr
 }
 
 // UpdateConfiguration updates the vmauth configuration file with the provided YAML content,
@@ -79,8 +97,3 @@ func (app *Vmauth) UpdateConfiguration(t *testing.T, configFileYAML string) {
 
 	t.Fatalf("config were not reloaded after SIGHUP signal; previous total: %d, current total: %d", prevTotal, currTotal)
 }
-
-// GetHTTPListenAddr returns listen http addr
-func (app *Vmauth) GetHTTPListenAddr() string {
-	return app.httpListenAddr
-}

apptest/vmbackup.go

@@ -1,15 +1,26 @@
 package apptest
 
-import "io"
+import (
+	"io"
+	"os"
+)
 
-// StartVmbackup starts an instance of vmbackup with the given flags and waits
-// until it exits.
+// StartVmbackup starts the latest version of vmbackup with the given flags and
+// waits until it exits.
+//
+// The path to the binary can be provided via VMBACKUP_PATH environment
+// variable. If the variable is not set, ../../bin/vmbackup-race will be
+// used.
 func StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string, output io.Writer) error {
+	binary := os.Getenv("VMBACKUP_PATH")
+	if binary == "" {
+		binary = "../../bin/vmbackup-race"
+	}
 	flags := []string{
 		"-storageDataPath=" + storageDataPath,
 		"-snapshot.createURL=" + snapshotCreateURL,
 		"-dst=" + dst,
 	}
-	_, _, err := startApp(instance, "../../bin/vmbackup-race", flags, &appOptions{wait: true, output: output})
+	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vmctl.go

@@ -1,9 +1,23 @@
 package apptest
 
-import "io"
+import (
+	"io"
+	"os"
+)
 
 // StartVmctl starts an instance of vmctl cli with the given flags
+
+// StartVmctl starts the latest version of vmctl with the given flags and
+// waits until it exits.
+//
+// The path to the binary can be provided via VMCTL_PATH environment
+// variable. If the variable is not set, ../../bin/vmctl-race will be
+// used.
 func StartVmctl(instance string, flags []string, output io.Writer) error {
-	_, _, err := startApp(instance, "../../bin/vmctl-race", flags, &appOptions{wait: true, output: output})
+	binary := os.Getenv("VMCTL_PATH")
+	if binary == "" {
+		binary = "../../bin/vmctl-race"
+	}
+	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vminsert.go

@@ -3,23 +3,13 @@ package apptest
 import (
 	"fmt"
 	"io"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
 )
 
-// Vminsert holds the state of a vminsert app and provides vminsert-specific
-// functions.
-type Vminsert struct {
-	*app
-	*metricsClient
-	*vminsertClient
-
-	httpListenAddr          string
-	clusternativeListenAddr string
-}
-
 // storageNodes returns the storage node addresses passed to vminsert via
 // -storageNode command line flag.
 func storageNodes(flags []string) []string {
@@ -31,9 +21,11 @@ func storageNodes(flags []string) []string {
 	return nil
 }
 
-// StartVminsert starts an instance of vminsert with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
+// StartVminsert starts the latest version of vminsert.
+//
+// The path to the binary can be provided via VMINSERT_PATH environment
+// variable. If the variable is not set, ../../bin/vminsert-race will be
+// used.
 func StartVminsert(instance string, flags []string, cli *Client, output io.Writer) (*Vminsert, error) {
 	extractREs := []*regexp.Regexp{
 		httpListenAddrRE,
@@ -48,11 +40,15 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		extractREs = append(extractREs, regexp.MustCompile(logRecord))
 	}
 
-	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert-race", flags, &appOptions{
+	binary := os.Getenv("VMINSERT_PATH")
+	if binary == "" {
+		binary = "../../bin/vminsert-race"
+	}
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":                              "127.0.0.1:0",
 			"-clusternativeListenAddr":                     "127.0.0.1:0",
-			"-graphiteListenAddr":                          ":0",
+			"-graphiteListenAddr":                          "127.0.0.1:0",
 			"-opentsdbListenAddr":                          "127.0.0.1:0",
 			"-clusternative.vminsertConnsShutdownDuration": "1ms",
 		},
@@ -63,27 +59,56 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
-	metricsClient := newMetricsClient(cli, stderrExtracts[0])
-	return &Vminsert{
-		app:           app,
-		metricsClient: metricsClient,
-		vminsertClient: &vminsertClient{
-			vminsertCli: cli,
-			url: func(op, path string, opts QueryOpts) string {
-				return getClusterPath(stderrExtracts[0], op, path, opts)
-			},
-			openTSDBURL: func(op, path string, opts QueryOpts) string {
-				return getClusterPath(stderrExtracts[3], op, path, opts)
-			},
-			graphiteListenAddr: stderrExtracts[2],
-			sendBlocking: func(t *testing.T, numRecordsToSend int, send func()) {
-				t.Helper()
-				sendBlocking(t, metricsClient, numRecordsToSend, send)
-			},
-		},
+	return newVminsert(app, cli, vminsertRuntimeValues{
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
-	}, nil
+		graphiteListenAddr:      stderrExtracts[2],
+		openTSDBListenAddr:      stderrExtracts[3],
+	}), nil
+}
+
+type vminsertRuntimeValues struct {
+	httpListenAddr          string
+	clusternativeListenAddr string
+	graphiteListenAddr      string
+	openTSDBListenAddr      string
+}
+
+func newVminsert(app *app, cli *Client, rt vminsertRuntimeValues) *Vminsert {
+	metricsClient := newMetricsClient(cli, rt.httpListenAddr)
+	vminsertClient := &vminsertClient{
+		vminsertCli: cli,
+		url: func(op, path string, opts QueryOpts) string {
+			return getClusterPath(rt.httpListenAddr, op, path, opts)
+		},
+		openTSDBURL: func(op, path string, opts QueryOpts) string {
+			return getClusterPath(rt.openTSDBListenAddr, op, path, opts)
+		},
+		graphiteListenAddr: rt.graphiteListenAddr,
+		sendBlocking: func(t *testing.T, numRecordsToSend int, send func()) {
+			t.Helper()
+			sendBlocking(t, metricsClient, numRecordsToSend, send)
+		},
+	}
+
+	return &Vminsert{
+		app:                     app,
+		metricsClient:           metricsClient,
+		vminsertClient:          vminsertClient,
+		httpListenAddr:          rt.httpListenAddr,
+		clusternativeListenAddr: rt.clusternativeListenAddr,
+	}
+}
+
+// Vminsert holds the state of a vminsert app and provides vminsert-specific
+// functions.
+type Vminsert struct {
+	*app
+	*metricsClient
+	*vminsertClient
+
+	httpListenAddr          string
+	clusternativeListenAddr string
 }
 
 // ClusternativeListenAddr returns the address at which the vminsert process is

apptest/vmrestore.go

@@ -1,14 +1,25 @@
 package apptest
 
-import "io"
+import (
+	"io"
+	"os"
+)
 
-// StartVmrestore starts an instance of vmrestore with the given flags and waits
-// until it exits.
+// StartVmrestore starts the latest version of vmrestore with the given flags
+// and waits until it exits.
+//
+// The path to the binary can be provided via VMRESTORE_PATH environment
+// variable. If the variable is not set, ../../bin/vmrestore-race will be
+// used.
 func StartVmrestore(instance, src, storageDataPath string, output io.Writer) error {
+	binary := os.Getenv("VMRESTORE_PATH")
+	if binary == "" {
+		binary = "../../bin/vmrestore-race"
+	}
 	flags := []string{
 		"-src=" + src,
 		"-storageDataPath=" + storageDataPath,
 	}
-	_, _, err := startApp(instance, "../../bin/vmrestore-race", flags, &appOptions{wait: true, output: output})
+	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vmselect.go

@@ -3,26 +3,21 @@ package apptest
 import (
 	"fmt"
 	"io"
+	"os"
 	"regexp"
 )
 
-// Vmselect holds the state of a vmselect app and provides vmselect-specific
-// functions.
-type Vmselect struct {
-	*app
-	*metricsClient
-	*vmselectClient
-
-	httpListenAddr          string
-	clusternativeListenAddr string
-	cli                     *Client
-}
-
-// StartVmselect starts an instance of vmselect with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
+// StartVmselect starts the latest version of vmselect.
+//
+// The path to the binary can be provided via VMSELECT_PATH environment
+// variable. If the variable is not set, ../../bin/vmselect-race will be
+// used.
 func StartVmselect(instance string, flags []string, cli *Client, output io.Writer) (*Vmselect, error) {
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect-race", flags, &appOptions{
+	binary := os.Getenv("VMSELECT_PATH")
+	if binary == "" {
+		binary = "../../bin/vmselect-race"
+	}
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-clusternativeListenAddr": "127.0.0.1:0",
@@ -37,21 +32,43 @@ func StartVmselect(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
+	return newVmselect(app, cli, vmselectRuntimeValues{
+		httpListenAddr:          stderrExtracts[0],
+		clusternativeListenAddr: stderrExtracts[1],
+	}), nil
+}
+
+type vmselectRuntimeValues struct {
+	httpListenAddr          string
+	clusternativeListenAddr string
+}
+
+func newVmselect(app *app, cli *Client, rt vmselectRuntimeValues) *Vmselect {
 	return &Vmselect{
 		app:           app,
-		metricsClient: newMetricsClient(cli, stderrExtracts[0]),
+		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
 		vmselectClient: &vmselectClient{
 			vmselectCli: cli,
 			url: func(op, path string, opts QueryOpts) string {
-				return getClusterPath(stderrExtracts[0], op, path, opts)
+				return getClusterPath(rt.httpListenAddr, op, path, opts)
 			},
-			metricNamesStatsResetURL: fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[0]),
-			tenantsURL:               fmt.Sprintf("http://%s/admin/tenants", stderrExtracts[0]),
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", rt.httpListenAddr),
+			tenantsURL:               fmt.Sprintf("http://%s/admin/tenants", rt.httpListenAddr),
 		},
-		httpListenAddr:          stderrExtracts[0],
-		clusternativeListenAddr: stderrExtracts[1],
-		cli:                     cli,
-	}, nil
+		httpListenAddr:          rt.httpListenAddr,
+		clusternativeListenAddr: rt.clusternativeListenAddr,
+	}
+}
+
+// Vmselect holds the state of a vmselect app and provides vmselect-specific
+// functions.
+type Vmselect struct {
+	*app
+	*metricsClient
+	*vmselectClient
+
+	httpListenAddr          string
+	clusternativeListenAddr string
 }
 
 // ClusternativeListenAddr returns the address at which the vmselect process is

apptest/vmsingle.go

@@ -9,28 +9,21 @@ import (
 	"time"
 )
 
-// Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
-// functions.
-type Vmsingle struct {
-	*app
-	*metricsClient
-	*vmstorageClient
-	*vmselectClient
-	*vminsertClient
-
-	storageDataPath string
-	httpListenAddr  string
-}
-
-// StartVmsingleAt starts an instance of vmsingle with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr).
-func StartVmsingleAt(instance, binary string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
+// StartVmsingle starts the latest version of vmsingle.
+//
+// The path to the binary can be provided via VMSINGLE_PATH environment
+// variable. If the variable is not set, ../../bin/victoria-metrics-race will be
+// used.
+func StartVmsingle(instance string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
+	binary := os.Getenv("VMSINGLE_PATH")
+	if binary == "" {
+		binary = "../../bin/victoria-metrics-race"
+	}
 	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-storageDataPath":    fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
 			"-httpListenAddr":     "127.0.0.1:0",
-			"-graphiteListenAddr": ":0",
+			"-graphiteListenAddr": "127.0.0.1:0",
 			"-opentsdbListenAddr": "127.0.0.1:0",
 		},
 		extractREs: []*regexp.Regexp{
@@ -45,38 +38,67 @@ func StartVmsingleAt(instance, binary string, flags []string, cli *Client, outpu
 		return nil, err
 	}
 
+	return newVmsingle(app, cli, vmsingleRuntimeValues{
+		storageDataPath:    stderrExtracts[0],
+		httpListenAddr:     stderrExtracts[1],
+		graphiteListenAddr: stderrExtracts[2],
+		openTSDBListenAddr: stderrExtracts[3],
+	}), nil
+}
+
+type vmsingleRuntimeValues struct {
+	storageDataPath    string
+	httpListenAddr     string
+	graphiteListenAddr string
+	openTSDBListenAddr string
+}
+
+func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
 	return &Vmsingle{
 		app:           app,
-		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
+		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
 		vmstorageClient: &vmstorageClient{
 			vmstorageCli:   cli,
-			httpListenAddr: stderrExtracts[1],
+			httpListenAddr: rt.httpListenAddr,
 		},
 		vmselectClient: &vmselectClient{
 			vmselectCli: cli,
 			url: func(op, path string, opts QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
 			},
-			metricNamesStatsResetURL: fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[1]),
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", rt.httpListenAddr),
 			tenantsURL:               "vmsingle-does-not-serve-tenants",
 		},
 		vminsertClient: &vminsertClient{
 			vminsertCli: cli,
 			url: func(_, path string, _ QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
 			},
 			openTSDBURL: func(_, path string, _ QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", stderrExtracts[3], path)
+				return fmt.Sprintf("http://%s/%s", rt.openTSDBListenAddr, path)
 			},
-			graphiteListenAddr: stderrExtracts[2],
+			graphiteListenAddr: rt.graphiteListenAddr,
 			sendBlocking: func(t *testing.T, _ int, send func()) {
 				t.Helper()
 				send()
 			},
 		},
-		storageDataPath: stderrExtracts[0],
-		httpListenAddr:  stderrExtracts[1],
-	}, nil
+		storageDataPath: rt.storageDataPath,
+		httpListenAddr:  rt.httpListenAddr,
+	}
+}
+
+// Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
+// functions.
+type Vmsingle struct {
+	*app
+	*metricsClient
+	*vmstorageClient
+	*vmselectClient
+	*vminsertClient
+
+	storageDataPath string
+	httpListenAddr  string
 }
 
 // HTTPAddr returns the address at which the vminsert process is

apptest/vmsingle_legacy.go

@@ -0,0 +1,43 @@
+package apptest
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"time"
+)
+
+// StartVmsingle_v1_132_0 starts vmsingle-v1.132.0 (the last version that uses
+// legacy index).
+//
+// The path to the binary must be provided via VMSINGLE_V1_132_0_PATH
+// environment variable.
+func StartVmsingle_v1_132_0(instance string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
+	binary := os.Getenv("VMSINGLE_V1_132_0_PATH")
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-storageDataPath":    fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
+			"-httpListenAddr":     "127.0.0.1:0",
+			"-graphiteListenAddr": "127.0.0.1:0",
+			"-opentsdbListenAddr": "127.0.0.1:0",
+		},
+		extractREs: []*regexp.Regexp{
+			storageDataPathRE,
+			httpListenAddrRE,
+			graphiteListenAddrRE,
+			openTSDBListenAddrRE,
+		},
+		output: output,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return newVmsingle(app, cli, vmsingleRuntimeValues{
+		storageDataPath:    stderrExtracts[0],
+		httpListenAddr:     stderrExtracts[1],
+		graphiteListenAddr: stderrExtracts[2],
+		openTSDBListenAddr: stderrExtracts[3],
+	}), nil
+}

apptest/vmstorage.go

@@ -8,23 +8,22 @@ import (
 	"time"
 )
 
-// Vmstorage holds the state of a vmstorage app and provides vmstorage-specific
-// functions.
-type Vmstorage struct {
-	*app
-	*metricsClient
-	*vmstorageClient
-
-	storageDataPath string
-	httpListenAddr  string
-	vminsertAddr    string
-	vmselectAddr    string
+// StartVmstorage starts the latest version of vmstorage.
+//
+// The path to the binary can be provided via VMSTORAGE_PATH environment
+// variable. If the variable is not set, ../../bin/vmstorage-race will be used.
+func StartVmstorage(instance string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
+	binary := os.Getenv("VMSTORAGE_PATH")
+	if binary == "" {
+		binary = "../../bin/vmstorage-race"
+	}
+	return startVmstorage(instance, binary, flags, cli, output)
 }
 
-// StartVmstorageAt starts an instance of vmstorage with the given flags. It also
+// startVmstorage starts an instance of vmstorage with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)
-func StartVmstorageAt(instance, binary string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
+func startVmstorage(instance, binary string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
 	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-storageDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
@@ -44,18 +43,47 @@ func StartVmstorageAt(instance, binary string, flags []string, cli *Client, outp
 		return nil, err
 	}
 
-	return &Vmstorage{
-		app:           app,
-		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
-		vmstorageClient: &vmstorageClient{
-			vmstorageCli:   cli,
-			httpListenAddr: stderrExtracts[1],
-		},
+	return newVmstorage(app, cli, vmstorageRuntimeValues{
 		storageDataPath: stderrExtracts[0],
 		httpListenAddr:  stderrExtracts[1],
 		vminsertAddr:    stderrExtracts[2],
 		vmselectAddr:    stderrExtracts[3],
-	}, nil
+	}), nil
+}
+
+type vmstorageRuntimeValues struct {
+	storageDataPath string
+	httpListenAddr  string
+	vminsertAddr    string
+	vmselectAddr    string
+}
+
+func newVmstorage(app *app, cli *Client, rt vmstorageRuntimeValues) *Vmstorage {
+	return &Vmstorage{
+		app:           app,
+		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
+		vmstorageClient: &vmstorageClient{
+			vmstorageCli:   cli,
+			httpListenAddr: rt.httpListenAddr,
+		},
+		storageDataPath: rt.storageDataPath,
+		httpListenAddr:  rt.httpListenAddr,
+		vminsertAddr:    rt.vminsertAddr,
+		vmselectAddr:    rt.vmselectAddr,
+	}
+}
+
+// Vmstorage holds the state of a vmstorage app and provides vmstorage-specific
+// functions.
+type Vmstorage struct {
+	*app
+	*metricsClient
+	*vmstorageClient
+
+	storageDataPath string
+	httpListenAddr  string
+	vminsertAddr    string
+	vmselectAddr    string
 }
 
 // VminsertAddr returns the address at which the vmstorage process is listening

apptest/vmstorage_legacy.go

@@ -0,0 +1,16 @@
+package apptest
+
+import (
+	"io"
+	"os"
+)
+
+// StartVmstorage_v1_132_0 starts vmstorage-v1.132.0 (the last version that uses
+// legacy index).
+//
+// The path to the binary must be provided via VMSTORAGE_V1_132_0_PATH
+// environment variable.
+func StartVmstorage_v1_132_0(instance string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
+	binary := os.Getenv("VMSTORAGE_V1_132_0_PATH")
+	return startVmstorage(instance, binary, flags, cli, output)
+}

dashboards/vm/vmalert.json

@@ -2804,10 +2804,10 @@
             "overrides": []
           },
           "gridPos": {
-            "h": 8,
+            "h": 7,
             "w": 12,
             "x": 0,
-            "y": 352
+            "y": 11
           },
           "id": 63,
           "options": {
@@ -2843,7 +2843,113 @@
           ],
           "title": "Restarts ($job)",
           "type": "timeseries"
-        }
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Group iteration reset can be caused by irregular delays during evaluation or by the system wall clock being moved backward.\nIf it is caused by host clock changes, vmalert could generate duplicate results for the group rules, since some evaluations could be repeated.\nCheck the host clock time synchronization configuration if this happens frequently.\n",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "bars",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 11
+          },
+          "id": 70,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(increase(vmalert_iteration_reset_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
+              "interval": "1m",
+              "legendFormat": "({{job}}) {{group}}({{file}})",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Group Iteration Reset ($instance)",
+          "type": "timeseries"
+        }        
       ],
       "title": "Troubleshooting",
       "type": "row"

dashboards/vmalert.json

@@ -2803,10 +2803,10 @@
             "overrides": []
           },
           "gridPos": {
-            "h": 8,
+            "h": 7,
             "w": 12,
             "x": 0,
-            "y": 352
+            "y": 11
           },
           "id": 63,
           "options": {
@@ -2842,7 +2842,113 @@
           ],
           "title": "Restarts ($job)",
           "type": "timeseries"
-        }
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Group iteration reset can be caused by irregular delays during evaluation or by the system wall clock being moved backward.\nIf it is caused by host clock changes, vmalert could generate duplicate results for the group rules, since some evaluations could be repeated.\nCheck the host clock time synchronization configuration if this happens frequently.\n",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "bars",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 11
+          },
+          "id": 70,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(increase(vmalert_iteration_reset_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
+              "interval": "1m",
+              "legendFormat": "({{job}}) {{group}}({{file}})",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Group Iteration Reset ($instance)",
+          "type": "timeseries"
+        }        
       ],
       "title": "Troubleshooting",
       "type": "row"

deployment/docker/Makefile

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.23.4
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.23.4
 
-GO_BUILDER_IMAGE := golang:1.26.3
+GO_BUILDER_IMAGE := golang:1.26.4
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/rules/alerts-vmalert.yml

@@ -64,6 +64,18 @@ groups:
             group \"{{ $labels.group }}\". See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups. 
             If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries."
 
+      - alert: GroupIterationReset
+        expr: increase(vmalert_iteration_reset_total[5m]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Evaluation iteration for group {{ $labels.group }} in file {{ $labels.file }} is reset"
+          description: "Evaluation iteration for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\" is reset on vmalert instance {{ $labels.instance }}.
+            This can be caused by irregular delays during evaluation or by the system wall clock being moved backward. If it is caused by host clock changes, vmalert could
+            generate duplicate results for the group rules since some evaluations could be repeated. Check host clock time synchronization configurations if this happens frequently."
+
+
       - alert: RemoteWriteErrors
         expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
         for: 15m
@@ -108,4 +120,3 @@ groups:
           summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
           description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
             Check vmalert's logs for detailed error message."
-

deployment/docker/rules/alerts-vmanomaly.yml

@@ -40,7 +40,7 @@ groups:
         annotations:
           summary: "Metrics have not been seen from \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for {{ $value }} seconds"
           description: >
-           The missing metric may indicate that vmanomaly is not running or is inaccessible from vmagent or the remotewrite endpoint.
+            The missing metric may indicate that vmanomaly is not running or is inaccessible from vmagent or the remotewrite endpoint.
 
       - alert: ProcessNearFDLimits
         expr: (process_max_fds{job=~".*vmanomaly.*"} - process_open_fds{job=~".*vmanomaly.*"}) < 100

docs/ai-tools/README.md

@@ -88,6 +88,7 @@ These skills provide predefined workflows and capabilities such as:
 * Multi-signal investigations 
 * Cardinality optimization 
 * Unused metric detection
+* Stream aggregation configuration
 
 To install the available skills for AI agents, run:
 ```sh

docs/playgrounds/README.md

@@ -150,8 +150,12 @@ You can experiment with your own data during the month‑long trial without depl
 are fast-booting Linux microVMs that run on a fleet of large bare-metal servers. You can start a playground right from your browser. 
 Once up and running, accessing a playground is no different from SSH-ing into a remote server rented from your favorite VPS or Cloud provider.
 
-Iximiuz Labs provides playgrounds for VictoriaMetrics software:
-- [VictoriaMetrics single node (on Ubuntu)](https://labs.iximiuz.com/playgrounds/victoriametrics-e2f9b613)
-- [VictoriaMetrics cluster (on Ubuntu)](https://labs.iximiuz.com/playgrounds/victoriametrics-cluster-8eacb19d)
-- [Getting Started with VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/tutorials/victoriametrics-getting-started-kubernetes-0e9c0993)
-- [VictoriaMetrics Operator](https://labs.iximiuz.com/playgrounds/victoriametrics-kubernetes-9eebc258)
+Iximiuz Labs provides various [learning-by-doing resources for VictoriaMetrics](https://labs.iximiuz.com/v/victoriametrics):
+- Tutorial:
+  - [Getting Started with VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/tutorials/victoriametrics-getting-started-kubernetes)
+- Playgrounds:
+  - [VictoriaMetrics single node](https://labs.iximiuz.com/playgrounds/victoriametrics)
+  - [VictoriaMetrics cluster](https://labs.iximiuz.com/playgrounds/victoriametrics-cluster)
+  - [VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/playgrounds/victoriametrics-kubernetes)
+
+Iximiuz Labs requires a [free account](https://labs.iximiuz.com/signup) to access the materials.

docs/victoriametrics/Quick-Start.md

@@ -43,6 +43,9 @@ Just download VictoriaMetrics and follow [these instructions](https://docs.victo
 See [available integrations](https://docs.victoriametrics.com/victoriametrics/integrations/) with other systems like
 [Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/) or [Grafana](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/).
 
+> Want to see VictoriaMetrics in action, but without installing anything?
+> Try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) - a list of publicly available playgrounds for VictoriaMetrics software.
+
 VictoriaMetrics is developed at a fast pace, so it is recommended to periodically check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/)
 and perform [regular upgrades](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
 

docs/victoriametrics/README.md

@@ -1136,6 +1136,8 @@ By default, the last point on the interval `[now - max_lookback ... now]` is scr
 For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
 with scrape intervals exceeding `5m`.
 
+ VictoriaMetrics supports Prometheus v3.0 utf-8 content encoding with `Accept` header. If `Accept: allow-utf-8` HTTP header provided, `/federate` API response changes according to [Prometheus utf-8](https://prometheus.io/docs/guides/utf8/#querying) specification - `metric_name{tag="value"}` transforms into `{"metric_name","tag"="value"}`.
+
 ## Capacity planning
 
 VictoriaMetrics uses lower amounts of CPU, RAM and storage space on production workloads compared to competing solutions (Prometheus, Thanos, Cortex, TimescaleDB, InfluxDB, QuestDB, M3DB) according to [our case studies](https://docs.victoriametrics.com/victoriametrics/casestudies/).

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,7 +26,22 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735-comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
+* SECURITY: upgrade Go builder from Go1.26.3 to Go1.26.4. See [the list of issues addressed in Go1.26.4](https://github.com/golang/go/issues?q=milestone%3AGo1.26.4%20label%3ACherryPickApproved).
+
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support `match[]=<label_selector>` query parameters in `/api/v1/rules` and `/api/v1/alerts` APIs to return only the rules that have configured labels satisfying the provided label selectors. See [11020](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11020).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-opentelemetry.promoteAllResourceAttributes` and `-opentelemetry.promoteScopeMetadata` command-line flags to allow managing label promotion for resource attributes and OTel scope metadata. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#10931](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10931).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) : introduce `vmagent_remotewrite_kafka_outbuf_latency_seconds` and  `vmagent_remotewrite_kafka_rtt_seconds` metrics for [kafka integration](https://docs.victoriametrics.com/victoriametrics/integrations/kafka/). The metrics could help identify throughput bottlenecks. See [#10730](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10730).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly log user information when a missing route error occurs. See [#11052](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11052).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl/): add the ability to migrate data from Mimir object storage to VictoriaMetrics. See [#7717](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7717). 
+
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix the `Notifiers` page in web UI appearing blank despite the API returning notifier data correctly. See [#11035](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11035).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): reset the group evaluation timestamp if it exceeds the current host time. Previously, vmalert could use future timestamps for evaluations if the system clock was shifted backward. See [#10985](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10985).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly parse [Prometheus Native Histograms](https://prometheus.io/docs/specs/native_histograms/), previously Protobuf parser could produce unexpected `vmrange` labels. See [#11041](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11041).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly calculate number of loaded users to be printed in startup log. Previously, it was only accounting for static users and skipped JWT configuration entries.
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): `integrate()` no longer extrapolates the last sample's value past the end of the time series. Previously, querying `integrate(metric[1h])` at a timestamp where the series had already ended would keep accruing area as if the last value continued indefinitely, producing values much larger than the true integral. See [#9474](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474). Thanks to @wtfashwin for contribution.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid returning HTTP 503 for queries with partial results when a storage group is unavailable and `-search.denyPartialResponse` is disabled.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `utf-8` label names for [/federate](https://docs.victoriametrics.com/victoriametrics/#federation) API requests. See [#10968](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10968).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the `Disable deduplication` toggle under its own local storage key. Before this fix, the toggle state was lost after reload and could overwrite the `Compact view` table setting. See [#11004](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11004). Thanks to @immanuwell for the contribution.
 
 ## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
 

docs/victoriametrics/changelog/CHANGELOG_2024.md

@@ -173,7 +173,7 @@ Released at 2024-10-02
 
 It is recommended upgrading to [v1.107.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11070) because [v1.104.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11040) contains a bug, which can lead to runtime panic at `vmselect` component. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7549) for details.
 
-**Update note 1: `*.passwordFile` and similar flags are trimming trailing whitespaces at the end of content. If authorization check performed with `*.passwordFile` content, make sure to update authorization settings to not include trailing whitespaces before the upgrade. In case of [operator](https://docs.victoriametrics.com/operator/) managed installations, make sure to update operator version to [v0.48.*](https://docs.victoriametrics.com/operator/changelog/#v0480---25-sep-2024). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6986) for the details. This change reverts behavior introduced at [v1.102.0-rc2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0-rc2) release**
+**Update note 1: `*.passwordFile` and similar flags are trimming trailing whitespaces at the end of content. If authorization check performed with `*.passwordFile` content, make sure to update authorization settings to not include trailing whitespaces before the upgrade. In case of [operator](https://docs.victoriametrics.com/operator/) managed installations, make sure to update operator version to [v0.48.*](https://docs.victoriametrics.com/operator/changelog/#v0480). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6986) for the details. This change reverts behavior introduced at [v1.102.0-rc2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0-rc2) release**
 
 * SECURITY: upgrade Go builder from Go1.23.0 to Go1.23.1. See the list of issues addressed in [Go1.23.1](https://github.com/golang/go/issues?q=milestone%3AGo1.23.1+label%3ACherryPickApproved).
 * SECURITY: upgrade base docker image (Alpine) from 3.20.2 to 3.20.3. See [alpine 3.20.3 release notes](https://alpinelinux.org/posts/Alpine-3.17.10-3.18.9-3.19.4-3.20.3-released.html).

docs/victoriametrics/integrations/grafana/_index.md

@@ -59,6 +59,10 @@ Once connected, you can build graphs and dashboards using [PromQL](https://prome
 _Creating a datasource may require [specific permissions](https://grafana.com/docs/grafana/latest/administration/data-source-management/).
 If you don't see an option to create a data source - try contacting system administrator._
 
+If you run [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) and want to see its rules in [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/),
+then set configure `-vmalert.proxyURL` on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
+or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert).
+
 ## Multi-tenant access with vmauth and OIDC
 
 [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) can proxy Grafana datasource requests and enforce

docs/victoriametrics/integrations/opentelemetry.md

@@ -28,9 +28,17 @@ The following label sanitization options can be enabled:
 
 > These flags can be applied on vmagent, vminsert or VictoriaMetrics single-node.
 
+## Instrumentation Scope
+
+By default, VictoriaMetrics promotes [OTel scope metadata](https://opentelemetry.io/docs/specs/otel/common/instrumentation-scope/) to metric labels. This behavior can be disabled via `-opentelemetry.promoteScopeMetadata`.
+
 ## Resource Attributes
 
 By default, VictoriaMetrics promotes all [OpenTelemetry resource](https://opentelemetry.io/docs/specs/otel/resource/data-model/) attributes to labels and attaches them to all ingested OTLP metrics.
+The following attribute promotion options can be configured:
+* `-opentelemetry.promoteAllResourceAttributes` - promotes all resource attributes to labels, except for the ones configured with `-opentelemetry.ignoreResourceAttributes`.
+* `-opentelemetry.promoteResourceAttributes` - promotes specific list of resource attributes to labels. It cannot be configured simultaneously with `-opentelemetry.promoteAllResourceAttributes`.
+* `-opentelemetry.ignoreResourceAttributes` - controls which resource attributes to ignore, can only be set when `-opentelemetry.promoteAllResourceAttributes` is true.
 
 ## Exponential histograms
 

docs/victoriametrics/stream-aggregation/README.md

@@ -26,6 +26,7 @@ Stream aggregation has the following features:
   and/or scraped from [Prometheus-compatible targets](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter)
 - It can filter out raw samples matched by aggregation rules, so raw data will never reach the remote destination. See `-streamAggr.keepInput` and `-streamAggr.dropInput` in [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/);
 - It allows building [flexible processing pipelines](#routing);
+- It is [horizontally scalable](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#scaling-aggregation-horizontally).
 
 # Limitations
 
@@ -598,6 +599,47 @@ Below is an example of an `aggr.yaml` configuration that drops the `replica` and
   keep_metric_names: true
 ```
 
+## Scaling aggregation horizontally
+
+Aggregation output is only correct when all contributing samples are processed by the same aggregator instance.
+
+To scale the aggregation horizontally, always shard the input samples in a deterministic way. This can be achieved by
+building a two layer topology of vmagents where the first layer is responsible for sharding, and the second layer is responsible for aggregating:
+```mermaid
+flowchart LR
+    V1[vmagent-shard-1] -- requests_total{env=test, pod=foo} --> SV1[vmagent-aggr-1]
+    V1[vmagent-shard-1] -- requests_total{env=prod, pod=bar} --> SV2[vmagent-aggr-1]
+    V2[vmagent-shard-2] -- requests_total{env=prod, pod=baz} --> SV2[vmagent-aggr-2]
+    SV1 -- requests_total:5m_without_pod_total{env=test} --> x(( ))
+    SV2 -- requests_total:5m_without_pod_total{env=prod} --> y(( ))
+style x fill:none,stroke:none
+style y fill:none,stroke:none
+```
+
+The sharding layer of vmagents can be configured via the `-remoteWrite.shardByURL.labels` or `-remoteWrite.shardByURL.ignoreLabels`
+command line flags. See how to [shard data across remote write destinations](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages) for more details.
+
+The following requirements must be met for sharded aggregation to work correctly:
+- All sharding vmagents should have the same deterministic sharding configuration.
+- The sharding configuration must align with the `by` and `without` lists:
+  - Labels listed in `by` setting should be a subset of shard's routing key `-remoteWrite.shardByURL.labels`. 
+    With `-remoteWrite.shardByURL.labels=env,job` aggregator's `by` should include `by: env`, `by: job` or both: `by: [env, job]`.
+    This makes sure that all the samples for the same `env` and `job` are aggregated together and produce the complete output.
+  - Labels listed in `without` setting should be a superset of shard's routing key `--remoteWrite.shardByURL.ignoreLabels`.
+    With `-remoteWrite.shardByURL.ignoreLabels=env,job` aggegator's `without` should include at least both labels `without: [env,job]`.
+    This makes sure that `requests_total{env=test, job=foo}` and `requests_total{env=prod, job=foo}` are routed to the same aggregator
+    and are aggregated together. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5938#issuecomment-2018470324).
+- Aggregating vmagents should not produce collisions: the aggregation output should be unique across all the sharded agents.
+  For example, `requests_total:5m_without_env_pod_total` produced by both `vmagent-aggr-1` and `vmagent-aggr-2` will collide
+  unless they have labels uniquely identifying them. These labels should be either preserved during sharding and aggregation config,
+  or enforced on the output via `-remoteWrite.label` - see [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#cluster-mode) for more details.
+
+> Never shard histograms by `le` (or `vmrange` in case of VM histograms) label. A histogram is a logical group of series differing
+only in the bucket label. All of those buckets must land on the same aggregator at the same time so it can produce a
+coherent bucket set. See more about [aggregating histograms](https://docs.victoriametrics.com/stream-aggregation/#aggregating-histograms).
+
+See also [why you shouldn't put an aggregator behind a load balancer](https://docs.victoriametrics.com/stream-aggregation/#put-aggregator-behind-load-balancer).
+
 # Troubleshooting
 
 - [Unexpected spikes for `total` or `increase` outputs](#staleness).

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -217,11 +217,19 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
   -opentelemetry.convertMetricNamesToPrometheus
      Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
+  -opentelemetry.ignoreResourceAttributes array
+     Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
   -opentelemetry.labelNameUnderscoreSanitization
      Whether to enable prepending of 'key' to labels starting with '_' when -opentelemetry.usePrometheusNaming is enabled. Reserved labels starting with '__' are not modified. See https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/ (default true)
   -opentelemetry.maxRequestSize size
      The maximum size in bytes of a single OpenTelemetry request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
+  -opentelemetry.promoteAllResourceAttributes
+     Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
+  -opentelemetry.promoteResourceAttributes array
+     Promote specific list of resource attributes to labels.
+  -opentelemetry.promoteScopeMetadata
+     Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
   -opentelemetry.usePrometheusNaming
      Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
   -opentsdbHTTPListenAddr string

docs/victoriametrics/vmagent.md

@@ -182,15 +182,15 @@ among remote storage systems specified in `-remoteWrite.url`.
 > For example, if you set `-remoteWrite.url=srv+foo` and it's resolved to three addresses (`192.168.1.1`, `192.168.1.2`, `192.168.1.3`),
 > vmagent will only choose **one** randomly every time it (re-)creates the connection. In contrast, specifying the addresses manually (`-remoteWrite.url=192.168.1.1 -remoteWrite.url=192.168.1.2 -remoteWrite.url=192.168.1.3`) will shard samples across all three URLs.
 
-Sometimes, it may be necessary to use only a particular set of labels for sharding. For example, it may be necessary to route all the metrics with the same `instance` label
-to the same `-remoteWrite.url`. In this case, you can specify a comma-separated list of these labels in the `-remoteWrite.shardByURL.labels`
-command-line flag. For example, `-remoteWrite.shardByURL.labels=instance,__name__` would shard metrics with the same name and `instance`
-label to the same `-remoteWrite.url`.
+Use `-remoteWrite.shardByURL.labels` to route metrics among `-remoteWrite.url` based on their label values. 
+For example, `-remoteWrite.shardByURL.labels=instance,__name__` would shard metrics with the same name and `instance`
+label to the same `-remoteWrite.url`. This command-line flag allows specifying a comma-separated list of labels.
 
-Sometimes, it may be necessary to ignore some labels when sharding samples across multiple `-remoteWrite.url` backends.
-For example, if all the [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) with the same set of labels
-except for the labels `instance` and `pod` must be routed to the same backend. In this case the list of ignored labels must be passed to
-`-remoteWrite.shardByURL.ignoreLabels` command-line flag: `-remoteWrite.shardByURL.ignoreLabels=instance,pod`.
+Alternatively, you can use `-remoteWrite.shardByURL.ignoreLabels` to route metrics among `-remoteWrite.url` based on their label values, excluding the specified labels.
+For example, `-remoteWrite.shardByURL.ignoreLabels=pod` would shard metrics `metric{pod="foo"}` and `metric{pod="bar"}` to the same `-remoteWrite.url`
+by ignoring the `pod` label. This command-line flag allows specifying a comma-separated list of labels.
+
+> Command-line flags `-remoteWrite.shardByURL.labels` and `-remoteWrite.shardByURL.ignoreLabels` are mutually exclusive.
 
 See also [how to scrape a large number of targets](#scraping-big-number-of-targets).
 

docs/victoriametrics/vmagent_common_flags.md

@@ -184,11 +184,19 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
   -opentelemetry.convertMetricNamesToPrometheus
      Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
+  -opentelemetry.ignoreResourceAttributes array
+     Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
   -opentelemetry.labelNameUnderscoreSanitization
      Whether to enable prepending of 'key' to labels starting with '_' when -opentelemetry.usePrometheusNaming is enabled. Reserved labels starting with '__' are not modified. See https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/ (default true)
   -opentelemetry.maxRequestSize size
      The maximum size in bytes of a single OpenTelemetry request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
+  -opentelemetry.promoteAllResourceAttributes
+     Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
+  -opentelemetry.promoteResourceAttributes array
+     Promote specific list of resource attributes to labels.
+  -opentelemetry.promoteScopeMetadata
+     Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
   -opentelemetry.usePrometheusNaming
      Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
   -opentsdbHTTPListenAddr string
@@ -494,7 +502,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Supports array of values separated by comma or specified via multiple flags.
      Empty values are set to default value.
   -remoteWrite.roundDigits array
-     Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics (default 100)
+     Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.significantFigures (default 100)
      Supports array of values separated by comma or specified via multiple flags.
      Empty values are set to default value.
   -remoteWrite.sendTimeout array