apptest: refactor apps (#11026)

- Separate process creation and type instantiation
- Get the binary path either from an env var or use default path
- move apps that use legacy indexdb to a separate *_legacy.go files

Related to https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926

---------

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
Signed-off-by: Artem Fetishev <149964189+rtm0@users.noreply.github.com>
Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>

.github/workflows/build.yml

@@ -59,11 +59,11 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/changelog-linter.yml

@@ -9,7 +9,7 @@ jobs:
   tip-lint:
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: 'actions/checkout@v6'
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           fetch-depth: 0  # we need full history for commit verification
 

.github/workflows/check-licenses.yml

@@ -15,11 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           go-version-file: 'go.mod'
           cache: false
@@ -27,7 +27,7 @@ jobs:
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-go.yml

@@ -29,18 +29,18 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Set up Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache: false
           go-version-file: 'go.mod'
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/go-build
@@ -50,14 +50,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4.35.2
+        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4.35.2
+        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4.35.2
+        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -16,19 +16,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: __vm-docs
 
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v7
+        uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd  # v7.0.0
         id: import-gpg
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}

.github/workflows/test.yml

@@ -32,11 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum
@@ -47,7 +47,7 @@ jobs:
       - run: go version
 
       - name: Cache golangci-lint
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/golangci-lint
@@ -72,11 +72,11 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum
@@ -94,11 +94,11 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/vmui.yml

@@ -32,11 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Cache node_modules
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: app/vmui/packages/vmui/node_modules
           key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
@@ -69,7 +69,7 @@ jobs:
           VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@v3
+        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           report-json: app/vmui/packages/vmui/vmui-lint-report.json

Makefile

@@ -293,8 +293,8 @@ apptest-legacy: vminsert-race vmselect-race vmstorage-race vmbackup-race vmresto
 		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
 		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
 	); \
-	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
-	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
+	VMSINGLE_V1_132_0_PATH=$${DIR}/victoria-metrics-prod \
+	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacyCluster.*"
 
 benchmark:
@@ -343,6 +343,15 @@ remove-golangci-lint:
 govulncheck: install-govulncheck
 	govulncheck ./...
 
+govulncheck-docker:
+	docker run -w $(PWD) -v $(PWD):$(PWD) \
+		-v govulncheck-gomod-cache:/root/go/pkg/mod \
+		-v govulncheck-gobuild-cache:/root/.cache/go-build \
+		-v govulncheck-go-bin:/root/go/bin \
+		--env="GOCACHE=/root/.cache/go-build" \
+		--env="GOMODCACHE=/root/go/pkg/mod" \
+		"$(GO_BUILDER_IMAGE)" /bin/sh -c "which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./..."
+
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 

app/vmagent/main.go

@@ -118,6 +118,7 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	opentelemetry.Init()
 	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 
 	if promscrape.IsDryRun() {

app/vmagent/opentelemetry/request_handler.go

@@ -25,6 +25,11 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
+// Init must be called after flag.Parse and before using the opentelemetry package.
+func Init() {
+	stream.InitDecodeOptions()
+}
+
 // InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {

app/vmagent/remotewrite/client.go

@@ -311,11 +311,6 @@ func (c *client) runWorker() {
 		if !ok {
 			return
 		}
-		if len(block) == 0 {
-			// skip empty data blocks from sending
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241
-			continue
-		}
 		go func() {
 			startTime := time.Now()
 			ch <- c.sendBlock(block)
@@ -530,10 +525,6 @@ func (c *client) drainInMemoryQueue(stopCtx context.Context, block []byte) {
 			// In this case it is guaranteed that fq will be empty
 			return
 		}
-		if len(block) == 0 {
-			// skip empty data blocks from sending
-			continue
-		}
 
 		// at this stage c.stopCh should be closed
 		// so sendBlock function should not perform retries

app/vmalert/config/config.go

@@ -113,15 +113,15 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
 			if err := exprValidator(r.Expr); err != nil {
-				return fmt.Errorf("invalid expression for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid expression for rule %q: %w", ruleName, err)
 			}
 		}
 		if validateTplFn != nil {
 			if err := validateTplFn(r.Annotations); err != nil {
-				return fmt.Errorf("invalid annotations for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid annotations for rule %q: %w", ruleName, err)
 			}
 			if err := validateTplFn(r.Labels); err != nil {
-				return fmt.Errorf("invalid labels for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid labels for rule %q: %w", ruleName, err)
 			}
 		}
 	}

app/vmalert/config/config_test.go

@@ -121,7 +121,7 @@ func TestParse_Failure(t *testing.T) {
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
 	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
+	f([]string{"testdata/rules/rules1-bad.rules"}, "bad GraphiteQL expr")
 	f([]string{"testdata/rules/vlog-rules0-bad.rules"}, "bad LogsQL expr")
 	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
 	f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields")
@@ -283,7 +283,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "up | 0",
 			},
 		},
-	}, true, "bad prometheus expr")
+	}, true, "bad MetricsQL expr")
 
 	f(&Group{
 		Name: "test graphite expr",
@@ -293,7 +293,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				"description": "some-description",
 			}},
 		},
-	}, true, "bad graphite expr")
+	}, true, "bad GraphiteQL expr")
 
 	f(&Group{
 		Name: "test vlogs expr",
@@ -327,7 +327,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "sum(up == 0 ) by (host)",
 			},
 		},
-	}, true, "bad graphite expr")
+	}, true, "bad GraphiteQL expr")
 
 	f(&Group{
 		Name: "test vlogs with prometheus exp",
@@ -351,7 +351,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				For:    promutil.NewDuration(10 * time.Millisecond),
 			},
 		},
-	}, true, "bad prometheus expr")
+	}, true, "bad MetricsQL expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/config/types.go

@@ -66,11 +66,11 @@ func (t *Type) ValidateExpr(expr string) error {
 	switch t.String() {
 	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
-			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad GraphiteQL expr: %q, err: %w", expr, err)
 		}
 	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
-			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad MetricsQL expr: %q, err: %w", expr, err)
 		}
 	case "vlogs":
 		q, err := logstorage.ParseStatsQuery(expr, 0)

app/vmauth/example_config.yml

@@ -130,6 +130,16 @@ users:
   - "http://vmselect1:8481/select/{{.MetricsTenant}}/prometheus"
   - "http://vmselect2:8481/select/{{.MetricsTenant}}/prometheus"
 
+# JWT-based routing using header-based tenant identification (VictoriaMetrics cluster)
+# The AccountID and ProjectID from JWT vm_access claims are injected as HTTP headers.
+- name: jwt-header-tenant
+  jwt:
+    skip_verify: true
+  headers:
+  - "AccountID: {{.MetricsAccountID}}"
+  - "ProjectID: {{.MetricsProjectID}}"
+  url_prefix: "http://vminsert:8480/insert/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -17,6 +17,8 @@ import (
 
 const (
 	metricsTenantPlaceholder       = `{{.MetricsTenant}}`
+	metricsAccountIDPlaceholder    = `{{.MetricsAccountID}}`
+	metricsProjectIDPlaceholder    = `{{.MetricsProjectID}}`
 	metricsExtraLabelsPlaceholder  = `{{.MetricsExtraLabels}}`
 	metricsExtraFiltersPlaceholder = `{{.MetricsExtraFilters}}`
 
@@ -30,6 +32,8 @@ const (
 
 var allPlaceholders = []string{
 	metricsTenantPlaceholder,
+	metricsAccountIDPlaceholder,
+	metricsProjectIDPlaceholder,
 	metricsExtraLabelsPlaceholder,
 	metricsExtraFiltersPlaceholder,
 	logsAccountIDPlaceholder,
@@ -40,6 +44,8 @@ var allPlaceholders = []string{
 
 var urlPathPlaceHolders = []string{
 	metricsTenantPlaceholder,
+	metricsAccountIDPlaceholder,
+	metricsProjectIDPlaceholder,
 	logsAccountIDPlaceholder,
 	logsProjectIDPlaceholder,
 }
@@ -371,6 +377,8 @@ func jwtClaimsData(vma *jwt.VMAccessClaim) map[string][]string {
 	data := map[string][]string{
 		// TODO: optimize at parsing stage
 		metricsTenantPlaceholder:       {fmt.Sprintf("%d:%d", vma.MetricsAccountID, vma.MetricsProjectID)},
+		metricsAccountIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsAccountID)},
+		metricsProjectIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsProjectID)},
 		metricsExtraLabelsPlaceholder:  vma.MetricsExtraLabels,
 		metricsExtraFiltersPlaceholder: vma.MetricsExtraFilters,
 

app/vmauth/jwt_test.go

@@ -170,13 +170,13 @@ users:
   url_prefix: http://foo.bar
 `, "cannot parse public key from file \""+publicKeyFile+"\": failed to parse key \"invalidPEM\": failed to decode PEM block containing public key")
 
-	// unsupported placeholder in a header
+	// unsupported placeholder in a URL path
 	f(`
 users:
 - jwt: 
     skip_verify: true
   url_prefix: http://foo.bar/{{.UnsupportedPlaceholder}}/foo`,
-		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 	// unsupported placeholder in a header
 	f(`
@@ -187,7 +187,7 @@ users:
     - "AccountID: {{.UnsupportedPlaceholder}}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// spaces in templating not allowed
@@ -199,7 +199,19 @@ users:
     - "AccountID: {{ .LogsAccountID }}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+	)
+
+	// placeholder must match the entire header value
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+  headers:
+    - "AccountID: foo {{.MetricsAccountID}}"
+  url_prefix: http://foo.bar
+`,
+		"request header: \"AccountID\" has unsupported placeholder: \"foo {{.MetricsAccountID}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// oidc is not an object
@@ -364,10 +376,25 @@ users:
   url_prefix: http://foo.bar
 `, validRSAPublicKey, validECDSAPublicKey))
 
+	// metrics header placeholders
 	f(`
 users:
 - jwt:
     skip_verify: true
+  headers:
+  - "MetricsAccountID: {{.MetricsAccountID}}"
+  - "MetricsProjectID: {{.MetricsProjectID}}"
+  url_prefix: http://foo.bar
+`)
+
+	// logs header placeholders
+	f(`
+users:
+- jwt:
+    skip_verify: true
+  headers:
+  - "LogsAccountID: {{.LogsAccountID}}"
+  - "LogsProjectID: {{.LogsProjectID}}"
   url_prefix: http://foo.bar
 `)
 

app/vmauth/main_test.go

@@ -851,6 +851,30 @@ users:
 		responseExpected,
 	)
 
+	// test header injection and URL templating with individual placeholders
+	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
+	request.Header.Set(`Authorization`, `Bearer `+fullToken)
+	responseExpected = `
+statusCode=200
+path: /select/123/234/api/v1/query
+query:
+headers:
+    AccountID=123
+    ProjectID=234`
+	f(fmt.Sprintf(
+		`
+users:
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: {BACKEND}/select/{{.MetricsAccountID}}/{{.MetricsProjectID}}
+  headers:
+  - "AccountID: {{.MetricsAccountID}}"
+  - "ProjectID: {{.MetricsProjectID}}"`, string(publicKeyPEM)),
+		request,
+		responseExpected,
+	)
+
 	// extra_label and extra_filters from vm_access claim merged with statically defined
 	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
 	request.Header.Set(`Authorization`, `Bearer `+fullToken)

app/vmctl/vmctlutil/time_test.go

@@ -20,6 +20,9 @@ func TestGetTime_Failure(t *testing.T) {
 
 	// negative time
 	f("-292273086-05-16T16:47:06Z")
+
+	// relative duration that resolves to a timestamp before 1970
+	f("-9223372036.855")
 }
 
 func TestGetTime_Success(t *testing.T) {
@@ -77,9 +80,6 @@ func TestGetTime_Success(t *testing.T) {
 	// float timestamp representation",
 	f("1562529662.324", time.Date(2019, 7, 7, 20, 01, 02, 324e6, time.UTC))
 
-	// negative timestamp
-	f("-9223372036.855", time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC))
-
 	// big timestamp
 	f("1223372036855", time.Date(2008, 10, 7, 9, 33, 56, 855e6, time.UTC))
 

app/vminsert/main.go

@@ -125,6 +125,7 @@ func main() {
 
 	relabel.Init()
 	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
+	opentelemetry.Init()
 	protoparserutil.StartUnmarshalWorkers()
 	if len(*clusternativeListenAddr) > 0 {
 		s, err := clusternative.NewVMinsertServer(*clusternativeListenAddr, nil)

app/vminsert/netstorage/netstorage.go

@@ -777,20 +777,22 @@ func allowRerouting(snSource *storageNode, sns []*storageNode) bool {
 
 	// Calculate median saturation
 	sort.Float64s(saturations)
-	var medianSaturation float64
-	n := len(saturations)
-	if n%2 == 0 {
-		medianSaturation = (saturations[n/2-1] + saturations[n/2]) / 2
-	} else {
-		medianSaturation = saturations[n/2]
-	}
+
+	// Try to get p90 saturation. If there's less than 10 nodes, then pick the second-slowest one.
+	p90SaturationIdx := len(saturations) - 1 - max(1, len(saturations)/10)
 
 	// Do not allow rerouting if the cluster is significantly overloaded.
-	if medianSaturation > 0.80 {
+	if saturations[p90SaturationIdx] > 0.6 {
 		return false
 	}
 
-	reroutingLogger.Warnf("reroute metrics from the slowest storage %q with saturation %.2f, where cluster median saturation is %.2f", snSource.dialer.Addr(), snSourceSaturation, medianSaturation)
+	// Perform rerouting only if snSource is slower than the p90 saturation by 20%.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10876
+	if snSourceSaturation < 1.2*saturations[p90SaturationIdx] {
+		return false
+	}
+
+	reroutingLogger.Warnf("reroute metrics from the slowest storage %q with saturation %.2f, where cluster p90 saturation is %.2f", snSource.dialer.Addr(), snSourceSaturation, saturations[p90SaturationIdx])
 	return true
 }
 

app/vminsert/netstorage/netstorage_test.go

@@ -66,55 +66,96 @@ func TestAllowRerouting(t *testing.T) {
 		}
 	}
 
-	// rerouting is triggered on the slowest node if cluster median saturation less than or equal 0.8
+	// rerouting triggered on the slowest node
 	f([]*storageNode{
-		newStorage(0.81, true),
-		newStorage(0.79, true),
-		newStorage(0.1, true),
+		newStorage(0.49*1.2, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
 	}, 0, true)
 
-	// four nodes median test
+	// 20 vmstorage nodes, we allow to perform rerouting from the slowest nodes if its saturation is higher than the p90 node than 20%.
 	f([]*storageNode{
-		newStorage(0.82, true),
-		newStorage(0.81, true),
-		newStorage(0.79, true),
-		newStorage(0.1, true),
+		newStorage(0.49*1.21, true),
+		newStorage(0.49*1.19, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
 	}, 0, true)
 
+	// 20 vmstorage nodes, we don't allow to perform rerouting from the slowest nodes if its saturation isn't higher than the p90 node than 20%.
+	f([]*storageNode{
+		newStorage(0.49*1.2, true),
+		newStorage(0.49*1.1, true),
+		newStorage(0.49*1.1, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+	}, 0, false)
+
+	// rerouting not triggered on the slowest node because the cluster is significantly overloaded.
+	f([]*storageNode{
+		newStorage(0.81*1.2, true),
+		newStorage(0.81, true),
+		newStorage(0.81, true),
+		newStorage(0.68, true),
+		newStorage(0.68, true),
+	}, 0, false)
+
+	// rerouting not triggered on the slowest node because the slowest node is not significantly slower than others
+	f([]*storageNode{
+		newStorage(0.49*1.1, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
+	}, 0, false)
+
 	// rerouting not triggered because rerouting disabled by flag
 	*disableRerouting = true
 	f([]*storageNode{
-		newStorage(0.81, true),
-		newStorage(0.79, true),
-		newStorage(0.79, true),
+		newStorage(0.49*1.2, true),
+		newStorage(0.49, true),
+		newStorage(0.49, true),
 	}, 0, false)
 	*disableRerouting = false
 
-	// rerouting not triggered because cluster median saturation more than 0.8
-	f([]*storageNode{
-		newStorage(0.81, true),
-		newStorage(0.801, true),
-		newStorage(0.1, true),
-	}, 0, false)
-
-	// four nodes median test
-	f([]*storageNode{
-		newStorage(0.82, true),
-		newStorage(0.82, true),
-		newStorage(0.79, true),
-		newStorage(0.1, true),
-	}, 0, false)
-
 	// rerouting not triggered because snSource not the slowest
 	f([]*storageNode{
-		newStorage(0.81, true),
-		newStorage(0.801, true),
+		newStorage(0.61, true),
+		newStorage(0.501, true),
 		newStorage(0.1, true),
 	}, 2, false)
 
 	// rerouting not triggered if not enough nodes
 	f([]*storageNode{
-		newStorage(0.81, true),
+		newStorage(0.49, true),
 		newStorage(0.01, true),
 	}, 0, false)
 

app/vminsert/opentelemetry/request_handler.go

@@ -24,6 +24,11 @@ var (
 	metadataInserted   = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="opentelemetry"}`)
 )
 
+// Init must be called after flag.Parse and before using the opentelemetry package.
+func Init() {
+	stream.InitDecodeOptions()
+}
+
 // InsertHandler processes opentelemetry metrics.
 func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := protoparserutil.GetExtraLabels(req)

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-C7gvW_Zn.js"></script>
+  <script type="module" crossorigin src="./assets/index-BjJ7fDL7.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-D2OEy8Ra.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BL7jEFBa.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -33,6 +33,7 @@ import (
 )
 
 var (
+	storageDataPath = flag.String("storageDataPath", "vmstorage-data", "Path to storage data")
 	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
@@ -41,12 +42,11 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	storageDataPath   = flag.String("storageDataPath", "vmstorage-data", "Path to storage data")
 	vminsertAddr      = flag.String("vminsertAddr", ":8400", "TCP address to accept connections from vminsert services")
 	vmselectAddr      = flag.String("vmselectAddr", ":8401", "TCP address to accept connections from vmselect services")
-	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages")
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages")
-	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages")
+	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
+	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
+	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
 	snapshotsMaxAge   = flagutil.NewRetentionDuration("snapshotsMaxAge", "3d", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
 	_                 = flag.Duration("snapshotCreateTimeout", 0, "Deprecated: this flag does nothing")
 
@@ -66,7 +66,10 @@ var (
 
 	logNewSeries = flag.Bool("logNewSeries", false, "Whether to log new series. This option is for debug purposes only. It can lead to performance issues "+
 		"when big number of new series are ingested into VictoriaMetrics")
-	maxHourlySeries = flag.Int("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
+	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod and -futureRetention. "+
+		"When set, then /api/v1/query_range will return an error for queries with 'from' value outside -retentionPeriod or 'to' value beyond -futureRetention. "+
+		"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee")
+	maxHourlySeries = flag.Int64("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
 		fmt.Sprintf("Setting this flag to '-1' sets limit to maximum possible value (%d) which is useful in order to enable series tracking without enforcing limits. ", math.MaxInt32)+
 		"See also -storage.maxDailySeries")
@@ -164,14 +167,15 @@ func main() {
 	logger.Infof("opening storage at %q with -retentionPeriod=%s", *storageDataPath, retentionPeriod)
 	startTime := time.Now()
 	opts := storage.OpenOptions{
-		Retention:             retentionPeriod.Duration(),
-		FutureRetention:       futureRetention.Duration(),
-		MaxHourlySeries:       getMaxHourlySeries(),
-		MaxDailySeries:        getMaxDailySeries(),
-		DisablePerDayIndex:    *disablePerDayIndex,
-		TrackMetricNamesStats: *trackMetricNamesStats,
-		IDBPrefillStart:       *idbPrefillStart,
-		LogNewSeries:          *logNewSeries,
+		Retention:                   retentionPeriod.Duration(),
+		FutureRetention:             futureRetention.Duration(),
+		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
+		MaxHourlySeries:             getMaxHourlySeries(),
+		MaxDailySeries:              getMaxDailySeries(),
+		DisablePerDayIndex:          *disablePerDayIndex,
+		TrackMetricNamesStats:       *trackMetricNamesStats,
+		IDBPrefillStart:             *idbPrefillStart,
+		LogNewSeries:                *logNewSeries,
 	}
 	strg := storage.MustOpenStorage(*storageDataPath, opts)
 	initStaleSnapshotsRemover(strg)

app/vmstorage/servers/vmselect.go

@@ -3,13 +3,10 @@ package servers
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
@@ -35,9 +32,6 @@ var (
 
 	disableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
 		"This reduces CPU usage at the cost of higher network bandwidth usage")
-	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside of the configured -retentionPeriod. "+
-		"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. "+
-		"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee")
 )
 
 var (
@@ -69,9 +63,6 @@ type vmstorageAPI struct {
 
 func (api *vmstorageAPI) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
 	tr := sq.GetTimeRange()
-	if err := checkTimeRange(api.s, tr); err != nil {
-		return nil, err
-	}
 	maxMetrics := getMaxMetrics(sq.MaxMetrics)
 	tfss, err := api.setupTfss(qt, sq, tr, maxMetrics, deadline)
 	if err != nil {
@@ -281,23 +272,6 @@ func (bi *blockIterator) Error() error {
 	return bi.sr.Error()
 }
 
-// checkTimeRange returns true if the given tr is denied for querying.
-func checkTimeRange(s *storage.Storage, tr storage.TimeRange) error {
-	if !*denyQueriesOutsideRetention {
-		return nil
-	}
-	retentionMsecs := s.RetentionMsecs()
-	minAllowedTimestamp := int64(fasttime.UnixTimestamp()*1000) - retentionMsecs
-	if tr.MinTimestamp > minAllowedTimestamp {
-		return nil
-	}
-	return &httpserver.ErrorWithStatusCode{
-		Err: fmt.Errorf("the given time range %s is outside the allowed retention %.3f days according to -denyQueriesOutsideRetention",
-			&tr, float64(retentionMsecs)/(24*3600*1000)),
-		StatusCode: http.StatusServiceUnavailable,
-	}
-}
-
 func getMaxMetrics(searchQueryLimit int) int {
 	if searchQueryLimit <= 0 {
 		return GetMaxUniqueTimeSeries()

app/vmui/packages/vmui/src/api/types.ts

@@ -3,6 +3,7 @@ export interface MetricBase {
   metric: {
     [key: string]: string;
   };
+  nullTimestamps?: number[];
 }
 
 export interface MetricResult extends MetricBase {

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -16,6 +16,7 @@ export interface ChartTooltipProps {
   point: { top: number, left: number };
   unit?: string;
   statsFormatted?: SeriesItemStatsFormatted;
+  description?: ReactNode;
   isSticky?: boolean;
   info?: ReactNode;
   marker?: string;
@@ -34,6 +35,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
   unit = "",
   info,
   statsFormatted,
+  description,
   isSticky,
   marker,
   duplicateCount = 0,
@@ -173,6 +175,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
           ))}
         </table>
       )}
+      {description && <p className="vm-chart-tooltip__description">{description}</p>}
       {info && <p className="vm-chart-tooltip__info">{info}</p>}
     </div>
   ), u.root);

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -143,4 +143,10 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     word-break: break-all;
     white-space: pre-wrap;
   }
+
+  &__description {
+    word-break: break-word;
+    white-space: pre-wrap;
+    opacity: 0.85;
+  }
 }

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -4,7 +4,7 @@ import { ChartTooltipProps } from "../../components/Chart/ChartTooltip/ChartTool
 import { SeriesItem } from "../../types";
 import dayjs from "dayjs";
 import { DATE_FULL_TIMEZONE_FORMAT } from "../../constants/date";
-import { formatPrettyNumber, getMetricName } from "../../utils/uplot";
+import { getMetricName } from "../../utils/uplot";
 import { MetricResult } from "../../api/types";
 import useEventListener from "../useEventListener";
 
@@ -15,19 +15,65 @@ interface LineTooltipHook {
   unit?: string;
 }
 
+// Pixel proximity for detecting hover over null-timestamp X markers drawn at chart bottom.
+const NULL_HOVER_PROX = 8;
+// Half the visual marker height in CSS px (BASE_POINT_SIZE * 1.4 / 2 from scatter.ts).
+// scatter.ts lifts the marker center by this amount above yMin so the icon sits inside
+// the plot area; the hover y-anchor must match that offset.
+const NULL_MARKER_HALF_CSS = 2.8;
+
+interface NullHover {
+  seriesIdx: number;
+  timestamp: number;
+}
+
+const findNullHover = (u: uPlot): NullHover | null => {
+  const cursorLeft = u.cursor.left ?? -1;
+  const cursorTop = u.cursor.top ?? -1;
+  if (cursorLeft < 0 || cursorTop < 0) return null;
+
+  const scaleY = u.scales["1"];
+  if (!scaleY || scaleY.min == null) return null;
+  const yPos = u.valToPos(scaleY.min, "1") - NULL_MARKER_HALF_CSS;
+  if (Math.abs(cursorTop - yPos) > NULL_HOVER_PROX) return null;
+
+  let best: { seriesIdx: number; timestamp: number; dist: number } | null = null;
+  for (let s = 1; s < u.series.length; s++) {
+    const seriesItem = u.series[s] as SeriesItem;
+    if (!seriesItem.show) continue;
+    const nullTs = seriesItem.nullTimestamps;
+    if (!nullTs || !nullTs.length) continue;
+
+    for (let i = 0; i < nullTs.length; i++) {
+      const t = nullTs[i];
+      const xPos = u.valToPos(t, "x");
+      const dist = Math.abs(cursorLeft - xPos);
+      if (dist < NULL_HOVER_PROX && (best === null || dist < best.dist)) {
+        best = { seriesIdx: s, timestamp: t, dist };
+      }
+    }
+  }
+  return best ? { seriesIdx: best.seriesIdx, timestamp: best.timestamp } : null;
+};
+
+const NULL_DESCRIPTION = "\"null\" can be a staleness marker or an actual NaN/null value produced by exporter.";
+
 const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   const [showTooltip, setShowTooltip] = useState(false);
   const [tooltipIdx, setTooltipIdx] = useState({ seriesIdx: -1, dataIdx: -1 });
+  const [nullTooltip, setNullTooltip] = useState<NullHover | null>(null);
   const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipProps[]>([]);
 
   const resetTooltips = () => {
     setStickyToolTips([]);
     setTooltipIdx({ seriesIdx: -1, dataIdx: -1 });
+    setNullTooltip(null);
   };
 
   const setCursor = (u: uPlot) => {
     const dataIdx = u.cursor.idx ?? -1;
     setTooltipIdx(prev => ({ ...prev, dataIdx }));
+    setNullTooltip(findNullHover(u));
   };
 
   const seriesFocus = (u: uPlot, sidx: (number | null)) => {
@@ -35,7 +81,36 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     setTooltipIdx(prev => ({ ...prev, seriesIdx }));
   };
 
+  const getNullTooltipProps = (hit: NullHover): ChartTooltipProps => {
+    const { seriesIdx, timestamp } = hit;
+    const metricItem = metrics[seriesIdx - 1];
+    const seriesItem = series[seriesIdx] as SeriesItem;
+
+    const groups = new Set(metrics.map(m => m.group));
+    const group = metricItem?.group || 0;
+
+    const yMin = u?.scales?.[1]?.min ?? 0;
+    const point = {
+      top: u ? u.valToPos(yMin, seriesItem?.scale || "1") - NULL_MARKER_HALF_CSS : 0,
+      left: u ? u.valToPos(timestamp, "x") : 0,
+    };
+
+    return {
+      u,
+      id: `null_${seriesIdx}_${timestamp}`,
+      title: groups.size > 1 ? `Query ${group}` : "",
+      dates: [dayjs(timestamp * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT)],
+      value: "null",
+      info: getMetricName(metricItem, seriesItem),
+      description: NULL_DESCRIPTION,
+      marker: `${seriesItem?.stroke}`,
+      point,
+    };
+  };
+
   const getTooltipProps = useCallback((): ChartTooltipProps => {
+    if (nullTooltip) return getNullTooltipProps(nullTooltip);
+
     const { seriesIdx, dataIdx } = tooltipIdx;
     const metricItem = metrics[seriesIdx - 1];
     const seriesItem = series[seriesIdx] as SeriesItem;
@@ -44,8 +119,6 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     const group = metricItem?.group || 0;
 
     const value = u?.data?.[seriesIdx]?.[dataIdx] || 0;
-    const min = u?.scales?.[1]?.min || 0;
-    const max = u?.scales?.[1]?.max || 1;
     const date = u?.data?.[0]?.[dataIdx] || 0;
 
     let duplicateCount = 1;
@@ -80,13 +153,13 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
       id: `${seriesIdx}_${dataIdx}`,
       title: groups.size > 1 ? `Query ${group}` : "",
       dates: [date ? dayjs(date * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT) : "-"],
-      value: formatPrettyNumber(value, min, max),
+      value: value.toLocaleString("en-US", { maximumFractionDigits: 20 }),
       info: getMetricName(metricItem, seriesItem),
       statsFormatted: seriesItem?.statsFormatted,
       marker: `${seriesItem?.stroke}`,
       duplicateCount,
     };
-  }, [u, tooltipIdx, metrics, series, unit]);
+  }, [u, tooltipIdx, metrics, series, unit, nullTooltip]);
 
   const handleClick = useCallback(() => {
     if (!showTooltip) return;
@@ -101,8 +174,9 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   };
 
   useEffect(() => {
-    setShowTooltip(tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1);
-  }, [tooltipIdx]);
+    const normalHit = tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1;
+    setShowTooltip(normalHit || nullTooltip !== null);
+  }, [tooltipIdx, nullTooltip]);
 
   useEventListener("click", handleClick);
 

app/vmui/packages/vmui/src/pages/CustomPanel/utils.test.ts

@@ -20,7 +20,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,value2");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123");
   });
 
   it("should return a valid CSV string for multiple metric entries with values", () => {
@@ -43,7 +43,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,value2\n456,value4");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123\n456,value4,1623949200,456");
   });
 
   it("should handle metric entries with multiple values field", () => {
@@ -58,7 +58,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123-456,values");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123-456,values,-,-");
   });
 
   it("should handle a combination of metric entries with value and values", () => {
@@ -81,6 +81,19 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,first\n456-789,second");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,first,1623945600,123\n456-789,second,-,-");
   });
+
+  it("should return value and timestamp if metric field is empty", () => {
+    const data: InstantMetricResult[] = [
+      {
+        value: [1623945600, "123"],
+        group: 0,
+        metric: {}
+      },
+    ];
+    const result = convertMetricsDataToCSV(data);
+    expect(result).toBe("__timestamp__,__value__\n1623945600,123");
+  });
+
 });

app/vmui/packages/vmui/src/pages/CustomPanel/utils.ts

@@ -3,16 +3,22 @@ import { getColumns, MetricCategory } from "../../hooks/useSortedCategories";
 import { formatValueToCSV } from "../../utils/csv";
 
 const getHeaders = (data: InstantMetricResult[]): string => {
-  return getColumns(data).map(({ key }) => key).join(",");
+  const metricHeaders = getColumns(data).map(({ key }) => key);
+  return [...metricHeaders, "__timestamp__", "__value__"].join(",");
 };
 
 const getRows = (data: InstantMetricResult[], headers: MetricCategory[]) => {
-  return data?.map(d => headers.map(c => formatValueToCSV(d.metric[c.key] || "-")).join(","));
+  return data?.map(d => {
+    const metricPart = headers.map(c => formatValueToCSV(d.metric[c.key] || "-"));
+    const timestamp = d.value ? formatValueToCSV(String(d.value[0])) : "-";
+    const value = d.value ? formatValueToCSV(d.value[1]) : "-";
+    return [...metricPart, timestamp, value].join(",");
+  });
 };
 
 export const convertMetricsDataToCSV = (data: InstantMetricResult[]): string => {
+  if (!data.length) return "";
   const headers = getHeaders(data);
-  if (!headers.length) return "";
   const rows = getRows(data, getColumns(data));
   return [headers, ...rows].join("\n");
 };

app/vmui/packages/vmui/src/pages/RawQueryPage/hooks/useFetchExport.ts

@@ -149,15 +149,21 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
             const pointsToTake = shouldDownsample ? maxPointsPerSeries : totalPoints;
             const step = shouldDownsample ? totalPoints / maxPointsPerSeries : 1;
 
-            const values: [number, number][] = Array.from({ length: pointsToTake }, (_, i) => {
+            const values: [number, number][] = new Array(pointsToTake);
+            const nullTimestamps: number[] = [];
+            for (let i = 0; i < pointsToTake; i++) {
               const idx = shouldDownsample ? Math.floor(i * step) : i;
-              return [rawTimestamps[idx] / 1000, rawValues[idx]];
-            });
+              const ts = rawTimestamps[idx] / 1000;
+              const raw = rawValues[idx];
+              if (raw === null) nullTimestamps.push(ts);
+              values[i] = [ts, raw as number];
+            }
 
             tempData.push({
               group: counter,
               metric: jsonLine.metric,
               values,
+              nullTimestamps,
             } as MetricBase);
           }
 

app/vmui/packages/vmui/src/types/uplot.ts

@@ -11,6 +11,7 @@ export interface SeriesItem extends Series {
     statsFormatted: SeriesItemStatsFormatted;
     median: number;
     hasAlias?: boolean;
+    nullTimestamps?: number[];
 }
 
 export interface HideSeriesArgs {

app/vmui/packages/vmui/src/utils/uplot/scatter.ts

@@ -103,6 +103,28 @@ export const drawPoints = (u: uPlot, seriesIdx: number) => {
     u.ctx.lineWidth = 1.4 * uPlot.pxRatio;
     u.ctx.strokeStyle = u.ctx.fillStyle;
     u.ctx.stroke(squaresPath);
+
+    const nullTs = (series as unknown as { nullTimestamps?: number[] }).nullTimestamps;
+    if (nullTs && nullTs.length) {
+      const xSize = BASE_POINT_SIZE * 1.4 * uPlot.pxRatio;
+      const xHalf = xSize / 2;
+      // Lift the marker by half its size so the entire icon sits inside the plot area
+      // (yMin maps to the plot's bottom edge, so centering on it would clip the lower half).
+      const cy = valToPosY(yMin, scaleY, yDim, yOff) - xHalf;
+      const xPath = new Path2D();
+      for (let i = 0; i < nullTs.length; i++) {
+        const t = nullTs[i];
+        if (t < xMin || t > xMax) continue;
+        const cx = valToPosX(t, scaleX, xDim, xOff);
+        xPath.moveTo(cx - xHalf, cy - xHalf);
+        xPath.lineTo(cx + xHalf, cy + xHalf);
+        xPath.moveTo(cx + xHalf, cy - xHalf);
+        xPath.lineTo(cx - xHalf, cy + xHalf);
+      }
+      u.ctx.lineWidth = 1.6 * uPlot.pxRatio;
+      u.ctx.strokeStyle = u.ctx.fillStyle;
+      u.ctx.stroke(xPath);
+    }
   };
 
   uPlot.orient(u, seriesIdx, orientCallback);

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -38,6 +38,7 @@ export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[],
       show: !includesHideSeries(label, hideSeries),
       scale: "1",
       paths: isRawQuery ? drawPoints : undefined,
+      nullTimestamps: d.nullTimestamps,
       ...getSeriesStatistics(d),
     };
   };

apptest/app.go

@@ -22,6 +22,7 @@ var (
 	vminsertAddrRE              = regexp.MustCompile(`accepting vminsert conns at (.*:\d{1,5})$`)
 	vminsertClusterNativeAddrRE = regexp.MustCompile(`started TCP clusternative server at "(.*:\d{1,5})"`)
 	vmselectAddrRE              = regexp.MustCompile(`accepting vmselect conns at (.*:\d{1,5})$`)
+	vmauthHttpListenAddrRE      = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
 )
 
 // app represents an instance of some VictoriaMetrics server (such as vmstorage,

apptest/testcase.go

@@ -88,19 +88,11 @@ func (tc *TestCase) MustStartDefaultVmsingle() *Vmsingle {
 }
 
 // MustStartVmsingle is a test helper function that starts an instance of
-// vmsingle located at ../../bin/victoria-metrics-race and fails the test if the app
-// fails to start.
+// vmsingle (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
 	tc.t.Helper()
-	return tc.MustStartVmsingleAt(instance, "../../bin/victoria-metrics-race", flags)
-}
 
-// MustStartVmsingleAt is a test helper function that starts an instance of
-// vmsingle and fails the test if the app fails to start.
-func (tc *TestCase) MustStartVmsingleAt(instance, binary string, flags []string) *Vmsingle {
-	tc.t.Helper()
-
-	app, err := StartVmsingleAt(instance, binary, flags, tc.cli, tc.output)
+	app, err := StartVmsingle(instance, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -109,19 +101,11 @@ func (tc *TestCase) MustStartVmsingleAt(instance, binary string, flags []string)
 }
 
 // MustStartVmstorage is a test helper function that starts an instance of
-// vmstorage located at ../../bin/vmstorage-race and fails the test if the app fails
-// to start.
+// vmstorage (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmstorage(instance string, flags []string) *Vmstorage {
 	tc.t.Helper()
-	return tc.MustStartVmstorageAt(instance, "../../bin/vmstorage-race", flags)
-}
 
-// MustStartVmstorageAt is a test helper function that starts an instance of
-// vmstorage and fails the test if the app fails to start.
-func (tc *TestCase) MustStartVmstorageAt(instance string, binary string, flags []string) *Vmstorage {
-	tc.t.Helper()
-
-	app, err := StartVmstorageAt(instance, binary, flags, tc.cli, tc.output)
+	app, err := StartVmstorage(instance, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -130,7 +114,7 @@ func (tc *TestCase) MustStartVmstorageAt(instance string, binary string, flags [
 }
 
 // MustStartVmselect is a test helper function that starts an instance of
-// vmselect and fails the test if the app fails to start.
+// vmselect (latest version) and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmselect(instance string, flags []string) *Vmselect {
 	tc.t.Helper()
 
@@ -290,10 +274,8 @@ func (tc *TestCase) MustStartDefaultCluster() *Vmcluster {
 // tests usually come paired with corresponding vmsingle tests.
 type ClusterOptions struct {
 	Vmstorage1Instance string
-	Vmstorage1Binary   string
 	Vmstorage1Flags    []string
 	Vmstorage2Instance string
-	Vmstorage2Binary   string
 	Vmstorage2Flags    []string
 	VminsertInstance   string
 	VminsertFlags      []string
@@ -305,15 +287,8 @@ type ClusterOptions struct {
 func (tc *TestCase) MustStartCluster(opts *ClusterOptions) *Vmcluster {
 	tc.t.Helper()
 
-	if opts.Vmstorage1Binary == "" {
-		opts.Vmstorage1Binary = "../../bin/vmstorage-race"
-	}
-	vmstorage1 := tc.MustStartVmstorageAt(opts.Vmstorage1Instance, opts.Vmstorage1Binary, opts.Vmstorage1Flags)
-
-	if opts.Vmstorage2Binary == "" {
-		opts.Vmstorage2Binary = "../../bin/vmstorage-race"
-	}
-	vmstorage2 := tc.MustStartVmstorageAt(opts.Vmstorage2Instance, opts.Vmstorage2Binary, opts.Vmstorage2Flags)
+	vmstorage1 := tc.MustStartVmstorage(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
+	vmstorage2 := tc.MustStartVmstorage(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
 
 	opts.VminsertFlags = append(opts.VminsertFlags, []string{
 		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),

apptest/testcase_legacy.go

@@ -0,0 +1,50 @@
+package apptest
+
+// MustStartVmsingle_v1_132_0 is a test helper function that starts an instance
+// of vmsingle-v1.132.0 (last version that uses legacy index) and fails the test
+// if the app fails to start.
+func (tc *TestCase) MustStartVmsingle_v1_132_0(instance string, flags []string) *Vmsingle {
+	tc.t.Helper()
+
+	app, err := StartVmsingle_v1_132_0(instance, flags, tc.cli, tc.output)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
+// MustStartVmstorage_v1_132_0 is a test helper function that starts an instance
+// of vmstorage-v1.132.0 (last version that uses legacy index)  and fails the
+// test if the app fails to start.
+func (tc *TestCase) MustStartVmstorage_v1_132_0(instance string, flags []string) *Vmstorage {
+	tc.t.Helper()
+
+	app, err := StartVmstorage_v1_132_0(instance, flags, tc.cli, tc.output)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
+// MustStartCluster_v1_132_0 starts a cluster with vmstorage-v1.132.0 with
+// custom flags.
+func (tc *TestCase) MustStartCluster_v1_132_0(opts *ClusterOptions) *Vmcluster {
+	tc.t.Helper()
+
+	vmstorage1 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
+	vmstorage2 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
+
+	opts.VminsertFlags = append(opts.VminsertFlags, []string{
+		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
+	}...)
+	vminsert := tc.MustStartVminsert(opts.VminsertInstance, opts.VminsertFlags)
+
+	opts.VmselectFlags = append(opts.VmselectFlags, []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	}...)
+	vmselect := tc.MustStartVmselect(opts.VmselectInstance, opts.VmselectFlags)
+
+	return &Vmcluster{vminsert, vmselect, []*Vmstorage{vmstorage1, vmstorage2}}
+}

apptest/tests/legacy_indexdb_test.go

@@ -2,7 +2,6 @@ package tests
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"slices"
 	"testing"
@@ -11,11 +10,6 @@ import (
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
-var (
-	legacyVmsinglePath  = os.Getenv("VM_LEGACY_VMSINGLE_PATH")
-	legacyVmstoragePath = os.Getenv("VM_LEGACY_VMSTORAGE_PATH")
-)
-
 type testLegacyDeleteSeriesOpts struct {
 	startLegacySUT func() at.PrometheusWriteQuerier
 	startNewSUT    func() at.PrometheusWriteQuerier
@@ -31,7 +25,7 @@ func TestLegacySingleDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.maxStalenessInterval=1m",
@@ -64,15 +58,13 @@ func TestLegacyClusterDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -191,7 +183,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - start legacy vmsingle
 	// - insert data1
-	// - confirm that metric names and samples are searcheable
+	// - confirm that metric names and samples are searchable
 	// - stop legacy vmsingle
 	const step = 24 * 3600 * 1000 // 24h
 	start1 := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
@@ -204,12 +196,12 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 	opts.stopLegacySUT()
 
 	// - start new vmsingle
-	// - confirm that data1 metric names and samples are searcheable
+	// - confirm that data1 metric names and samples are searchable
 	// - delete data1
-	// - confirm that data1 metric names and samples are not searcheable anymore
+	// - confirm that data1 metric names and samples are not searchable anymore
 	// - insert data2 (same metric names, different dates)
-	// - confirm that metric names become searcheable again
-	// - confirm that data1 samples are not searchable and data2 samples are searcheable
+	// - confirm that metric names become searchable again
+	// - confirm that data1 samples are not searchable and data2 samples are searchable
 
 	newSUT := opts.startNewSUT()
 	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
@@ -230,7 +222,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - restart new vmsingle
 	// - confirm that metric names still searchable, data1 samples are not
-	//   searchable, and data2 samples are searcheable
+	//   searchable, and data2 samples are searchable
 
 	opts.stopNewSUT()
 	newSUT = opts.startNewSUT()
@@ -255,7 +247,7 @@ func TestLegacySingleBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -298,15 +290,13 @@ func TestLegacyClusterBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -583,7 +573,7 @@ func TestLegacySingleDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -618,15 +608,13 @@ func TestLegacyClusterDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster(&at.ClusterOptions{
+			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
-				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
-				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",

apptest/vmagent.go

@@ -16,43 +16,63 @@ import (
 	"github.com/golang/snappy"
 )
 
-// Vmagent holds the state of a vmagent app and provides vmagent-specific functions
-type Vmagent struct {
-	*app
-	*metricsClient
-
-	httpListenAddr string
-
-	cli *Client
-}
-
-// StartVmagent starts an instance of vmagent with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
+// StartVmagent starts the latest version of vmagent.
+//
+// The path to the binary can be provided via VMAGENT_PATH environment
+// variable. If the variable is not set, ../../bin/vmagent-race will be
+// used.
 func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfigFilePath string, output io.Writer) (*Vmagent, error) {
-	extractREs := []*regexp.Regexp{
-		httpListenAddrRE,
+	binary := os.Getenv("VMAGENT_PATH")
+	if binary == "" {
+		binary = "../../bin/vmagent-race"
 	}
-
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmagent-race", flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-promscrape.config":       promScrapeConfigFilePath,
 			"-remoteWrite.tmpDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
 		},
-		extractREs: extractREs,
-		output:     output,
+		extractREs: []*regexp.Regexp{
+			httpListenAddrRE,
+		},
+		output: output,
 	})
 	if err != nil {
 		return nil, err
 	}
 
+	return newVmagent(app, cli, vmagentRuntimeValues{
+		httpListenAddr: stderrExtracts[0],
+	}), nil
+}
+
+type vmagentRuntimeValues struct {
+	httpListenAddr string
+}
+
+func newVmagent(app *app, cli *Client, rt vmagentRuntimeValues) *Vmagent {
 	return &Vmagent{
 		app:            app,
-		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
-		httpListenAddr: stderrExtracts[0],
 		cli:            cli,
-	}, nil
+		metricsClient:  newMetricsClient(cli, rt.httpListenAddr),
+		httpListenAddr: rt.httpListenAddr,
+	}
+}
+
+// Vmagent holds the state of a vmagent app and provides vmagent-specific
+// functions.
+type Vmagent struct {
+	*app
+	*metricsClient
+
+	cli            *Client
+	httpListenAddr string
+}
+
+// HTTPAddr returns the address at which the vmagent process is listening
+// for http connections.
+func (app *Vmagent) HTTPAddr() string {
+	return app.httpListenAddr
 }
 
 // APIV1ImportPrometheus is a test helper function that inserts a
@@ -203,12 +223,6 @@ func (app *Vmagent) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, o
 	})
 }
 
-// HTTPAddr returns the address at which the vmagent process is listening
-// for http connections.
-func (app *Vmagent) HTTPAddr() string {
-	return app.httpListenAddr
-}
-
 // sendBlocking sends the data to vmstorage by executing `send` function and
 // waits until the data is actually sent.
 //

apptest/vmauth.go

@@ -2,6 +2,7 @@ package apptest
 
 import (
 	"io"
+	"os"
 	"regexp"
 	"syscall"
 	"testing"
@@ -10,7 +11,48 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 )
 
-var httpBuilitinListenAddrRE = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
+// StartVmauth starts the latest version of vmauth.
+//
+// The path to the binary can be provided via VMAUTH_PATH environment
+// variable. If the variable is not set, ../../bin/vmauth-race will be
+// used.
+func StartVmauth(instance string, flags []string, cli *Client, configFilePath string, output io.Writer) (*Vmauth, error) {
+	binary := os.Getenv("VMAUTH_PATH")
+	if binary == "" {
+		binary = "../../bin/vmauth-race"
+	}
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr": "127.0.0.1:0",
+			"-auth.config":    configFilePath,
+		},
+		extractREs: []*regexp.Regexp{
+			vmauthHttpListenAddrRE,
+		},
+		output: output,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return newVmauth(app, cli, configFilePath, vmauthRuntimeValues{
+		httpListenAddr: stderrExtracts[0],
+	}), nil
+}
+
+type vmauthRuntimeValues struct {
+	httpListenAddr string
+}
+
+func newVmauth(app *app, cli *Client, configFilePath string, rt vmauthRuntimeValues) *Vmauth {
+	return &Vmauth{
+		app:            app,
+		metricsClient:  newMetricsClient(cli, rt.httpListenAddr),
+		httpListenAddr: rt.httpListenAddr,
+		configFilePath: configFilePath,
+		cli:            cli,
+	}
+}
 
 // Vmauth holds the state of a vmauth app and provides vmauth-specific
 // functions.
@@ -18,38 +60,14 @@ type Vmauth struct {
 	*app
 	*metricsClient
 
+	cli            *Client
 	httpListenAddr string
 	configFilePath string
-	cli            *Client
 }
 
-// StartVmauth starts an instance of vmauth with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
-func StartVmauth(instance string, flags []string, cli *Client, configFilePath string, output io.Writer) (*Vmauth, error) {
-	extractREs := []*regexp.Regexp{
-		httpBuilitinListenAddrRE,
-	}
-
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmauth-race", flags, &appOptions{
-		defaultFlags: map[string]string{
-			"-httpListenAddr": "127.0.0.1:0",
-			"-auth.config":    configFilePath,
-		},
-		extractREs: extractREs,
-		output:     output,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return &Vmauth{
-		app:            app,
-		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
-		httpListenAddr: stderrExtracts[0],
-		configFilePath: configFilePath,
-		cli:            cli,
-	}, nil
+// GetHTTPListenAddr returns listen http addr
+func (app *Vmauth) GetHTTPListenAddr() string {
+	return app.httpListenAddr
 }
 
 // UpdateConfiguration updates the vmauth configuration file with the provided YAML content,
@@ -79,8 +97,3 @@ func (app *Vmauth) UpdateConfiguration(t *testing.T, configFileYAML string) {
 
 	t.Fatalf("config were not reloaded after SIGHUP signal; previous total: %d, current total: %d", prevTotal, currTotal)
 }
-
-// GetHTTPListenAddr returns listen http addr
-func (app *Vmauth) GetHTTPListenAddr() string {
-	return app.httpListenAddr
-}

apptest/vmbackup.go

@@ -1,15 +1,26 @@
 package apptest
 
-import "io"
+import (
+	"io"
+	"os"
+)
 
-// StartVmbackup starts an instance of vmbackup with the given flags and waits
-// until it exits.
+// StartVmbackup starts the latest version of vmbackup with the given flags and
+// waits until it exits.
+//
+// The path to the binary can be provided via VMBACKUP_PATH environment
+// variable. If the variable is not set, ../../bin/vmbackup-race will be
+// used.
 func StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string, output io.Writer) error {
+	binary := os.Getenv("VMBACKUP_PATH")
+	if binary == "" {
+		binary = "../../bin/vmbackup-race"
+	}
 	flags := []string{
 		"-storageDataPath=" + storageDataPath,
 		"-snapshot.createURL=" + snapshotCreateURL,
 		"-dst=" + dst,
 	}
-	_, _, err := startApp(instance, "../../bin/vmbackup-race", flags, &appOptions{wait: true, output: output})
+	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vmctl.go

@@ -1,9 +1,23 @@
 package apptest
 
-import "io"
+import (
+	"io"
+	"os"
+)
 
 // StartVmctl starts an instance of vmctl cli with the given flags
+
+// StartVmctl starts the latest version of vmctl with the given flags and
+// waits until it exits.
+//
+// The path to the binary can be provided via VMCTL_PATH environment
+// variable. If the variable is not set, ../../bin/vmctl-race will be
+// used.
 func StartVmctl(instance string, flags []string, output io.Writer) error {
-	_, _, err := startApp(instance, "../../bin/vmctl-race", flags, &appOptions{wait: true, output: output})
+	binary := os.Getenv("VMCTL_PATH")
+	if binary == "" {
+		binary = "../../bin/vmctl-race"
+	}
+	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vminsert.go

@@ -3,23 +3,13 @@ package apptest
 import (
 	"fmt"
 	"io"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
 )
 
-// Vminsert holds the state of a vminsert app and provides vminsert-specific
-// functions.
-type Vminsert struct {
-	*app
-	*metricsClient
-	*vminsertClient
-
-	httpListenAddr          string
-	clusternativeListenAddr string
-}
-
 // storageNodes returns the storage node addresses passed to vminsert via
 // -storageNode command line flag.
 func storageNodes(flags []string) []string {
@@ -31,9 +21,11 @@ func storageNodes(flags []string) []string {
 	return nil
 }
 
-// StartVminsert starts an instance of vminsert with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
+// StartVminsert starts the latest version of vminsert.
+//
+// The path to the binary can be provided via VMINSERT_PATH environment
+// variable. If the variable is not set, ../../bin/vminsert-race will be
+// used.
 func StartVminsert(instance string, flags []string, cli *Client, output io.Writer) (*Vminsert, error) {
 	extractREs := []*regexp.Regexp{
 		httpListenAddrRE,
@@ -48,11 +40,15 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		extractREs = append(extractREs, regexp.MustCompile(logRecord))
 	}
 
-	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert-race", flags, &appOptions{
+	binary := os.Getenv("VMINSERT_PATH")
+	if binary == "" {
+		binary = "../../bin/vminsert-race"
+	}
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":                              "127.0.0.1:0",
 			"-clusternativeListenAddr":                     "127.0.0.1:0",
-			"-graphiteListenAddr":                          ":0",
+			"-graphiteListenAddr":                          "127.0.0.1:0",
 			"-opentsdbListenAddr":                          "127.0.0.1:0",
 			"-clusternative.vminsertConnsShutdownDuration": "1ms",
 		},
@@ -63,27 +59,56 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
-	metricsClient := newMetricsClient(cli, stderrExtracts[0])
-	return &Vminsert{
-		app:           app,
-		metricsClient: metricsClient,
-		vminsertClient: &vminsertClient{
-			vminsertCli: cli,
-			url: func(op, path string, opts QueryOpts) string {
-				return getClusterPath(stderrExtracts[0], op, path, opts)
-			},
-			openTSDBURL: func(op, path string, opts QueryOpts) string {
-				return getClusterPath(stderrExtracts[3], op, path, opts)
-			},
-			graphiteListenAddr: stderrExtracts[2],
-			sendBlocking: func(t *testing.T, numRecordsToSend int, send func()) {
-				t.Helper()
-				sendBlocking(t, metricsClient, numRecordsToSend, send)
-			},
-		},
+	return newVminsert(app, cli, vminsertRuntimeValues{
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
-	}, nil
+		graphiteListenAddr:      stderrExtracts[2],
+		openTSDBListenAddr:      stderrExtracts[3],
+	}), nil
+}
+
+type vminsertRuntimeValues struct {
+	httpListenAddr          string
+	clusternativeListenAddr string
+	graphiteListenAddr      string
+	openTSDBListenAddr      string
+}
+
+func newVminsert(app *app, cli *Client, rt vminsertRuntimeValues) *Vminsert {
+	metricsClient := newMetricsClient(cli, rt.httpListenAddr)
+	vminsertClient := &vminsertClient{
+		vminsertCli: cli,
+		url: func(op, path string, opts QueryOpts) string {
+			return getClusterPath(rt.httpListenAddr, op, path, opts)
+		},
+		openTSDBURL: func(op, path string, opts QueryOpts) string {
+			return getClusterPath(rt.openTSDBListenAddr, op, path, opts)
+		},
+		graphiteListenAddr: rt.graphiteListenAddr,
+		sendBlocking: func(t *testing.T, numRecordsToSend int, send func()) {
+			t.Helper()
+			sendBlocking(t, metricsClient, numRecordsToSend, send)
+		},
+	}
+
+	return &Vminsert{
+		app:                     app,
+		metricsClient:           metricsClient,
+		vminsertClient:          vminsertClient,
+		httpListenAddr:          rt.httpListenAddr,
+		clusternativeListenAddr: rt.clusternativeListenAddr,
+	}
+}
+
+// Vminsert holds the state of a vminsert app and provides vminsert-specific
+// functions.
+type Vminsert struct {
+	*app
+	*metricsClient
+	*vminsertClient
+
+	httpListenAddr          string
+	clusternativeListenAddr string
 }
 
 // ClusternativeListenAddr returns the address at which the vminsert process is

apptest/vmrestore.go

@@ -1,14 +1,25 @@
 package apptest
 
-import "io"
+import (
+	"io"
+	"os"
+)
 
-// StartVmrestore starts an instance of vmrestore with the given flags and waits
-// until it exits.
+// StartVmrestore starts the latest version of vmrestore with the given flags
+// and waits until it exits.
+//
+// The path to the binary can be provided via VMRESTORE_PATH environment
+// variable. If the variable is not set, ../../bin/vmrestore-race will be
+// used.
 func StartVmrestore(instance, src, storageDataPath string, output io.Writer) error {
+	binary := os.Getenv("VMRESTORE_PATH")
+	if binary == "" {
+		binary = "../../bin/vmrestore-race"
+	}
 	flags := []string{
 		"-src=" + src,
 		"-storageDataPath=" + storageDataPath,
 	}
-	_, _, err := startApp(instance, "../../bin/vmrestore-race", flags, &appOptions{wait: true, output: output})
+	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vmselect.go

@@ -3,26 +3,21 @@ package apptest
 import (
 	"fmt"
 	"io"
+	"os"
 	"regexp"
 )
 
-// Vmselect holds the state of a vmselect app and provides vmselect-specific
-// functions.
-type Vmselect struct {
-	*app
-	*metricsClient
-	*vmselectClient
-
-	httpListenAddr          string
-	clusternativeListenAddr string
-	cli                     *Client
-}
-
-// StartVmselect starts an instance of vmselect with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr)
+// StartVmselect starts the latest version of vmselect.
+//
+// The path to the binary can be provided via VMSELECT_PATH environment
+// variable. If the variable is not set, ../../bin/vmselect-race will be
+// used.
 func StartVmselect(instance string, flags []string, cli *Client, output io.Writer) (*Vmselect, error) {
-	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect-race", flags, &appOptions{
+	binary := os.Getenv("VMSELECT_PATH")
+	if binary == "" {
+		binary = "../../bin/vmselect-race"
+	}
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-clusternativeListenAddr": "127.0.0.1:0",
@@ -37,21 +32,43 @@ func StartVmselect(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
+	return newVmselect(app, cli, vmselectRuntimeValues{
+		httpListenAddr:          stderrExtracts[0],
+		clusternativeListenAddr: stderrExtracts[1],
+	}), nil
+}
+
+type vmselectRuntimeValues struct {
+	httpListenAddr          string
+	clusternativeListenAddr string
+}
+
+func newVmselect(app *app, cli *Client, rt vmselectRuntimeValues) *Vmselect {
 	return &Vmselect{
 		app:           app,
-		metricsClient: newMetricsClient(cli, stderrExtracts[0]),
+		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
 		vmselectClient: &vmselectClient{
 			vmselectCli: cli,
 			url: func(op, path string, opts QueryOpts) string {
-				return getClusterPath(stderrExtracts[0], op, path, opts)
+				return getClusterPath(rt.httpListenAddr, op, path, opts)
 			},
-			metricNamesStatsResetURL: fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[0]),
-			tenantsURL:               fmt.Sprintf("http://%s/admin/tenants", stderrExtracts[0]),
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", rt.httpListenAddr),
+			tenantsURL:               fmt.Sprintf("http://%s/admin/tenants", rt.httpListenAddr),
 		},
-		httpListenAddr:          stderrExtracts[0],
-		clusternativeListenAddr: stderrExtracts[1],
-		cli:                     cli,
-	}, nil
+		httpListenAddr:          rt.httpListenAddr,
+		clusternativeListenAddr: rt.clusternativeListenAddr,
+	}
+}
+
+// Vmselect holds the state of a vmselect app and provides vmselect-specific
+// functions.
+type Vmselect struct {
+	*app
+	*metricsClient
+	*vmselectClient
+
+	httpListenAddr          string
+	clusternativeListenAddr string
 }
 
 // ClusternativeListenAddr returns the address at which the vmselect process is

apptest/vmsingle.go

@@ -9,28 +9,21 @@ import (
 	"time"
 )
 
-// Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
-// functions.
-type Vmsingle struct {
-	*app
-	*metricsClient
-	*vmstorageClient
-	*vmselectClient
-	*vminsertClient
-
-	storageDataPath string
-	httpListenAddr  string
-}
-
-// StartVmsingleAt starts an instance of vmsingle with the given flags. It also
-// sets the default flags and populates the app instance state with runtime
-// values extracted from the application log (such as httpListenAddr).
-func StartVmsingleAt(instance, binary string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
+// StartVmsingle starts the latest version of vmsingle.
+//
+// The path to the binary can be provided via VMSINGLE_PATH environment
+// variable. If the variable is not set, ../../bin/victoria-metrics-race will be
+// used.
+func StartVmsingle(instance string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
+	binary := os.Getenv("VMSINGLE_PATH")
+	if binary == "" {
+		binary = "../../bin/victoria-metrics-race"
+	}
 	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-storageDataPath":    fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
 			"-httpListenAddr":     "127.0.0.1:0",
-			"-graphiteListenAddr": ":0",
+			"-graphiteListenAddr": "127.0.0.1:0",
 			"-opentsdbListenAddr": "127.0.0.1:0",
 		},
 		extractREs: []*regexp.Regexp{
@@ -45,38 +38,67 @@ func StartVmsingleAt(instance, binary string, flags []string, cli *Client, outpu
 		return nil, err
 	}
 
+	return newVmsingle(app, cli, vmsingleRuntimeValues{
+		storageDataPath:    stderrExtracts[0],
+		httpListenAddr:     stderrExtracts[1],
+		graphiteListenAddr: stderrExtracts[2],
+		openTSDBListenAddr: stderrExtracts[3],
+	}), nil
+}
+
+type vmsingleRuntimeValues struct {
+	storageDataPath    string
+	httpListenAddr     string
+	graphiteListenAddr string
+	openTSDBListenAddr string
+}
+
+func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
 	return &Vmsingle{
 		app:           app,
-		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
+		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
 		vmstorageClient: &vmstorageClient{
 			vmstorageCli:   cli,
-			httpListenAddr: stderrExtracts[1],
+			httpListenAddr: rt.httpListenAddr,
 		},
 		vmselectClient: &vmselectClient{
 			vmselectCli: cli,
 			url: func(op, path string, opts QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
 			},
-			metricNamesStatsResetURL: fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[1]),
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", rt.httpListenAddr),
 			tenantsURL:               "vmsingle-does-not-serve-tenants",
 		},
 		vminsertClient: &vminsertClient{
 			vminsertCli: cli,
 			url: func(_, path string, _ QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
 			},
 			openTSDBURL: func(_, path string, _ QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", stderrExtracts[3], path)
+				return fmt.Sprintf("http://%s/%s", rt.openTSDBListenAddr, path)
 			},
-			graphiteListenAddr: stderrExtracts[2],
+			graphiteListenAddr: rt.graphiteListenAddr,
 			sendBlocking: func(t *testing.T, _ int, send func()) {
 				t.Helper()
 				send()
 			},
 		},
-		storageDataPath: stderrExtracts[0],
-		httpListenAddr:  stderrExtracts[1],
-	}, nil
+		storageDataPath: rt.storageDataPath,
+		httpListenAddr:  rt.httpListenAddr,
+	}
+}
+
+// Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
+// functions.
+type Vmsingle struct {
+	*app
+	*metricsClient
+	*vmstorageClient
+	*vmselectClient
+	*vminsertClient
+
+	storageDataPath string
+	httpListenAddr  string
 }
 
 // HTTPAddr returns the address at which the vminsert process is

apptest/vmsingle_legacy.go

@@ -0,0 +1,43 @@
+package apptest
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"time"
+)
+
+// StartVmsingle_v1_132_0 starts vmsingle-v1.132.0 (the last version that uses
+// legacy index).
+//
+// The path to the binary must be provided via VMSINGLE_V1_132_0_PATH
+// environment variable.
+func StartVmsingle_v1_132_0(instance string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
+	binary := os.Getenv("VMSINGLE_V1_132_0_PATH")
+	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-storageDataPath":    fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
+			"-httpListenAddr":     "127.0.0.1:0",
+			"-graphiteListenAddr": "127.0.0.1:0",
+			"-opentsdbListenAddr": "127.0.0.1:0",
+		},
+		extractREs: []*regexp.Regexp{
+			storageDataPathRE,
+			httpListenAddrRE,
+			graphiteListenAddrRE,
+			openTSDBListenAddrRE,
+		},
+		output: output,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return newVmsingle(app, cli, vmsingleRuntimeValues{
+		storageDataPath:    stderrExtracts[0],
+		httpListenAddr:     stderrExtracts[1],
+		graphiteListenAddr: stderrExtracts[2],
+		openTSDBListenAddr: stderrExtracts[3],
+	}), nil
+}

apptest/vmstorage.go

@@ -8,23 +8,22 @@ import (
 	"time"
 )
 
-// Vmstorage holds the state of a vmstorage app and provides vmstorage-specific
-// functions.
-type Vmstorage struct {
-	*app
-	*metricsClient
-	*vmstorageClient
-
-	storageDataPath string
-	httpListenAddr  string
-	vminsertAddr    string
-	vmselectAddr    string
+// StartVmstorage starts the latest version of vmstorage.
+//
+// The path to the binary can be provided via VMSTORAGE_PATH environment
+// variable. If the variable is not set, ../../bin/vmstorage-race will be used.
+func StartVmstorage(instance string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
+	binary := os.Getenv("VMSTORAGE_PATH")
+	if binary == "" {
+		binary = "../../bin/vmstorage-race"
+	}
+	return startVmstorage(instance, binary, flags, cli, output)
 }
 
-// StartVmstorageAt starts an instance of vmstorage with the given flags. It also
+// startVmstorage starts an instance of vmstorage with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)
-func StartVmstorageAt(instance, binary string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
+func startVmstorage(instance, binary string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
 	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-storageDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
@@ -44,18 +43,47 @@ func StartVmstorageAt(instance, binary string, flags []string, cli *Client, outp
 		return nil, err
 	}
 
-	return &Vmstorage{
-		app:           app,
-		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
-		vmstorageClient: &vmstorageClient{
-			vmstorageCli:   cli,
-			httpListenAddr: stderrExtracts[1],
-		},
+	return newVmstorage(app, cli, vmstorageRuntimeValues{
 		storageDataPath: stderrExtracts[0],
 		httpListenAddr:  stderrExtracts[1],
 		vminsertAddr:    stderrExtracts[2],
 		vmselectAddr:    stderrExtracts[3],
-	}, nil
+	}), nil
+}
+
+type vmstorageRuntimeValues struct {
+	storageDataPath string
+	httpListenAddr  string
+	vminsertAddr    string
+	vmselectAddr    string
+}
+
+func newVmstorage(app *app, cli *Client, rt vmstorageRuntimeValues) *Vmstorage {
+	return &Vmstorage{
+		app:           app,
+		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
+		vmstorageClient: &vmstorageClient{
+			vmstorageCli:   cli,
+			httpListenAddr: rt.httpListenAddr,
+		},
+		storageDataPath: rt.storageDataPath,
+		httpListenAddr:  rt.httpListenAddr,
+		vminsertAddr:    rt.vminsertAddr,
+		vmselectAddr:    rt.vmselectAddr,
+	}
+}
+
+// Vmstorage holds the state of a vmstorage app and provides vmstorage-specific
+// functions.
+type Vmstorage struct {
+	*app
+	*metricsClient
+	*vmstorageClient
+
+	storageDataPath string
+	httpListenAddr  string
+	vminsertAddr    string
+	vmselectAddr    string
 }
 
 // VminsertAddr returns the address at which the vmstorage process is listening

apptest/vmstorage_legacy.go

@@ -0,0 +1,16 @@
+package apptest
+
+import (
+	"io"
+	"os"
+)
+
+// StartVmstorage_v1_132_0 starts vmstorage-v1.132.0 (the last version that uses
+// legacy index).
+//
+// The path to the binary must be provided via VMSTORAGE_V1_132_0_PATH
+// environment variable.
+func StartVmstorage_v1_132_0(instance string, flags []string, cli *Client, output io.Writer) (*Vmstorage, error) {
+	binary := os.Getenv("VMSTORAGE_V1_132_0_PATH")
+	return startVmstorage(instance, binary, flags, cli, output)
+}

dashboards/victoriametrics-cluster.json

@@ -3109,7 +3109,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/victoriametrics.json

@@ -3406,7 +3406,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/victoriametrics-cluster.json

@@ -3110,7 +3110,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/victoriametrics.json

@@ -3407,7 +3407,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/vmagent.json

@@ -2946,7 +2946,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/vmalert.json

@@ -2324,7 +2324,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vmagent.json

@@ -2945,7 +2945,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vmalert.json

@@ -2323,7 +2323,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.143.0
+    image: victoriametrics/vmagent:v1.144.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.143.0-cluster
+    image: victoriametrics/vmstorage:v1.144.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.143.0-cluster
+    image: victoriametrics/vmstorage:v1.144.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.143.0-cluster
+    image: victoriametrics/vminsert:v1.144.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.143.0-cluster
+    image: victoriametrics/vminsert:v1.144.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.143.0-cluster
+    image: victoriametrics/vmselect:v1.144.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.143.0-cluster
+    image: victoriametrics/vmselect:v1.144.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.143.0
+    image: victoriametrics/vmauth:v1.144.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.143.0
+    image: victoriametrics/vmalert:v1.144.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.143.0
+    image: victoriametrics/vmagent:v1.144.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.143.0
+    image: victoriametrics/victoria-metrics:v1.144.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.143.0
+    image: victoriametrics/vmalert:v1.144.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.143.0
+    image: victoriametrics/vmagent:v1.144.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.143.0
+    image: victoriametrics/victoria-metrics:v1.144.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.143.0
+    image: victoriametrics/vmalert:v1.144.0
     depends_on:
       - "victoriametrics"
     ports:

docs/ai-tools/README.md

@@ -88,6 +88,7 @@ These skills provide predefined workflows and capabilities such as:
 * Multi-signal investigations 
 * Cardinality optimization 
 * Unused metric detection
+* Stream aggregation configuration
 
 To install the available skills for AI agents, run:
 ```sh

docs/anomaly-detection/UI.md

@@ -652,12 +652,11 @@ vmanomaly version: [v1.29.4](https://docs.victoriametrics.com/anomaly-detection/
   - run 2-100x faster for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) depending on a configuration vs [1.6.1](#v161) timings
   - show stage-aware progress bar, e.g. "getting data", "fitting model", "inferring on chunk".
 - IMPROVEMENT: "Advanced Options" design is improved, section is collapsed by default to save vertical space, yet can be encoded as always open in UI state URL.
-- IMPROVEMENT: dropdowns in UI are now searcheable and constrained in size for better UX, especially when many options are available (e.g. models list, tenants list, etc.).
+- IMPROVEMENT: dropdowns in UI are now searchable and constrained in size for better UX, especially when many options are available (e.g. models list, tenants list, etc.).
 - BUGFIX: in generated config ("Show Config" menu)
-  - special YAML values (like `-.inf`, `.inf`, `.nan`) are now properly quoted and formatted to avoid JSON converting issues when copied to Kubernetes CRs, which previously lead to rejection of the manifest before vmanomaly can consume it.
+  - special YAML values (like `-.inf`, `.inf`, `.nan`) are now properly quoted and formatted to avoid JSON converting issues when copied to Kubernetes CRs, which previously led to rejection of the manifest before vmanomaly can consume it.
   - with Data Source selected as "Logs/Traces" `reader.class` is now correctly set to `vlogs` (previously `vm`) in generated config, when "model only" toggle is turned off.
 - BUGFIX: now "Hide Common Labels" toggle in "Table View" works correctly and does not show common labels in the legend columns when turned on.
-- BUGFIX: improved dropdowns for better accessibility to be searcheable and constrained in size.
 - BUGFIX: "Prettify query" now works correctly for "Metrics" datasources.
 
 ### v1.6.1

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -240,23 +240,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.143.0
+    image: victoriametrics/victoria-metrics:v1.144.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.143.0-cluster
+    image: victoriametrics/vmstorage:v1.144.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.143.0-cluster
+    image: victoriametrics/vminsert:v1.144.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.143.0-cluster
+    image: victoriametrics/vmselect:v1.144.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.143.0
+    image: victoriametrics/vmagent:v1.144.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -308,7 +308,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.143.0
+    image: docker.io/victoriametrics/vmauth:v1.144.0
     ports:
       - 8427:8427
     volumes:

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.143.0-cluster
+    image: victoriametrics/vmstorage:v1.144.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.143.0-cluster
+    image: victoriametrics/vminsert:v1.144.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.143.0-cluster
+    image: victoriametrics/vmselect:v1.144.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.143.0-enterprise
+    image: victoriametrics/vmauth:v1.144.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.143.0
+    image: victoriametrics/vmagent:v1.144.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.143.0
+    image: victoriametrics/victoria-metrics:v1.144.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.143.0
+    image: victoriametrics/vmalert:v1.144.0
     depends_on:
       - victoriametrics
       - alertmanager

docs/playgrounds/README.md

@@ -150,8 +150,12 @@ You can experiment with your own data during the month‑long trial without depl
 are fast-booting Linux microVMs that run on a fleet of large bare-metal servers. You can start a playground right from your browser. 
 Once up and running, accessing a playground is no different from SSH-ing into a remote server rented from your favorite VPS or Cloud provider.
 
-Iximiuz Labs provides playgrounds for VictoriaMetrics software:
-- [VictoriaMetrics single node (on Ubuntu)](https://labs.iximiuz.com/playgrounds/victoriametrics-e2f9b613)
-- [VictoriaMetrics cluster (on Ubuntu)](https://labs.iximiuz.com/playgrounds/victoriametrics-cluster-8eacb19d)
-- [Getting Started with VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/tutorials/victoriametrics-getting-started-kubernetes-0e9c0993)
-- [VictoriaMetrics Operator](https://labs.iximiuz.com/playgrounds/victoriametrics-kubernetes-9eebc258)
+Iximiuz Labs provides various [learning-by-doing resources for VictoriaMetrics](https://labs.iximiuz.com/v/victoriametrics-bb1fdaa1):
+- Tutorial:
+  - [Getting Started with VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/tutorials/victoriametrics-getting-started-kubernetes-0e9c0993)
+- Playgrounds:
+  - [VictoriaMetrics single node](https://labs.iximiuz.com/playgrounds/victoriametrics-e2f9b613)
+  - [VictoriaMetrics cluster](https://labs.iximiuz.com/playgrounds/victoriametrics-cluster-8eacb19d)
+  - [VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/playgrounds/victoriametrics-kubernetes-9eebc258)
+
+Iximiuz Labs requires a [free account](https://labs.iximiuz.com/signup) to access the materials.

docs/victoriametrics/CONTRIBUTING.md

@@ -4,8 +4,8 @@ title: Contributing
 menu:
   docs:
     identifier: vm-contributing
-    parent: victoriametrics
     weight: 400
+    pageRef: "/victoriametrics/contributing/"
 tags: []
 aliases:
 - /CONTRIBUTING.html

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -566,7 +566,7 @@ The following optional command-line flags related to mTLS are supported:
 - `-cluster.tlsCAFile` can be set at `vminsert`, `vmselect` and `vmstorage` for verifying peer certificates issued with custom [certificate authority](https://en.wikipedia.org/wiki/Certificate_authority). By default, system-wide certificate authority is used for peer certificate verification.
 - `-cluster.tlsCipherSuites` can be set to the list of supported TLS cipher suites at `vmstorage`. See [the list of supported TLS cipher suites](https://pkg.go.dev/crypto/tls#pkg-constants).
 
-When `vmselect` runs with `-clusternativeListenAddr` command-line option, then it can be configured with `-clusternative.tls*` options similar to `-cluster.tls*` for accepting `mTLS` connections from top-level `vmselect` nodes in [multi-level cluster setup](#multi-level-cluster-setup).
+When `vmselect` or `vminsert` runs with `-clusternativeListenAddr` command-line option, then it can be configured with `-clusternative.tls*` options similar to `-cluster.tls*` for accepting `mTLS` connections from top-level `vmselect` or `vminsert` nodes in [multi-level cluster setup](#multi-level-cluster-setup).
 
 See [these docs](https://gist.github.com/f41gh7/76ed8e5fb1ebb9737fe746bae9175ee6) on how to set up mTLS in VictoriaMetrics cluster.
 
@@ -700,6 +700,7 @@ Also in the cluster version the `/prometheus/api/v1` endpoint ingests  `jsonl`,
 
 - `vmstorage` nodes provide the following HTTP endpoints on `8482` port:
   - `/internal/force_merge` - initiate [forced compactions](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-merge) on the given `vmstorage` node.
+  - `/internal/force_flush` - [flush in-memory buffers](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-flush) for recently ingested samples into searchable parts on the given `vmstorage` node.
   - `/snapshot/create` - create [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282),
     which can be used for backups in background. Snapshots are created in `<storageDataPath>/snapshots` folder, where `<storageDataPath>` is the corresponding
     command-line flag value.

docs/victoriametrics/Quick-Start.md

@@ -43,6 +43,9 @@ Just download VictoriaMetrics and follow [these instructions](https://docs.victo
 See [available integrations](https://docs.victoriametrics.com/victoriametrics/integrations/) with other systems like
 [Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/) or [Grafana](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/).
 
+> Want to see VictoriaMetrics in action, but without installing anything?
+> Try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) - a list of publicly available playgrounds for VictoriaMetrics software.
+
 VictoriaMetrics is developed at a fast pace, so it is recommended to periodically check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/)
 and perform [regular upgrades](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
 
@@ -58,9 +61,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.143.0
+docker pull victoriametrics/victoria-metrics:v1.144.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.143.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.144.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
@@ -147,7 +150,7 @@ After=network.target
 Type=simple
 User=victoriametrics
 Group=victoriametrics
-ExecStart=/usr/local/bin/victoria-metrics-prod -storageDataPath=/var/lib/victoria-metrics -retentionPeriod=90d -selfScrapeInterval=10s
+ExecStart=/usr/local/bin/victoria-metrics-prod -storageDataPath=/var/lib/victoria-metrics -selfScrapeInterval=10s
 SyslogIdentifier=victoriametrics
 Restart=always
 
@@ -231,7 +234,7 @@ Type=simple
 User=victoriametrics
 Group=victoriametrics
 Restart=always
-ExecStart=/usr/local/bin/vmstorage-prod -retentionPeriod=90d -storageDataPath=/var/lib/vmstorage
+ExecStart=/usr/local/bin/vmstorage-prod -storageDataPath=/var/lib/vmstorage
 
 PrivateTmp=yes
 NoNewPrivileges=yes
@@ -434,6 +437,14 @@ the main monitoring installation.
 
 See more details in the article [VictoriaMetrics Monitoring](https://victoriametrics.com/blog/victoriametrics-monitoring/).
 
+### Retention
+
+VictoriaMetrics Single-node and `vmstorage` in VictoriaMetrics Cluster retain data for 1 month by default.
+Data older than the retention period will be automatically deleted. To change the retention period, use the `-retentionPeriod` flag (e.g. `-retentionPeriod=90d`).
+See the [retention](https://docs.victoriametrics.com/victoriametrics/#retention) documentation for more details. 
+
+If free disk space falls below `-storage.minFreeDiskSpaceBytes`, VictoriaMetrics Single-node or `vmstorage` switches to read-only mode and stops accepting new data. To prevent this, ensure proper [capacity planning](#capacity-planning) and set up monitoring and alerting for disk usage.
+
 ### Capacity planning
 
 See capacity planning sections in [docs](https://docs.victoriametrics.com) for

docs/victoriametrics/README.md

@@ -701,18 +701,37 @@ It's better to use the `-retentionPeriod` command-line flag for efficient prunin
 
 ## Forced merge
 
-VictoriaMetrics performs [data compactions in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
-in order to keep good performance characteristics when accepting new data. These compactions (merges) are performed independently on per-month partitions.
-This means that compactions are stopped for per-month partitions if no new data is ingested into these partitions.
-Sometimes it is necessary to trigger compactions for old partitions. For instance, in order to free up disk space occupied by [deleted time series](#how-to-delete-time-series).
-In this case forced compaction may be initiated on the specified per-month partition by sending request to `/internal/force_merge?partition_prefix=YYYY_MM`,
-where `YYYY_MM` is per-month partition name. For example, `http://victoriametrics:8428/internal/force_merge?partition_prefix=2020_08` would initiate forced
-merge for August 2020 partition. The call to `/internal/force_merge` returns immediately, while the corresponding forced merge continues running in background.
+VictoriaMetrics performs [data compaction in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+to keep high performance during ingestion. These compactions (merges) are performed independently on per-month partitions.
+Partitions that don't receive new data will eventually stop being merged, as they reach an optimal state.
+Sometimes it is necessary to trigger merges for old partitions. For example, to free up disk space occupied by [deleted time series](https://docs.victoriametrics.com/victoriametrics/#how-to-delete-time-series).
+The user can **force** compaction on the specified per-month partition by sending a request to `/internal/force_merge?partition_prefix=YYYY_MM`,
+where `YYYY_MM` is the per-month partition name. For example, `http://victoriametrics:8428/internal/force_merge?partition_prefix=2020_08` would initiate a forced
+merge for the August 2020 partition. The call to `/internal/force_merge` returns immediately, while the corresponding forced merge continues running in the background.
 
-Forced merges may require additional CPU, disk IO and storage space resources. It is unnecessary to run forced merge under normal conditions,
+Forced merges may require additional CPU, disk I/O, and storage space. It is unnecessary to run a forced merge under normal conditions,
 since VictoriaMetrics automatically performs [optimal merges in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
 when new data is ingested into it.
 
+The `/internal/force_merge` endpoint can be protected from unauthorized access via the `--forceMergeAuthKey` command-line flag.
+See [General security recommendations](https://docs.victoriametrics.com/victoriametrics/#general-security-recommendations) for more details.
+
+## Forced flush
+
+VictoriaMetrics puts the recently [ingested samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#write-data) into in-memory buffers,
+which aren't available for [querying](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data) for up to a few seconds.
+If you need to query the samples immediately after they are ingested, then call the `/internal/force_flush` HTTP endpoint before running your query.
+This endpoint converts in-memory buffers containing recently ingested samples into searchable data blocks.
+
+It isn't recommended to force flush on a regular basis, as this increases CPU usage and slows data ingestion.
+The `/internal/force_flush` endpoint exists for debug and test purposes (for instance, for automated tests) and should be avoided in production.
+
+> VictoriaMetrics may intentionally hide samples with timestamps close to the current time during querying.
+> This behavior is controlled via `-search.latencyOffset` command-line flag. See more details in [Query latency](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-latency) documentation.
+
+The `/internal/force_flush` endpoint can be protected from unauthorized access via `-forceFlushAuthKey` command-line flag.
+See [General security recommendations](https://docs.victoriametrics.com/victoriametrics/#general-security-recommendations) for more details.
+
 ## How to export time series
 
 VictoriaMetrics provides the following handlers for exporting data:
@@ -1736,16 +1755,16 @@ VictoriaMetrics provides the following security-related command-line flags:
 * `-mtls` and `-mtlsCAFile` for enabling [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) for requests to `-httpListenAddr`. See [these docs](#mtls-protection).
 * `-httpAuth.username` and `-httpAuth.password` for protecting all the HTTP endpoints
   with [HTTP Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
-* `-deleteAuthKey` for protecting `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
-* `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
-* `-forceFlushAuthKey` for protecting `/internal/force_flush` endpoint. See [these docs](#troubleshooting).
-* `-forceMergeAuthKey` for protecting `/internal/force_merge` endpoint. See [force merge docs](#forced-merge).
-* `-search.resetCacheAuthKey` for protecting `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
-* `-reloadAuthKey` for protecting `/-/reload` endpoint, which is used for force reloading of [`-promscrape.config`](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
-* `-configAuthKey` for protecting `/config` endpoint, since it may contain sensitive information such as passwords.
-* `-flagsAuthKey` for protecting `/flags` endpoint.
-* `-pprofAuthKey` for protecting `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
-* `-metricNamesStatsResetAuthKey` for protecting `/api/v1/admin/status/metric_names_stats/reset` endpoint, used for [Metric Names Tracker](#track-ingested-metrics-usage).
+* `-deleteAuthKey` for protecting the `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
+* `-snapshotAuthKey` for protecting the `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
+* `-forceFlushAuthKey` for protecting the `/internal/force_flush` endpoint. See [force flush docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-flush).
+* `-forceMergeAuthKey` for protecting the `/internal/force_merge` endpoint. See [force merge docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-merge).
+* `-search.resetCacheAuthKey` for protecting the `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
+* `-reloadAuthKey` for protecting the `/-/reload` endpoint, which is used for force reloading of [`-promscrape.config`](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
+* `-configAuthKey` for protecting the `/config` endpoint, since it may contain sensitive information such as passwords.
+* `-flagsAuthKey` for protecting the `/flags` endpoint.
+* `-pprofAuthKey` for protecting the `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
+* `-metricNamesStatsResetAuthKey` for protecting the `/api/v1/admin/status/metric_names_stats/reset` endpoint, used for [Metric Names Tracker](#track-ingested-metrics-usage).
 * `-denyQueryTracing` for disallowing [query tracing](#query-tracing).
 * `-http.header.hsts`, `-http.header.csp`, and `-http.header.frameOptions` for serving `Strict-Transport-Security`, `Content-Security-Policy`
   and `X-Frame-Options` HTTP response headers.

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,11 +26,21 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-opentelemetry.promoteAllResourceAttributes` and `-opentelemetry.promoteScopeMetadata` command-line flags to allow managing label promotion for resource attributes and OTel scope metadata. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#10931](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10931).
+
+## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
+
+Released at 2026-05-22
+
 * FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
+* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `clusternative.tls` `vminsert` configuration flags for [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). See [#10958](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10958).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
+* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Improve [slowness-based rerouting](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) to prevent rerouting storms under high cluster load. Previously, rerouting could cascade across storage nodes when the whole cluster was saturated, making the situation worse. Now rerouting only activates when the cluster p90 saturation is below 60%, and the slowest node is more than 20% slower than p90. See [#10876](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10876).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `{{.MetricsAccountID}}` and `{{.MetricsProjectID}}` [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) for use in `headers` and `url_prefix` config fields. Previously, only the combined `{{.MetricsTenant}}` (`accountID:projectID`) JWT placeholder was supported, making it impossible to configure [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers). See [#10927](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10927). Thanks to @Vinayak9769 for the contribution.
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
@@ -43,6 +53,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
+* BUGFIX: all VictoriaMetrics components: fix int64 overflow when parsing [timestamp parameters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats) with relative durations. See [#10880](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
 
 ## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)
 
@@ -228,6 +242,28 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.10](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.10)
+
+Released at 2026-05-22
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive credentials.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): hide values passed to `-remoteWrite.headers`,`remoteRead.headers`, `datasource.headers` and `notifier.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly establish [mtls](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection) connection between vmstorage and vminsert. Regression was introduced in v1.130.0 release for the enterprise version of vmstorage. See [#10972](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10972).
+* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
+
 ## [v1.136.9](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.9)
 
 Released at 2026-05-08
@@ -549,6 +585,26 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.23](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.23)
+
+Released at 2026-05-22
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive credentials.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): hide values passed to `-remoteWrite.headers`,`remoteRead.headers`, `datasource.headers` and `notifier.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
+
 ## [v1.122.22](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.22)
 
 Released at 2026-05-08

docs/victoriametrics/changelog/CHANGELOG_2022.md

@@ -711,7 +711,7 @@ Released at 2022-06-20
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): expose `/api/v1/status/config` endpoint in the same way as Prometheus does. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#config).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-promscrape.suppressScrapeErrorsDelay` command-line flag, which can be used for delaying and aggregating the logging of per-target scrape errors. This may reduce the amounts of logs when `vmagent` scrapes many unreliable targets. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2575). Thanks to @jelmd for [the initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2576).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-promscrape.cluster.name` command-line flag, which allows proper data de-duplication when the same target is scraped from multiple [vmagent clusters](https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2679).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `action: graphite` relabeling rules optimized for extracting labels from Graphite-style metric names. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmagent/#graphite-relabeling) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2737).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `action: graphite` relabeling rules optimized for extracting labels from Graphite-style metric names. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#graphite-relabeling) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2737).
 * FEATURE: [VictoriaMetrics Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/): expose `vm_downsampling_partitions_scheduled` and `vm_downsampling_partitions_scheduled_size_bytes` metrics, which can be used for tracking the progress of initial [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling) for historical data. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2612).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): do not spend up to 5 seconds when trying to connect to unavailable `vmstorage` nodes. This should improve query latency when some of `vmstorage` nodes aren't available. Expose `vm_tcpdialer_addr_available{addr="..."}` metric at `http://vmselect:8481/metrics` for determining whether the given `addr` is available for establishing new connections. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711#issuecomment-1160363187).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-vmstorageDialTimeout` command-line flags to `vmselect` and `vminsert` for tuning the maximum duration for connection establishing to `vmstorage` nodes. This should help resolving [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711).

docs/victoriametrics/changelog/CHANGELOG_2023.md

@@ -402,7 +402,7 @@ so rolling back to the previous versions of VictoriaMetrics may result in partia
 the following samples to the configured remote storage by default:
 
 - aggregated samples;
-- the original input samples, which match zero `match` options from the provided [config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config).
+- the original input samples, which match zero `match` options from the provided [config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
 
 Previously only aggregated samples were written to the storage by default.
 The previous behavior can be restored in the following ways:
@@ -429,8 +429,8 @@ The previous behavior can be restored in the following ways:
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow sharding outgoing time series among the configured remote storage systems. This can be useful for building horizontally scalable [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/), when samples for the same time series must be aggregated by the same `vmagent` instance at the second level. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4637) for details.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow configuring staleness interval in [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) config. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4667) for details.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow specifying a list of [series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) inside `if` option of relabeling rules. The corresponding relabeling rule is executed when at least a single series selector matches. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-enhancements).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): allow specifying a list of [series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) inside `match` option of [stream aggregation configs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). The input sample is aggregated when at least a single series selector matches. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4635).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): preserve input samples, which match zero `match` options from the [configured aggregations](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). Previously all the input samples were dropped by default, so only the aggregated samples are written to the output storage. The previous behavior can be restored by passing `-streamAggr.dropInput` command-line flag to single-node VictoriaMetrics or by passing `-remoteWrite.streamAggr.dropInput` command-line flag to `vmagent`.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): allow specifying a list of [series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) inside `match` option of [stream aggregation configs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). The input sample is aggregated when at least a single series selector matches. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4635).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): preserve input samples, which match zero `match` options from the [configured aggregations](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). Previously all the input samples were dropped by default, so only the aggregated samples are written to the output storage. The previous behavior can be restored by passing `-streamAggr.dropInput` command-line flag to single-node VictoriaMetrics or by passing `-remoteWrite.streamAggr.dropInput` command-line flag to `vmagent`.
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): interrupt backoff retries when import process is cancelled. The change makes vmctl more responsive in case of errors during the import. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4442).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): update backoff policy on retries to reduce probability of overloading for `source` or `destination` databases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4402).

docs/victoriametrics/changelog/CHANGELOG_2024.md

@@ -75,7 +75,7 @@ Released at 2024-11-29
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `dryRun` flag to validate configuration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7505) for details.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `removeXFFHTTPHeaderValue` flag to remove content of `X-Forwarded-For` HTTP Header before proxy it to the backend. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6883) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): revert the default value of `-remoteWrite.maxQueueSize` from `1_000_000` to `100_000`. It was bumped in [v1.104.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.104.0), which increases memory usage and is not needed for most setups. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7471).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `ignore_first_sample_interval` param to [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). It allows users to control the time interval when aggregation skips sending aggregated samples to avoid unexpected spikes in values. By default, this interval is set to x2 of `staleness_interval`. The new setting is applicable only to `total`, `total_prometheus`, `increase`, `increase_prometheus` and `histogram_bucket` outputs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7116) for details. Thanks to @iyuroch for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7313).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `ignore_first_sample_interval` param to [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). It allows users to control the time interval when aggregation skips sending aggregated samples to avoid unexpected spikes in values. By default, this interval is set to x2 of `staleness_interval`. The new setting is applicable only to `total`, `total_prometheus`, `increase`, `increase_prometheus` and `histogram_bucket` outputs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7116) for details. Thanks to @iyuroch for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7313).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add support for template alias in predefined panels. This allows creating more readable metric names in the legend using constructions like `{{label_name}}`, where `label_name` is the name of the label. [See this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/116101da78a4dee8bd7c4ba0e66458fd05a10469#diff-95141489b32468cf852d2705d96eaa48c50a8b1cdd0424a29e7ca289912a6dcbR140-R151)
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): properly parse `multitenant` token value for multitenant endpoints. Before, pushing data into vmagent using [multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format) resulted in unsupported path. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7694).
@@ -231,7 +231,7 @@ Released at 2024-08-28
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add command-line flag `-notifier.headers` to allow configuring additional headers for all requests sent to the corresponding `-notifier.url`.
 * FEATURE: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): add `-external.label` and `-external.url` command-line flags, in the same way as these flags are supported by vmalert. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735).
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/), [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): use exponential backoff for retries when uploading or downloading data from S3. This should reduce the number of failed uploads and downloads when S3 is temporarily unavailable. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6732).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): do not allow enabling `-streamAggr.keepInput` and `keep_metric_names` options together in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config), as it may result in time series collision.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): do not allow enabling `-streamAggr.keepInput` and `keep_metric_names` options together in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config), as it may result in time series collision.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add search functionality to the column display settings in the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the ability to select all columns in the column display settings of the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6680).
 * FEATURE: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add command-line flag `-search.inmemoryBufSizeBytes` for configuring size of in-memory buffers used by vmselect during processing of vmstorage responses. A new summary metric `vm_tmp_blocks_inmemory_file_size_bytes` is exposed to show the size of the buffer during requests processing. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6851) for details. Thanks to @tydhot for implementation.
@@ -427,7 +427,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
 
 **Update note 1: support for snap packages was removed due to lack of interest from community. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6543) for details. Please read about supported package types [here](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#install).**
 
-**Update note 2: [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config) now prevents setting multiple identical outputs. For example, `outputs: [total, total]` will fail the validation phase. In addition, `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` will fail the validation as well - use `outputs: ["quantiles(0.5, 0.9)"]` instead.**
+**Update note 2: [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config) now prevents setting multiple identical outputs. For example, `outputs: [total, total]` will fail the validation phase. In addition, `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` will fail the validation as well - use `outputs: ["quantiles(0.5, 0.9)"]` instead.**
 
 * SECURITY: upgrade Go builder from Go1.22.4 to Go1.22.5. See the list of issues addressed in [Go1.22.5](https://github.com/golang/go/issues?q=milestone%3AGo1.22.5+label%3ACherryPickApproved).
 * SECURITY: upgrade base docker image (Alpine) from 3.20.0 to 3.20.1. See [alpine 3.20.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.1-released.html).
@@ -437,11 +437,11 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
   * `vm_streamaggr_output_samples_total` - the number of output samples produced by the corresponding aggregation rule
   * `vm_streamaggr_samples_lag_seconds` - [histogram](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) with the lag between the current time and the timestamp seen in the aggregated input samples
 * FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add new labels to `vm_streamaggr_*` metrics:
-  * `name` - the name of the streaming aggregation rule, which can be configured via `name` option - see [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config).
+  * `name` - the name of the streaming aggregation rule, which can be configured via `name` option - see [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
   * `url` - `-remoteWrite.url` for the corresponding `-remoteWrite.streamAggr.config`
   * `path` - path to the corresponding streaming aggregation config file
   * `position` - the position of the aggregation rule in the corresponding streaming aggregation config file
-* FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent having duplicated aggregation function as `outputs` in one [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). It also prevents using `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` instead of  `outputs: ["quantiles(0.5, 0.9)"]`, as the former has higher computation cost for producing the same result.
+* FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent having duplicated aggregation function as `outputs` in one [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). It also prevents using `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` instead of  `outputs: ["quantiles(0.5, 0.9)"]`, as the former has higher computation cost for producing the same result.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-graphite.sanitizeMetricName` command-line flag for sanitizing metrics ingested via [Graphite protocol](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6077).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): [`yandexcloud_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs): add support for obtaining IAM token in [GCE format](https://yandex.cloud/en-ru/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm) additionally to the [deprecated Amazon EC2 IMDSv1 format](https://yandex.cloud/en/docs/security/standard/authentication#aws-token). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5513).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): make `-replay.timeTo` optional in [replay mode](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling). When omitted, the current timestamp will be used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6492).
@@ -485,7 +485,7 @@ Released at 2024-06-24
 * BUGFIX: all VictoriaMetrics components: prioritize `-configAuthKey` and `-reloadAuthKey` over `-httpAuth.*` settings. This change aligns behavior of mentioned flags with other auth flags like `-metricsAuthKey`, `-flagsAuthKey`, `-pprofAuthKey`. Check [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6329).
 * BUGFIX: all VictoriaMetrics components: do not trim trailing spaces when reading content from `*.passwordFile` and similar flags. Previously, trailing spaces were trimmed from the content of the password file, which could lead to unexpected authentication errors.
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--disable-progress-bar` global command-line flag. It can be used for disabling dynamic progress bar for all migration modes. `--vm-disable-progress-bar`  command-line flag is deprecated and will be removed in the future releases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6367).
-* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_avg) producing `NaN` results for stale time series. Before, when series matched for aggregation became stale or weren't updated during aggregation interval, the `rate_sum` or `rate_avg` could produce data point with `NaN` value. During visualization, such aggregation results would be displayed as gaps in time series.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_avg) producing `NaN` results for stale time series. Before, when series matched for aggregation became stale or weren't updated during aggregation interval, the `rate_sum` or `rate_avg` could produce data point with `NaN` value. During visualization, such aggregation results would be displayed as gaps in time series.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix path for system links printed on default vmalert's UI page when `-http.pathPrefix` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6433).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): exit replay mode with non-zero code if generated samples are not successfully written into remoteWrite url. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6512).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/): properly configure authentication with S3 when `-configFilePath` cmd-line flag is specified. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6353).
@@ -520,7 +520,7 @@ Released at 2024-06-07
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add service discovery support for [Vultr](https://www.vultr.com/). See [these docs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#vultr_sd_configs) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6041).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow specifying `-remoteWrite.disableOnDiskQueue` command-line flag per each `-remoteWrite.url`. If multiple `-remoteWrite.disableOnDiskQueue` command-line flags are configured, then the `-remoteWrite.dropSamplesOnOverload` is automatically set to true, so samples are automatically dropped if they cannot be sent to the corresponding `-remoteWrite.url` in a timely manner. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6065). Thanks to @rbizos for implementation!
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `path` and `url` labels to `vmagent_remotewrite_push_failures_total` and `vmagent_remotewrite_samples_dropped_total` [metrics](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring), so the number of failed pushes and dropped samples can be tracked per each `-remoteWrite.url`.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_avg) aggregation outputs.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_avg) aggregation outputs.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): reduce the number of allocated objects in heap during deduplication and aggregation. The change supposed to reduce pressure on Garbage Collector, as it will need to scan less objects. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6402).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-datasource.idleConnTimeout`, `-remoteWrite.idleConnTimeout` and `-remoteRead.idleConnTimeout` flags. These flags are set to 50s by default and should reduce the probability of `broken pipe` or `connection reset by peer` errors in vmalert logs. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5661) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add auto request retry for trivial network errors, such as `broken pipe` and `connection reset` for requests to `remoteRead`, `remoteWrite` and `datasource` URLs. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5661) for details.
@@ -531,7 +531,7 @@ Released at 2024-06-07
 * FEATURE: expose metric `vm_indexdb_items_dropped_total` to track the number of IndexDB records that had to be dropped during ingestion. The reason of dropping the record will be annotated in `reason` label of the exposed metric. This change also comes with a new [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) to track changes of this metric.
 * FEATURE: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): add new alerting rules `TooLongLabelValues` and `TooLongLabelNames` to notify about truncation of label values or names respectively.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): expose `vm_streamaggr_ignored_samples_total` [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter) at [`/metrics` page](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring), which can be used for detecting amount of too old or NaN valued ignored samples. Expose also `vm_streamaggr_samples_lag_seconds` [histogram](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) to monitor aggregated samples lag.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): improve filtering speed of the received data samples if [match](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config) field is matching only [metric name](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#structure-of-a-metric).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): improve filtering speed of the received data samples if [match](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config) field is matching only [metric name](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#structure-of-a-metric).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): adds `-relabelConfigCheckInterval` flag and `/-/reload` endpoint for reloading relabeling rules. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3923) for details.
 
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix bug that prevents the first query trace from expanding on click event. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6186). The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11000) release.
@@ -540,7 +540,7 @@ Released at 2024-06-07
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix handling of URL params for browser history navigation (back and forward buttons). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6126) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5516#issuecomment-1867507232).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent potential panic during [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) when more than one `-remoteWrite.url` command-line flags are passed to `vmagent` together with non-zero `-remoteWrite.streamAggr.dedupInterval` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6205).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): skip empty data blocks before sending to the remote write destination. Thanks to @viperstars for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241).
-* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): set correct suffix `<output>_prometheus` for aggregation outputs [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total_prometheus). Before, outputs `total` and `total_prometheus` or `increase` and `increase_prometheus` had the same suffix.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): set correct suffix `<output>_prometheus` for aggregation outputs [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total_prometheus). Before, outputs `total` and `total_prometheus` or `increase` and `increase_prometheus` had the same suffix.
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent from excessive resource usage when stream aggregation config file is empty.
 * BUGFIX: properly estimate the needed memory for query execution if it has the format [`aggr_func`](https://docs.victoriametrics.com/victoriametrics/metricsql/#aggregate-functions)([`rollup_func[d]`](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions) (for example, `sum(rate(request_duration_seconds_bucket[5m]))`). This should allow performing aggregations over bigger number of time series when VictoriaMetrics runs in environments with small amounts of available memory. The issue has been introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/5138eaeea0791caa34bcfab410e0ca9cd253cd8f) in [v1.83.0](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2022/#v1830).
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly apply `-inmemoryDataFlushInterval` when it's set to minimum supported value 1s.
@@ -616,14 +616,14 @@ Released at 2024-04-04
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): reduce memory usage by up to 5x when aggregating over big number of unique [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series). The memory usage reduction is most visible when [stream deduplication](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) is enabled.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): allow using `-streamAggr.dedupInterval` and `-remoteWrite.streamAggr.dedupInterval` command-line flags without the need to specify `-streamAggr.config` and `-remoteWrite.streamAggr.config`. See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `-streamAggr.dropInputLabels` command-line flag, which can be used for dropping the listed labels from input samples before applying stream [de-duplication](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) and aggregation. This is faster and easier to use alternative to [input_relabel_configs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#relabeling). See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `dedup_interval` option, which allows configuring individual [deduplication intervals](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) per each [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `dedup_interval` option, which allows configuring individual [deduplication intervals](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) per each [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): use the same logic in [stream deduplication](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) as in [the deduplication at VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5643).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): ignore out of order samples when calculating [`increase`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase), [`increase_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase_prometheus), [`total`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total) and [`total_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total_prometheus) outputs. Thanks to @edma2 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5931).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): ignore out of order samples when calculating [`increase`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase), [`increase_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase_prometheus), [`total`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total) and [`total_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total_prometheus) outputs. Thanks to @edma2 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5931).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add an ability to ignore input samples with old timestamps outside the current aggregation interval. See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples) for details.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `keep_metric_names` option, which can be set at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config) in order to keep the original metric names in the output aggregated samples instead of using [the default output metric naming scheme](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#output-metric-names).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): align the time of aggregated data flush to the specified aggregation `interval`. For example, if `interval` is set to `1m`, then the aggregated data will be flushed at the end of every minute. The alignment can be disabled by setting `no_align_flush_to_interval: true` option at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#flush-time-alignment) for details.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [unique_samples](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#unique_samples) output, which can be used for calculating the number of unique sample values over the given `interval`.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total_prometheus) outputs, which can be used for `increase` and `total` aggregations when the first sample of every new [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) must be ignored.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `keep_metric_names` option, which can be set at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config) in order to keep the original metric names in the output aggregated samples instead of using [the default output metric naming scheme](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#output-metric-names).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): align the time of aggregated data flush to the specified aggregation `interval`. For example, if `interval` is set to `1m`, then the aggregated data will be flushed at the end of every minute. The alignment can be disabled by setting `no_align_flush_to_interval: true` option at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#flush-time-alignment) for details.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [unique_samples](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#unique_samples) output, which can be used for calculating the number of unique sample values over the given `interval`.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total_prometheus) outputs, which can be used for `increase` and `total` aggregations when the first sample of every new [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) must be ignored.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): expose `vm_streamaggr_flush_timeouts_total` and `vm_streamaggr_dedup_flush_timeouts_total` [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter) at [`/metrics` page](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring), which can be used for detecting flush timeouts for stream aggregation states. Expose also `vm_streamaggr_flush_duration_seconds` and `vm_streamaggr_dedup_flush_duration_seconds` [histograms](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) for monitoring the real flush durations of stream aggregation states.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve trace display for better visual separation of branches. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5926).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): ability to limit the ingestion rate via `-maxIngestionRate` command-line flag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5900).
@@ -989,7 +989,7 @@ The v1.97.x line will be supported for at least 12 months since [v1.97.0](https:
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduce initial memory usage when scraping targets, which return huge responses (for example, [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) in large Kubernetes cluster may return 100MB+ responses), without the need to explicitly enable [stream parsing mode](https://docs.victoriametrics.com/victoriametrics/vmagent/#stream-parsing-mode). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5567).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): expose ability to set OAuth2 endpoint parameters per each `-remoteWrite.url` via the command-line flag `-remoteWrite.oauth2.endpointParams`. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmagent/#advanced-usage). Thanks to @mhill-holoplot for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5427).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add ability to set `attach_metadata.node=true` option for all the [`kubernetes_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs) defined at [`-promscrape.config`](https://docs.victoriametrics.com/victoriametrics/vmagent/#quick-start) via `-promscrape.kubernetes.attachNodeMetadataAll` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4640). Thanks to @wasim-nihal for [the initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5593).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): do not send unfinished [aggregation state](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) on shutdown or [hot config reload](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#configuration-update) by default, as it tend to produce unexpected anomalies with lower values. The old behavior can be restored by specifying `flush_on_shutdown: true` setting in streaming aggregation config. See more details [here](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-outputs).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): do not send unfinished [aggregation state](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) on shutdown or [hot config reload](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#configuration-update) by default, as it tend to produce unexpected anomalies with lower values. The old behavior can be restored by specifying `flush_on_shutdown: true` setting in streaming aggregation config. See more details [here](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#aggregation-outputs).
 * FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): expand `%{ENV_VAR}` placeholders in config files with the corresponding environment variable values.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmagent/): expose ability to set OAuth2 endpoint parameters via the following command-line flags:
   * `-datasource.oauth2.endpointParams` for `-datasource.url`

docs/victoriametrics/enterprise.md

@@ -121,7 +121,7 @@ It is allowed to run Enterprise components in [cases listed here](https://docs.v
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
 [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
 and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.144.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -139,8 +139,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.144.0/victoria-metrics-linux-amd64-v1.144.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.144.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -155,7 +155,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.143.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.144.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
@@ -165,13 +165,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.143.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.144.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.143.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.144.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -181,7 +181,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.143.0
+    image: victoriametrics/victoria-metrics:v1.144.0
     ports:
       - 8428:8428
     volumes:
@@ -213,7 +213,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.143.0-enterprise
+    tag: v1.144.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -224,7 +224,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.143.0-enterprise
+    tag: v1.144.0-enterprise
 
 license:
   secret:
@@ -274,7 +274,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.143.0-enterprise
+    tag: v1.144.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -291,7 +291,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.143.0-enterprise
+    tag: v1.144.0-enterprise
 ```
 
 Example secret with license key:
@@ -342,7 +342,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.144.0-enterprise.tar.gz`
 
 Includes:
 
@@ -351,7 +351,19 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.143.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.144.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+
+## What Happens to Licensed Components When a License Expires
+
+When a license expires, all licensed components continue to function normally until a restart occurs.
+
+License checks happen only at startup. If a license expires while the component is running, nothing changes; the component continues to run until the next restart. 
+
+This means you don't need to restart components to install a new license. The component automatically picks up the new license the next time it restarts. The exception is when the `-license` flag is used, because the license is supplied at startup and changing it requires restarting VictoriaMetrics with the updated flag value.
+
+If your license has expired and you decide to not renew it, you can switch to the VictoriaMetrics Open Source version without data loss, as both versions share the same data model. In doing so, however, you will lose access to the [VictoriaMetrics Enterprise features](https://docs.victoriametrics.com/victoriametrics/enterprise/#victoriametrics-enterprise-features).
+
+See [updating the license key](https://docs.victoriametrics.com/victoriametrics/enterprise/#updating-the-license-key) for more details.
 
 ## Monitoring license expiration
 

docs/victoriametrics/goals.md

@@ -1,10 +1,11 @@
 ---
 weight: 500
-title: Development goals
+title: Development Goals
 menu:
   docs:
-    parent: 'victoriametrics'
     weight: 500
+    identifier: goals
+    pageRef: "/victoriametrics/goals/"
 tags: []
 aliases:
   - /goals/index.html

docs/victoriametrics/integrations/grafana/_index.md

@@ -59,6 +59,10 @@ Once connected, you can build graphs and dashboards using [PromQL](https://prome
 _Creating a datasource may require [specific permissions](https://grafana.com/docs/grafana/latest/administration/data-source-management/).
 If you don't see an option to create a data source - try contacting system administrator._
 
+If you run [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) and want to see its rules in [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/),
+then set configure `-vmalert.proxyURL` on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
+or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert).
+
 ## Multi-tenant access with vmauth and OIDC
 
 [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) can proxy Grafana datasource requests and enforce

docs/victoriametrics/integrations/opentelemetry.md

@@ -28,9 +28,17 @@ The following label sanitization options can be enabled:
 
 > These flags can be applied on vmagent, vminsert or VictoriaMetrics single-node.
 
+## Instrumentation Scope
+
+By default, VictoriaMetrics promotes [OTel scope metadata](https://opentelemetry.io/docs/specs/otel/common/instrumentation-scope/) to metric labels. This behavior can be disabled via `-opentelemetry.promoteScopeMetadata`.
+
 ## Resource Attributes
 
 By default, VictoriaMetrics promotes all [OpenTelemetry resource](https://opentelemetry.io/docs/specs/otel/resource/data-model/) attributes to labels and attaches them to all ingested OTLP metrics.
+The following attribute promotion options can be configured:
+* `-opentelemetry.promoteAllResourceAttributes` - promotes all resource attributes to labels, except for the ones configured with `-opentelemetry.ignoreResourceAttributes`.
+* `-opentelemetry.promoteResourceAttributes` - promotes specific list of resource attributes to labels. It cannot be configured simultaneously with `-opentelemetry.promoteAllResourceAttributes`.
+* `-opentelemetry.ignoreResourceAttributes` - controls which resource attributes to ignore, can only be set when `-opentelemetry.promoteAllResourceAttributes` is true.
 
 ## Exponential histograms
 

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/victoria-metrics-linux-amd64-v1.143.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.143.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.144.0/victoria-metrics-linux-amd64-v1.144.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.144.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/victoria-metrics-linux-amd64-v1.143.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.143.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.144.0/victoria-metrics-linux-amd64-v1.144.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.144.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/stream-aggregation/README.md

@@ -26,6 +26,7 @@ Stream aggregation has the following features:
   and/or scraped from [Prometheus-compatible targets](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter)
 - It can filter out raw samples matched by aggregation rules, so raw data will never reach the remote destination. See `-streamAggr.keepInput` and `-streamAggr.dropInput` in [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/);
 - It allows building [flexible processing pipelines](#routing);
+- It is [horizontally scalable](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#scaling-aggregation-horizontally).
 
 # Limitations
 
@@ -360,9 +361,9 @@ the received data, scraped or pushed. See the [processing order for vmagent](htt
 
 Typical scenarios for data routing with `vmagent`:
 
-1. **Aggregate incoming data and replicate to N destinations**. Specify [`-streamAggr.config`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#configuration) command-line flag
+1. **Aggregate incoming data and replicate to N destinations**. Specify [`-streamAggr.config`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/) command-line flag
    to aggregate the incoming data before replicating it to all the configured `-remoteWrite.url` destinations.
-2. **Individually aggregate incoming data for each destination**. Specify [`-remoteWrite.streamAggr.config`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#configuration)
+2. **Individually aggregate incoming data for each destination**. Specify [`-remoteWrite.streamAggr.config`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/)
    command-line flag for each `-remoteWrite.url` destination. [Relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) via `-remoteWrite.urlRelabelConfig`
    can be used for routing only the selected metrics to each `-remoteWrite.url` destination.
 
@@ -598,6 +599,47 @@ Below is an example of an `aggr.yaml` configuration that drops the `replica` and
   keep_metric_names: true
 ```
 
+## Scaling aggregation horizontally
+
+Aggregation output is only correct when all contributing samples are processed by the same aggregator instance.
+
+To scale the aggregation horizontally, always shard the input samples in a deterministic way. This can be achieved by
+building a two layer topology of vmagents where the first layer is responsible for sharding, and the second layer is responsible for aggregating:
+```mermaid
+flowchart LR
+    V1[vmagent-shard-1] -- requests_total{env=test, pod=foo} --> SV1[vmagent-aggr-1]
+    V1[vmagent-shard-1] -- requests_total{env=prod, pod=bar} --> SV2[vmagent-aggr-1]
+    V2[vmagent-shard-2] -- requests_total{env=prod, pod=baz} --> SV2[vmagent-aggr-2]
+    SV1 -- requests_total:5m_without_pod_total{env=test} --> x(( ))
+    SV2 -- requests_total:5m_without_pod_total{env=prod} --> y(( ))
+style x fill:none,stroke:none
+style y fill:none,stroke:none
+```
+
+The sharding layer of vmagents can be configured via the `-remoteWrite.shardByURL.labels` or `-remoteWrite.shardByURL.ignoreLabels`
+command line flags. See how to [shard data across remote write destinations](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages) for more details.
+
+The following requirements must be met for sharded aggregation to work correctly:
+- All sharding vmagents should have the same deterministic sharding configuration.
+- The sharding configuration must align with the `by` and `without` lists:
+  - Labels listed in `by` setting should be a subset of shard's routing key `-remoteWrite.shardByURL.labels`. 
+    With `-remoteWrite.shardByURL.labels=env,job` aggregator's `by` should include `by: env`, `by: job` or both: `by: [env, job]`.
+    This makes sure that all the samples for the same `env` and `job` are aggregated together and produce the complete output.
+  - Labels listed in `without` setting should be a superset of shard's routing key `--remoteWrite.shardByURL.ignoreLabels`.
+    With `-remoteWrite.shardByURL.ignoreLabels=env,job` aggegator's `without` should include at least both labels `without: [env,job]`.
+    This makes sure that `requests_total{env=test, job=foo}` and `requests_total{env=prod, job=foo}` are routed to the same aggregator
+    and are aggregated together. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5938#issuecomment-2018470324).
+- Aggregating vmagents should not produce collisions: the aggregation output should be unique across all the sharded agents.
+  For example, `requests_total:5m_without_env_pod_total` produced by both `vmagent-aggr-1` and `vmagent-aggr-2` will collide
+  unless they have labels uniquely identifying them. These labels should be either preserved during sharding and aggregation config,
+  or enforced on the output via `-remoteWrite.label` - see [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#cluster-mode) for more details.
+
+> Never shard histograms by `le` (or `vmrange` in case of VM histograms) label. A histogram is a logical group of series differing
+only in the bucket label. All of those buckets must land on the same aggregator at the same time so it can produce a
+coherent bucket set. See more about [aggregating histograms](https://docs.victoriametrics.com/stream-aggregation/#aggregating-histograms).
+
+See also [why you shouldn't put an aggregator behind a load balancer](https://docs.victoriametrics.com/stream-aggregation/#put-aggregator-behind-load-balancer).
+
 # Troubleshooting
 
 - [Unexpected spikes for `total` or `increase` outputs](#staleness).
@@ -628,7 +670,7 @@ the following settings:
   `-remoteWrite.streamAggr.enableWindows` flag can be specified individually for each `-remoteWrite.url`.
   If one of these flags is set, all aggregators will use fixed windows. In conjunction with `-remoteWrite.streamAggr.dedupInterval` or
   `-streamAggr.dedupInterval` fixed aggregation windows are enabled on the deduplicator as well.
- - `enable_windows` option in [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config).
+ - `enable_windows` option in [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
   It allows enabling aggregation windows for a specific aggregator.
 
 ## Counter resets

docs/victoriametrics/stream-aggregation/configuration.md

@@ -132,7 +132,7 @@ specified individually per each `-remoteWrite.url`:
   # by: [job, vmrange]
 
   # outputs is the list of unique aggregations to perform on the input data.
-  # See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-outputs
+  # See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#aggregation-outputs
   #
   outputs: [total]
 

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -37,7 +37,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Flag value can be read from the given file when using -deleteAuthKey=file:///abs/path/to/file or -deleteAuthKey=file://./relative/path/to/file.
      Flag value can be read from the given http/https url when using -deleteAuthKey=http://host/path or -deleteAuthKey=https://host/path
   -denyQueriesOutsideRetention
-     Whether to deny queries outside the configured -retentionPeriod. When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
+     Whether to deny queries outside the configured -retentionPeriod and -futureRetention. When set, then /api/v1/query_range will return an error for queries with 'from' value outside -retentionPeriod or 'to' value beyond -futureRetention. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
   -denyQueryTracing
      Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing
   -disablePerDayIndex
@@ -198,7 +198,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
   -maxLabelsPerTimeseries int
      The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented (default 40)
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
@@ -217,9 +217,19 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
   -opentelemetry.convertMetricNamesToPrometheus
      Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
+  -opentelemetry.ignoreResourceAttributes array
+     Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
+  -opentelemetry.labelNameUnderscoreSanitization
+     Whether to enable prepending of 'key' to labels starting with '_' when -opentelemetry.usePrometheusNaming is enabled. Reserved labels starting with '__' are not modified. See https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/ (default true)
   -opentelemetry.maxRequestSize size
      The maximum size in bytes of a single OpenTelemetry request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
+  -opentelemetry.promoteAllResourceAttributes
+     Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
+  -opentelemetry.promoteResourceAttributes array
+     Promote specific list of resource attributes to labels.
+  -opentelemetry.promoteScopeMetadata
+     Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
   -opentelemetry.usePrometheusNaming
      Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
   -opentsdbHTTPListenAddr string

docs/victoriametrics/vmagent.md

@@ -182,15 +182,15 @@ among remote storage systems specified in `-remoteWrite.url`.
 > For example, if you set `-remoteWrite.url=srv+foo` and it's resolved to three addresses (`192.168.1.1`, `192.168.1.2`, `192.168.1.3`),
 > vmagent will only choose **one** randomly every time it (re-)creates the connection. In contrast, specifying the addresses manually (`-remoteWrite.url=192.168.1.1 -remoteWrite.url=192.168.1.2 -remoteWrite.url=192.168.1.3`) will shard samples across all three URLs.
 
-Sometimes, it may be necessary to use only a particular set of labels for sharding. For example, it may be necessary to route all the metrics with the same `instance` label
-to the same `-remoteWrite.url`. In this case, you can specify a comma-separated list of these labels in the `-remoteWrite.shardByURL.labels`
-command-line flag. For example, `-remoteWrite.shardByURL.labels=instance,__name__` would shard metrics with the same name and `instance`
-label to the same `-remoteWrite.url`.
+Use `-remoteWrite.shardByURL.labels` to route metrics among `-remoteWrite.url` based on their label values. 
+For example, `-remoteWrite.shardByURL.labels=instance,__name__` would shard metrics with the same name and `instance`
+label to the same `-remoteWrite.url`. This command-line flag allows specifying a comma-separated list of labels.
 
-Sometimes, it may be necessary to ignore some labels when sharding samples across multiple `-remoteWrite.url` backends.
-For example, if all the [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) with the same set of labels
-except for the labels `instance` and `pod` must be routed to the same backend. In this case the list of ignored labels must be passed to
-`-remoteWrite.shardByURL.ignoreLabels` command-line flag: `-remoteWrite.shardByURL.ignoreLabels=instance,pod`.
+Alternatively, you can use `-remoteWrite.shardByURL.ignoreLabels` to route metrics among `-remoteWrite.url` based on their label values, excluding the specified labels.
+For example, `-remoteWrite.shardByURL.ignoreLabels=pod` would shard metrics `metric{pod="foo"}` and `metric{pod="bar"}` to the same `-remoteWrite.url`
+by ignoring the `pod` label. This command-line flag allows specifying a comma-separated list of labels.
+
+> Command-line flags `-remoteWrite.shardByURL.labels` and `-remoteWrite.shardByURL.ignoreLabels` are mutually exclusive.
 
 See also [how to scrape a large number of targets](#scraping-big-number-of-targets).
 

docs/victoriametrics/vmagent_common_flags.md

@@ -169,7 +169,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
   -maxLabelsPerTimeseries int
      The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
@@ -184,9 +184,19 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
   -opentelemetry.convertMetricNamesToPrometheus
      Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
+  -opentelemetry.ignoreResourceAttributes array
+     Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
+  -opentelemetry.labelNameUnderscoreSanitization
+     Whether to enable prepending of 'key' to labels starting with '_' when -opentelemetry.usePrometheusNaming is enabled. Reserved labels starting with '__' are not modified. See https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/ (default true)
   -opentelemetry.maxRequestSize size
      The maximum size in bytes of a single OpenTelemetry request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
+  -opentelemetry.promoteAllResourceAttributes
+     Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
+  -opentelemetry.promoteResourceAttributes array
+     Promote specific list of resource attributes to labels.
+  -opentelemetry.promoteScopeMetadata
+     Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
   -opentelemetry.usePrometheusNaming
      Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
   -opentsdbHTTPListenAddr string

docs/victoriametrics/vmalert.md

@@ -21,23 +21,9 @@ Recording rules results are persisted via remote write protocols and require `-r
 `vmalert` is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
 implementation and aims to be compatible with its syntax.
 
-Configure `-vmalert.proxyURL` on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
-or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert)
-to proxy requests to `vmalert`. Proxying is needed for the following cases:
-
-* to proxy requests from [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/);
-* to access `vmalert`'s UI through [vmui](https://docs.victoriametrics.com/victoriametrics/#vmui).
-
-[VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_vmalert_intro)
-provides out-of-the-box alerting functionality based on `vmalert`. This service simplifies the setup
-and management of alerting and recording rules as well as the integration with Alertmanager. For more details,
-please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriametrics.com/victoriametrics-cloud/alertmanager-setup-for-deployment/).
-
 ## Features
 
-* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) and [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/);
-* Integration with [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/);
-* Integration with [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) which also uses [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victoriatraces/vmalert/);
+* Integration with VictoriaMetrics, VictoriaLogs, VictoriaTraces, Graphite and Prometheus compatible storages. See [Integrations](https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations) for details;
 * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
   support;
 * Integration with [Alertmanager](https://github.com/prometheus/alertmanager) starting from [Alertmanager v0.16.0-alpha](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0-alpha.0);
@@ -62,8 +48,8 @@ please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriam
 
 To start using `vmalert` you will need the following things:
 
-* list of rules - PromQL/MetricsQL expressions to execute;
-* datasource address - reachable endpoint with [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) support for running queries against;
+* list of rules - PromQL/MetricsQL/LogsQL/GraphiteQL expressions to execute;
+* datasource address - a storage that [vmalert integrates with](https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations) for executing queries;
 * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
   aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
   [config file](https://docs.victoriametrics.com/victoriametrics/vmalert/#notifier-configuration-file).
@@ -73,7 +59,7 @@ To start using `vmalert` you will need the following things:
 * remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
 
 You can use the existing [docker-compose environment](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#victoriametrics-single-server)
-as example. It already contains vmalert configured with list of alerting rules and integrated with Alert Manager and VictoriaMetrics.
+as an example. It already contains vmalert configured with the list of alerting rules and integrated with Alert Manager and VictoriaMetrics.
 
 Alternatively, build `vmalert` from sources:
 
@@ -87,7 +73,7 @@ Then run `vmalert`:
 
 ```sh
 ./bin/vmalert -rule=alert.rules \            # Path to the file with rules configuration. Supports wildcard and HTTP URL (S3/GCS are available in Enterprise).
-    -datasource.url=http://localhost:8428 \  # Prometheus HTTP API compatible datasource
+    -datasource.url=http://localhost:8428 \  # VictoriaMetrics URL to query for rules evaluation. See other available Integrations above.
     -notifier.url=http://localhost:9093 \    # AlertManager URL (required if alerting rules are used)
     -notifier.url=http://127.0.0.1:9093 \    # AlertManager replica URL
     -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
@@ -107,7 +93,7 @@ See also [stream aggregation](https://docs.victoriametrics.com/victoriametrics/s
 
 See the full list of configuration flags in [configuration](#configuration) section.
 
-If you run multiple `vmalert` services on the same datastore or AlertManager and need to distinguish the results or alerts, 
+If you run multiple `vmalert` services for the same datasource or AlertManager and need to distinguish the results or alerts, 
 specify different `-external.label` command-line flags to indicate which `vmalert` generated them. 
 If rule result metrics have label that conflict with `-external.label`, `vmalert` will automatically rename
 it with prefix `exported_`.
@@ -116,6 +102,7 @@ Configuration for [recording](https://prometheus.io/docs/prometheus/latest/confi
 and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very
 similar to Prometheus rules and configured using YAML. Configuration examples may be found
 in [testdata](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/config/testdata) folder.
+
 Every `rule` belongs to a `group` and every configuration file may contain arbitrary number of groups:
 
 ```yaml
@@ -123,13 +110,7 @@ groups:
   [ - <rule_group> ]
 ```
 
-> Explore how to integrate `vmalert` with [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/) in the following [guide](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
-
-> For users of [VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_vmalert_config),
-> many of the configuration steps (including highly available setup of `vmalert` for cluster deployments) are handled automatically.
-> Please, refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriametrics.com/victoriametrics-cloud/alertmanager-setup-for-deployment/) for more details.
-
-### Groups
+## Groups
 
 Each group has the following attributes:
 
@@ -230,9 +211,11 @@ rules:
 
 ### Rules
 
-Every rule contains `expr` field for [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/)
-or [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expression. `vmalert` will execute the configured
-expression and then act according to the Rule type.
+Every rule contains an `expr` field for the expression to evaluate against the configured datasource.
+Depending on `group.type` value or `-rule.defaultRuleType` cmd-line flag expression can be one of the following types:
+- `prometheus` (default) - [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) or [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expression.
+- `vlogs` - [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/vmalert/) expression.
+- `graphite` - [Graphite](https://graphite.readthedocs.io/en/stable/render_api.html) expression.
 
 There are two types of Rules:
 
@@ -244,8 +227,7 @@ There are two types of Rules:
   `-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
   expensive expressions and save their result as a new set of time series ([Prometheus recording rules docs](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)).
 
-`vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
-within one group.
+> `vmalert` forbids defining duplicates - rules with the same combination of name, expression and labels within one group.
 
 #### Alerting rules
 
@@ -256,8 +238,8 @@ The syntax for alerting rule is the following:
 alert: <string>
 
 # The expression to evaluate. The expression language depends on the type value.
-# By default, PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
-# must contain valid Graphite expression.
+# By default, PromQL/MetricsQL expression is used. Other available types are "graphite" and "vlogs".
+# See https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations
 expr: <string>
 
 # Alerts are considered firing once they have been returned for this long.
@@ -303,7 +285,44 @@ annotations:
   [ <labelname>: <tmpl_string> ]
 ```
 
-#### Templating
+#### Recording rules
+
+The syntax for recording rules is the following:
+
+```yaml
+# The name of the time series to output to. Must be a valid metric name.
+record: <string>
+
+# The expression to evaluate. The expression language depends on the type value.
+# By default, PromQL/MetricsQL expression is used. Other available types are "graphite" and "vlogs".
+# See https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations
+expr: <string>
+
+# Labels to add or overwrite labels from other external label sources, such as group labels, before storing the result.
+#
+# In case of conflicts, original labels are kept with prefix `exported_`.
+# As a special case, specifying a label with an empty string value removes the label from the result if it exists 
+# in the original query result; otherwise, it is ignored.
+#
+# Labels do not support templating in https://docs.victoriametrics.com/victoriametrics/vmalert/#templating due to cardinality concerns. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8171.
+labels:
+  [ <labelname>: <labelvalue> ]
+
+# Whether to print debug information into logs.
+# Information includes requests sent to the datasource.
+# information - it will be printed to logs.
+# Logs are printed with INFO level, so make sure that -loggerLevel=INFO to see the output.
+[ debug: <bool> | default = false ]
+
+# Defines the number of rule updates entries stored in memory
+# and available for view on rule Details page.
+# Overrides `rule.updateEntriesLimit` value for this specific rule.
+[ update_entries_limit: <integer> | default 0 ]
+```
+
+For recording rules to work `-remoteWrite.url` must be specified.
+
+## Templating
 
 It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations and labels(with limited support) to format data, iterate over
 or execute expressions.
@@ -313,7 +332,7 @@ The following variables are available in templating:
 |------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | $value or .Value                   | The current alert's value. Avoid using value in labels, it may cause unexpected issues.                                                               | `Number of connections is {{ $value }}`                                                                                                                                                |
 | $activeAt or .ActiveAt             | The moment of [time](https://pkg.go.dev/time) when alert became active (`pending` or `firing`).                                                       | `http://vm-grafana.com/<dashboard-id>?viewPanel=<panel-id>&from={{($activeAt.Add (parseDurationTime \"-1h\")).UnixMilli}}&to={{($activeAt.Add (parseDurationTime \"1h\")).UnixMilli}}` |
-| $labels or .Labels                 | The list of labels of the current alert. Use as `.Labels.<label_name>`.                                                                               | `Too high number of connections for {{ .Labels.instance }}`                                                                                                                            |
+| $labels or .Labels                 | The list of labels of the current alert. Use as `.Labels.<label_name>`. When the label name contains `.`, such as `cpu.mode`, use `{{ index .Labels "cpu.mode" }}` to access label value instead.                                                                      | `Too high number of connections for {{ .Labels.instance }}`                                                                                                                            |
 | $type or .Type                     | The rule type: "graphite", "prometheus" or "vlogs"                                                                                                    | `Link: /explore?left={"datasource":"{{ if eq .Type "vlogs" }}VictoriaLogs{{ else }}VictoriaMetrics{{ end }}","queries":[{"expr":"{{ .Expr }}"}]}`                                      |
 | $alertID or .AlertID               | The current alert's ID generated by vmalert.                                                                                                          | `Link: /vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}`                                                                                                                     |
 | $groupID or .GroupID               | The current alert's group ID generated by vmalert.                                                                                                    | `Link: /vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}`                                                                                                                     |
@@ -325,7 +344,7 @@ The following variables are available in templating:
 
 Additionally, `vmalert` provides some extra templating functions listed in [template functions](#template-functions) and [reusable templates](#reusable-templates).
 
-#### Template functions
+### Template functions
 
 `vmalert` provides the following template functions, which can be used during [templating](#templating):
 
@@ -349,7 +368,7 @@ Additionally, `vmalert` provides some extra templating functions listed in [temp
 * `parseDurationTime` - parses the input string into [time.Duration](https://pkg.go.dev/time#Duration).
 * `pathEscape` - escapes the input string, so it can be safely put inside path part of URL.
 * `pathPrefix` - returns the path part of the `-external.url` command-line flag.
-* `query` - executes the [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) query against `-datasource.url` and returns the query result.
+* `query` - executes query against `-datasource.url` and returns the query result.
   For example, `{{ query "sort_desc(process_resident_memory_bytes)" | first | value }}` executes the `sort_desc(process_resident_memory_bytes)`
   query at `-datasource.url` and returns the first result.
 * `queryEscape` - escapes the input string, so it can be safely put inside [query arg](https://en.wikipedia.org/wiki/Percent-encoding) part of URL.
@@ -369,7 +388,7 @@ Additionally, `vmalert` provides some extra templating functions listed in [temp
 * `toUpper` - converts all the chars in the input string to uppercase.
 * `value` - returns the numeric value from the input query result.
 
-#### Reusable templates
+### Reusable templates
 
 Like in Alertmanager you can define [reusable templates](https://prometheus.io/docs/prometheus/latest/configuration/template_examples/#defining-reusable-templates)
 to share same templates across annotations. Just define the templates in a file and
@@ -407,44 +426,8 @@ groups:
 The `-rule.templates` flag supports wildcards so multiple files with templates can be loaded.
 The content of `-rule.templates` can be also [hot reloaded](#hot-config-reload).
 
-#### Recording rules
 
-The syntax for recording rules is following:
-
-```yaml
-# The name of the time series to output to. Must be a valid metric name.
-record: <string>
-
-# The expression to evaluate. The expression language depends on the type value.
-# By default, MetricsQL expression is used. If group.type="graphite", then the expression
-# must contain valid Graphite expression.
-expr: <string>
-
-# Labels to add or overwrite labels from other external label sources, such as group labels, before storing the result.
-#
-# In case of conflicts, original labels are kept with prefix `exported_`.
-# As a special case, specifying a label with an empty string value removes the label from the result if it exists 
-# in the original query result; otherwise, it is ignored.
-#
-# Labels do not support templating in https://docs.victoriametrics.com/victoriametrics/vmalert/#templating due to cardinality concerns. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8171.
-labels:
-  [ <labelname>: <labelvalue> ]
-
-# Whether to print debug information into logs.
-# Information includes requests sent to the datasource.
-# information - it will be printed to logs.
-# Logs are printed with INFO level, so make sure that -loggerLevel=INFO to see the output.
-[ debug: <bool> | default = false ]
-
-# Defines the number of rule's updates entries stored in memory
-# and available for view on rule's Details page.
-# Overrides `rule.updateEntriesLimit` value for this specific rule.
-[ update_entries_limit: <integer> | default 0 ]
-```
-
-For recording rules to work `-remoteWrite.url` must be specified.
-
-### Alerts state on restarts
+## Alerts state on restarts
 
 `vmalert` holds alerts state in the memory. Restart of the `vmalert` process will reset the state of all active alerts
 in the memory. To prevent `vmalert` from losing the state on restarts configure it to persist the state
@@ -465,7 +448,7 @@ in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by
 or received state doesn't match current `vmalert` rules configuration. `vmalert` marks successfully restored rules
 with `restored` label in [web UI](#web).
 
-### Link to alert source
+## Link to alert source
 
 Alerting notifications sent by vmalert always contain a `source` link. By default, the link format
 is the following `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>`. On click, it opens
@@ -501,7 +484,9 @@ In addition to `source` link, some extra links could be added to alert's [annota
 field. See [how we use them](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/9751ea10983d42068487624849cac7ad6fd7e1d8/deployment/docker/rules/alerts-cluster.yml#L44)
 to link alerting rule and the corresponding panel on Grafana dashboard.
 
-### Multitenancy
+## Multitenancy
+
+> See how to use [multitenancy in rules for VictoriaLogs](https://docs.victoriametrics.com/victorialogs/vmalert/#how-to-use-multitenancy-in-rules).
 
 There are the following approaches exist for alerting and recording rules across
 [multiple tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy):
@@ -559,7 +544,7 @@ The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz`
 at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and in `*-enterprise`
 tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags) and [Quay](https://quay.io/repository/victoriametrics/vmalert?tab=tags).
 
-### Reading rules from object storage
+## Reading rules from object storage
 
 [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
 from object storage:
@@ -578,7 +563,7 @@ The following [command-line flags](#flags) can be used for fine-tuning access to
 * `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
 * `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
 
-### Topology examples
+## Topology examples
 
 The following sections are showing how `vmalert` may be used and configured
 for different scenarios.
@@ -589,7 +574,7 @@ Please note, not all flags in examples are required:
   you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
 * `-notifier.url` is optional and is needed only if you have alerting rules.
 
-#### Single-node VictoriaMetrics
+### Single-node VictoriaMetrics
 
 The simplest configuration where one single-node VM server is used for
 rules execution, storing recording rules results and alerts state.
@@ -607,7 +592,7 @@ rules execution, storing recording rules results and alerts state.
 ![vmalert single](vmalert_single.webp)
 {width="500"}
 
-#### Cluster VictoriaMetrics
+### Cluster VictoriaMetrics
 
 In [cluster mode](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/)
 VictoriaMetrics has separate components for writing and reading path:
@@ -630,7 +615,7 @@ Cluster mode could have multiple `vminsert` and `vmselect` components.
 In case when you want to spread the load on these components - add balancers before them and configure
 `vmalert` with balancer addresses. Please, see more about [VictoriaMetrics cluster architecture](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#architecture-overview).
 
-#### HA vmalert
+### HA vmalert
 
 For High Availability(HA) user can run multiple identically configured `vmalert` instances.
 It means all of them will execute the same rules, write state and results to
@@ -674,12 +659,12 @@ to ensure [high availability](https://github.com/prometheus/alertmanager#high-av
 This example uses single-node VM server for the sake of simplicity.
 Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed.
 
-#### Downsampling and aggregation via vmalert
+### Downsampling and aggregation via vmalert
 
 _Please note, [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) might be more efficient
 for cases when downsampling or aggregation need to be applied **before data gets into the TSDB.**_
 
-`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries
+`vmalert` can't modify existing data. But it can run arbitrary queries
 via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`.
 This ability allows to aggregate data. For example, the following rule will calculate the average value for
 metric `http_requests` on the `5m` interval:
@@ -732,7 +717,7 @@ Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only rec
 
 See also [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) and [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling).
 
-#### Multiple remote writes
+### Multiple remote writes
 
 For persisting recording or alerting rule results `vmalert` requires `-remoteWrite.url` to be set.
 But this flag supports only one destination. To persist rule results to multiple destinations
@@ -746,7 +731,72 @@ Using `vmagent` as a proxy provides additional benefits such as
 [data persisting when storage is unreachable](https://docs.victoriametrics.com/victoriametrics/vmagent/#replication-and-high-availability),
 or time series modification via [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/).
 
-### Web
+## Integrations
+
+vmalert can be integrated with different data sources for alerting and recording rules. But it deliberately allows
+configuring only one `datasource.url`. We recommend running separate instances of vmalert for each datasource type
+with the specified `-rule.defaultRuleType=<datasource_type>` command-line flag.
+
+###### VictoriaMetrics
+
+vmalert natively integrates with [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) for alerting and
+recording rules.
+
+###### VictoriaLogs
+
+vmalert integrates with [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and allows configuring alerting and recording rules using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/).
+Results of recording rules and alerting state should be persisted to the remote-write compatible storage, such as VictoriaMetrics.
+To enable VictoriaLogs compatibility set the `-rule.defaultRuleType=vlogs` command-line flag.
+
+See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/) for details.
+
+###### VictoriaTraces
+
+vmalert integrates with [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) in exactly the same way as
+with [VictoriaLogs](https://docs.victoriametrics.com/victoriametrics/vmalert/#victorialogs).
+
+###### Graphite
+
+vmalert integrates with [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) and allows configuring alerting and recording rules.
+During evaluation, vmalert will send requests to `<-datasource.url>/render?format=json`.
+To enable Graphite compatibility set the `-rule.defaultRuleType=graphite` command-line flag.
+
+Since VictoriaMetrics supports both Graphite and Prometheus APIs, it is possible to mix Graphite and VictoriaMetrics rules.
+On the group level, set the `type` field to specify to which datasource type it should belong: `prometheus` (MetricsQL) or `graphite` (GraphiteQL).
+When using vmalert with both `graphite` and `prometheus` rules configured against the cluster version of VictoriaMetrics, don't forget
+to set the `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.
+
+###### Prometheus
+
+vmalert uses [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) for querying
+and [Prometheus Remote Write v1 protocol](https://prometheus.io/docs/specs/prw/remote_write_spec/) for persisting
+recording rules results and alerting state. Hence, it can be integrated with any Prometheus-compatible storage
+that supports these protocols.
+
+###### Grafana
+
+To proxy requests from [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/) configure `-vmalert.proxyURL`
+on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
+or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert).
+
+###### vmui
+
+To access rules UI through [vmui](https://docs.victoriametrics.com/victoriametrics/#vmui) configure `-vmalert.proxyURL`
+on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
+or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert).
+
+###### vmanomaly
+
+See how to integrate vmalert with [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/)
+in the following [guide](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
+
+###### VictoriaMetrics Cloud
+
+For users of [VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_vmalert_config),
+many of the configuration steps (including highly available setup of `vmalert` for cluster deployments) are handled automatically.
+Please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriametrics.com/victoriametrics-cloud/alertmanager-setup-for-deployment/) for more details.
+
+## Web
 
 `vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
 
@@ -771,28 +821,11 @@ This may be used for better integration with Grafana unified alerting system. Se
 * [How to query vmalert from single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
 * [How to query vmalert from VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert)
 
-## Graphite
-
-vmalert sends requests to `<-datasource.url>/render?format=json` during evaluation of alerting and recording rules
-if the corresponding group or rule contains `type: "graphite"` config option. It is expected that the `<-datasource.url>/render`
-implements [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) for `format=json`.
-When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VM do not forget
-to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.
-
-## VictoriaLogs
-
-vmalert supports [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) as a datasource for writing alerting and recording rules using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/) for details.
-
-## VictoriaTraces
-
-vmalert supports [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) as a (`vlogs`) datasource for writing alerting and recording rules using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victoriatraces/vmalert/) for details.
-
 ## Rules backfilling
 
 vmalert supports alerting and recording rules backfilling (aka `replay`). In replay mode vmalert
 can read the same rules configuration as normal, evaluate them on the given time range and backfill
-results via remote write to the configured storage. vmalert supports any PromQL/MetricsQL compatible
-data source for backfilling.
+results via remote write to the configured storage. vmalert supports only the `prometheus` datasource type for backfilling.
 
 Please note, that response caching may lead to unexpected results during and after backfilling process.
 In order to avoid this you need to reset cache contents or disable caching when using backfilling
@@ -852,13 +885,9 @@ vmalert respects `evaluationInterval` value set by flag or per-group during the
 vmalert automatically disables caching on VictoriaMetrics side by sending `nocache=1` param. It allows
 to prevent cache pollution and unwanted time range boundaries adjustment during backfilling.
 
-#### Recording rules
+Results of recording rules `replay` should match the results of normal rules evaluation.
 
-The result of recording rules `replay` should match with results of normal rules evaluation.
-
-#### Alerting rules
-
-The result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
+Results of alerting rules `replay` are the time series reflecting the [state of the alert](#alerts-state-on-restarts).
 To see if `replayed` alert has fired in the past use the following PromQL/MetricsQL expression:
 
 ```

docs/victoriametrics/vmalert_common_flags.md

@@ -166,7 +166,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)

docs/victoriametrics/vmauth.md

@@ -554,15 +554,17 @@ At request time each placeholder is replaced with the corresponding value from t
 
 The following placeholders are supported:
 
-| Placeholder                   | JWT claim field                                                             |
-|-------------------------------|-----------------------------------------------------------------------------|
+| Placeholder                   | JWT claim field                                         |
+|-------------------------------|---------------------------------------------------------|
+| `{{.MetricsAccountID}}`       | `metrics_account_id` int                                |
+| `{{.MetricsProjectID}}`       | `metrics_project_id` int                                |
 | `{{.MetricsTenant}}` -> `0:0` | `metrics_account_id` int, <br/>`metrics_project_id` int |
-| `{{.MetricsExtraLabels}}`     | `metrics_extra_labels` string array                               |
-| `{{.MetricsExtraFilters}}`    | `metrics_extra_filters` string array                              |
-| `{{.LogsAccountID}}`          | `logs_account_id` int                                             |
-| `{{.LogsProjectID}}`          | `logs_project_id` int                                             |
-| `{{.LogsExtraFilters}}`       | `logs_extra_filters` string array                                 |
-| `{{.LogsExtraStreamFilters}}` | `logs_extra_stream_filters` string array                          |
+| `{{.MetricsExtraLabels}}`     | `metrics_extra_labels` string array                     |
+| `{{.MetricsExtraFilters}}`    | `metrics_extra_filters` string array                    |
+| `{{.LogsAccountID}}`          | `logs_account_id` int                                   |
+| `{{.LogsProjectID}}`          | `logs_project_id` int                                   |
+| `{{.LogsExtraFilters}}`       | `logs_extra_filters` string array                       |
+| `{{.LogsExtraStreamFilters}}` | `logs_extra_stream_filters` string array                |
 
 Placeholders are supported in the following locations:
 

docs/victoriametrics/vmauth_common_flags.md

@@ -135,7 +135,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
      The maximum request body size to buffer in memory for potential retries at other backends. Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables retries. See also -requestBufferSize
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16384)
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/vmutils-darwin-arm64-v1.143.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.144.0/vmutils-darwin-arm64-v1.144.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.143.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.144.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

docs/victoriametrics/vminsert_common_flags.md

@@ -169,7 +169,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -maxLabelsPerTimeseries int
      The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason="too_many_labels"} metric at /metrics page is incremented (default 40)
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
@@ -184,9 +184,19 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
   -opentelemetry.convertMetricNamesToPrometheus
      Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
+  -opentelemetry.ignoreResourceAttributes array
+     Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
+  -opentelemetry.labelNameUnderscoreSanitization
+     Whether to enable prepending of 'key' to labels starting with '_' when -opentelemetry.usePrometheusNaming is enabled. Reserved labels starting with '__' are not modified. See https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/ (default true)
   -opentelemetry.maxRequestSize size
      The maximum size in bytes of a single OpenTelemetry request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
+  -opentelemetry.promoteAllResourceAttributes
+     Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
+  -opentelemetry.promoteResourceAttributes array
+     Promote specific list of resource attributes to labels.
+  -opentelemetry.promoteScopeMetadata
+     Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
   -opentelemetry.usePrometheusNaming
      Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
   -opentsdbHTTPListenAddr string

docs/victoriametrics/vminsert_enterprise_flags.md

@@ -18,6 +18,20 @@ sitemap:
      Whether to skip verification of TLS certificates provided by -storageNode nodes if -cluster.tls flag is set. Note that disabled TLS certificate verification breaks security. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -cluster.tlsKeyFile string
      Path to client-side TLS key file to use when connecting to -storageNode if -cluster.tls flag is set. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+  -clusternative.tls
+     Whether to use TLS when accepting connections at -clusternativeListenAddr. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+  -clusternative.tlsCAFile string
+     Path to TLS CA file to use for verifying certificates provided by vminsert, which connects at -clusternativeListenAddr if -clusternative.tls flag is set. By default system CA is used. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+  -clusternative.tlsCertFile string
+     Path to server-side TLS certificate file to use when accepting connections at -clusternativeListenAddr if -clusternative.tls flag is set. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+  -clusternative.tlsCipherSuites array
+     Optional list of TLS cipher suites used for connections at -clusternativeListenAddr if -clusternative.tls flag is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+     Supports an array of values separated by comma or specified via multiple flags.
+     Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
+  -clusternative.tlsInsecureSkipVerify
+     Whether to skip verification of TLS certificates provided by vminsert, which connects to -clusternativeListenAddr if -clusternative.tls flag is set. Note that disabled TLS certificate verification breaks security. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
+  -clusternative.tlsKeyFile string
+     Path to server-side TLS key file to use when accepting connections at -clusternativeListenAddr if -clusternative.tls flag is set. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -eula
      Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
   -license string

docs/victoriametrics/vmselect_common_flags.md

@@ -126,7 +126,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)

docs/victoriametrics/vmstorage_common_flags.md

@@ -22,7 +22,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -dedup.minScrapeInterval duration
      Leave only the last sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication for details
   -denyQueriesOutsideRetention
-     Whether to deny queries outside of the configured -retentionPeriod. When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
+     Whether to deny queries outside the configured -retentionPeriod and -futureRetention. When set, then /api/v1/query_range will return an error for queries with 'from' value outside -retentionPeriod or 'to' value beyond -futureRetention. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
   -denyQueryTracing
      Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing
   -disablePerDayIndex
@@ -131,7 +131,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -maxConcurrentInserts int
      The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 2*cgroup.AvailableCPUs())
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
      Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)