lib/timerpool: remove check of Reset output value

This reverts commit ccf97a4143.

reason for revert: this change may break tests, which expect that ServesMetrics.GetMetric() fails
when the given metric doesn't exist in the output.

It is better to add 'TryGetMetric() (float64, bool)' function, which would return '(0, false)'
when the given metric doesn't exist, so the caller could decide what to do next.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9773

.github/copilot-instructions.md

@@ -0,0 +1,23 @@
+# Project Overview
+
+VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
+
+## Folder Structure
+
+- `/app`: Contains the compilable binaries.
+- `/lib`: Contains the golang reusable libraries
+- `/docs/victoriametrics`: Contains documentation for the project.
+- `/apptest/tests`: Contains integration tests.
+
+## Libraries and Frameworks
+
+- Backend: Golang, no framework. Use third-party libraries sparingly.
+- Frontend: React.
+
+## Code review guidelines
+
+Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
+Verify the entry is under the ## tip section and matches the structure and style of existing entries.
+Chore-only changes may be omitted from the changelog.
+
+

.github/workflows/build.yml

@@ -31,43 +31,48 @@ concurrency:
 
 jobs:
   build:
-    name: Build
+    name: ${{ matrix.os }}-${{ matrix.arch }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: linux
+            arch: 386
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+          - os: linux
+            arch: arm
+          - os: linux
+            arch: ppc64le
+          - os: darwin
+            arch: amd64
+          - os: darwin
+            arch: arm64
+          - os: freebsd
+            arch: amd64
+          - os: openbsd
+            arch: amd64
+          - os: windows
+            arch: amd64
     steps:
-      - name: Free space
-        run: |
-          # cleanup up space to free additional ~20GiB of memory
-          # which are lacking for multiplaform images build
-          formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); }
-          getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); }
-          BEFORE=$(getAvailableSpace)
-          sudo rm -rf /usr/local/lib/android || true
-          sudo rm -rf /usr/share/dotnet || true
-          sudo rm -rf /opt/ghc || true
-          sudo rm -rf /usr/local/.ghcup || true
-          AFTER=$(getAvailableSpace)
-          SAVED=$((AFTER-BEFORE))
-          echo "Saved $(formatByteCount $SAVED)"
-
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
           go-version: stable
-          cache: false
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
+      - name: Build victoria-metrics for ${{ matrix.os }}-${{ matrix.arch }}
+        run: make victoria-metrics-${{ matrix.os }}-${{ matrix.arch }}
 
-      - name: Run crossbuild
-        run: make crossbuild
+      - name: Build vmutils for ${{ matrix.os }}-${{ matrix.arch }}
+        run: make vmutils-${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/check-commit-signed.yml

@@ -0,0 +1,37 @@
+name: check-commit-signed
+
+on:
+  pull_request:
+
+jobs:
+  check-commit-signed:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0  # we need full history for commit verification
+
+      - name: Check commit signatures
+        run: |
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            echo "Not a PR event, skipping signature check"
+            exit 0
+          fi
+          
+          RANGE="${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
+          echo "Checking commits in PR range: $RANGE"
+          
+          if [ -z "$(git rev-list $RANGE)" ]; then
+            echo "No new commits in this PR, skipping signature check"
+            exit 0
+          fi
+          
+          unsigned=$(git log --pretty="%H %G?" $RANGE | grep -vE " (G|E)$" || true)
+          if [ -n "$unsigned" ]; then
+            echo "Found unsigned commits:"
+            echo "$unsigned"
+            exit 1
+          fi
+          
+          echo "All commits in PR are signed (G or E)"

.github/workflows/check-licenses.yml

@@ -19,7 +19,7 @@ jobs:
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: stable
           cache: false

.github/workflows/codeql-analysis-go.yml

@@ -29,11 +29,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Go
         id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           cache: false
           go-version: stable

.github/workflows/docs.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/main.yml

@@ -1,126 +0,0 @@
-name: main
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-all-
-
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
-
-  test:
-    name: test
-    needs: lint
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        scenario:
-          - 'test-full'
-          - 'test-full-386'
-          - 'test-pure'
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v5
-        with:
-          files: ./coverage.txt
-
-  integration-test:
-    name: integration-test
-    needs: [lint, test]
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: Run integration tests
-        run: make integration-test

.github/workflows/test.yml

@@ -0,0 +1,113 @@
+name: test
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - 'go.*'
+      - '.github/workflows/main.yml'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - 'go.*'
+      - '.github/workflows/main.yml'
+
+permissions:
+  contents: read
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
+          go-version: stable
+
+
+      - name: Cache golangci-lint
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/golangci-lint
+            ~/go/bin
+          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
+
+      - name: Run check-all
+        run: |
+          make check-all
+          git diff --exit-code
+
+  unit:
+    name: unit
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        scenario:
+          - 'test-full'
+          - 'test-full-386'
+          - 'test-pure'
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
+          go-version: stable
+
+      - name: Run tests
+        run: GOGC=10 make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.txt
+
+  integration:
+    name: integration
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
+          go-version: stable
+
+      - name: Run integration tests
+        run: make integration-test

.github/workflows/vmui.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Node
         uses: actions/setup-node@v4

Makefile

@@ -12,11 +12,12 @@ PKG_TAG := $(BUILDINFO_TAG)
 endif
 
 EXTRA_DOCKER_TAG_SUFFIX ?=
+EXTRA_GO_BUILD_TAGS ?=
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.2.1
+GOLANGCI_LINT_VERSION := 2.4.0
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -169,9 +170,11 @@ vmutils-windows-amd64: \
 	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 crossbuild:
 	$(MAKE_PARALLEL) victoria-metrics-crossbuild vmutils-crossbuild
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 victoria-metrics-crossbuild: \
 	victoria-metrics-linux-386 \
 	victoria-metrics-linux-amd64 \
@@ -184,6 +187,7 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-openbsd-amd64 \
 	victoria-metrics-windows-amd64
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 vmutils-crossbuild: \
 	vmutils-linux-386 \
 	vmutils-linux-amd64 \
@@ -467,16 +471,16 @@ vendor-update:
 	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-goos-goarch:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-windows-goarch:
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc

SECURITY.md

@@ -4,12 +4,11 @@
 
 The following versions of VictoriaMetrics receive regular security fixes:
 
-| Version | Supported          |
-|---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| other releases  | :x:                |
+| Version                                                                        | Supported          |
+|--------------------------------------------------------------------------------|--------------------|
+| [Latest release](https://docs.victoriametrics.com/victoriametrics/changelog/)  | :white_check_mark: |
+| [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| other releases                                                                 | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 

app/victoria-metrics/main.go

@@ -169,7 +169,7 @@ func usage() {
 	const s = `
 victoria-metrics is a time series database and monitoring solution.
 
-See the docs at https://docs.victoriametrics.com/
+See the docs at https://docs.victoriametrics.com/victoriametrics/
 `
 	flagutil.Usage(s)
 }

app/victoria-metrics/self_scraper.go

@@ -57,7 +57,8 @@ func selfScraper(scrapeInterval time.Duration) {
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
-		rows.Unmarshal(s)
+		// VictoriaMetrics components don't expose metadata yet, only need to parse samples
+		rows.UnmarshalWithErrLogger(s, nil)
 		mrs = mrs[:0]
 		for i := range rows.Rows {
 			r := &rows.Rows[i]

app/vmagent/opentelemetry/request_handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
@@ -16,9 +17,11 @@ import (
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
+	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
+	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="opentelemetry"}`)
+	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
+	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="opentelemetry"}`)
+	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
 // InsertHandler processes opentelemetry metrics.
@@ -36,12 +39,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries) error {
-		return insertRows(at, tss, extraLabels)
+	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
+		return insertRows(at, tss, mms, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, tss []prompb.TimeSeries, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -63,14 +66,33 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, extraLabels []prompb.La
 		})
 	}
 	ctx.WriteRequest.Timeseries = tssDst
+
+	var metadataTotal int
+	if prommetadata.IsEnabled() {
+		var accountID, projectID uint32
+		if at != nil {
+			accountID = at.AccountID
+			projectID = at.ProjectID
+			for i := range mms {
+				mm := &mms[i]
+				mm.AccountID = accountID
+				mm.ProjectID = projectID
+			}
+		}
+		ctx.WriteRequest.Metadata = mms
+		metadataTotal = len(mms)
+	}
+
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
 	rowsInserted.Add(rowsTotal)
+	metadataInserted.Add(metadataTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
+		metadataTenantInserted.Get(at).Add(metadataTotal)
 	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
@@ -16,9 +17,12 @@ import (
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="prometheus"}`)
+	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
+	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="prometheus"}`)
+
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
@@ -32,18 +36,19 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, func(rows []prometheus.Row) error {
-		return insertRows(at, rows, extraLabels)
+	return stream.Parse(req.Body, defaultTimestamp, encoding, true, prommetadata.IsEnabled(), func(rows []prometheus.Row, mms []prometheus.Metadata) error {
+		return insertRows(at, rows, mms, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)
 	})
 }
 
-func insertRows(at *auth.Token, rows []prometheus.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []prometheus.Row, mms []prometheus.Metadata, extraLabels []prompb.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
 	tssDst := ctx.WriteRequest.Timeseries[:0]
+	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range rows {
@@ -70,15 +75,35 @@ func insertRows(at *auth.Token, rows []prometheus.Row, extraLabels []prompb.Labe
 			Samples: samples[len(samples)-1:],
 		})
 	}
+	var accountID, projectID uint32
+	if at != nil {
+		accountID = at.AccountID
+		projectID = at.ProjectID
+	}
+	for i := range mms {
+		mm := &mms[i]
+		mmsDst = append(mmsDst, prompb.MetricMetadata{
+			MetricFamilyName: mm.Metric,
+			Help:             mm.Help,
+			Type:             mm.Type,
+			// there is no unit in Prometheus exposition formats
+
+			AccountID: accountID,
+			ProjectID: projectID,
+		})
+	}
 	ctx.WriteRequest.Timeseries = tssDst
+	ctx.WriteRequest.Metadata = mmsDst
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
 	rowsInserted.Add(len(rows))
+	metadataInserted.Add(len(mms))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))
+		metadataTenantInserted.Get(at).Add(len(mms))
 	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/promremotewrite/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -14,9 +15,11 @@ import (
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="promremotewrite"}`)
+	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
+	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="promremotewrite"}`)
+	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
@@ -26,17 +29,18 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
-		return insertRows(at, tss, extraLabels)
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
+		return insertRows(at, tss, mms, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
+	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range timeseries {
@@ -65,6 +69,30 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 		})
 	}
 	ctx.WriteRequest.Timeseries = tssDst
+
+	var metadataTotal int
+	if prommetadata.IsEnabled() {
+		var accountID, projectID uint32
+		if at != nil {
+			accountID = at.AccountID
+			projectID = at.ProjectID
+		}
+		for i := range mms {
+			mm := &mms[i]
+			mmsDst = append(mmsDst, prompb.MetricMetadata{
+				MetricFamilyName: mm.MetricFamilyName,
+				Help:             mm.Help,
+				Type:             mm.Type,
+				Unit:             mm.Unit,
+
+				AccountID: accountID,
+				ProjectID: projectID,
+			})
+		}
+		ctx.WriteRequest.Metadata = mmsDst
+		metadataTotal = len(mms)
+	}
+
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
@@ -73,7 +101,9 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
+		metadataTenantInserted.Get(at).Add(metadataTotal)
 	}
+	metadataInserted.Add(metadataTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmagent/remotewrite/client.go

@@ -463,12 +463,6 @@ again:
 		// - Real-world implementations of v1 use both 400 and 415 status codes.
 		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
 	case 415, 400:
-		if c.canDowngradeVMProto.Swap(false) {
-			logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
-				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-			c.useVMProto.Store(false)
-		}
-
 		if encoding.IsZstd(block) {
 			logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
 				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)

app/vmagent/remotewrite/client_test.go

@@ -93,10 +93,7 @@ func TestParseRetryAfterHeader(t *testing.T) {
 
 // helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
 func helper(d time.Duration) time.Duration {
-	dv := d / 10
-	if dv > 10*time.Second {
-		dv = 10 * time.Second
-	}
+	dv := min(d/10, 10*time.Second)
 
 	return d + dv
 }

app/vmagent/remotewrite/pendingseries.go

@@ -24,9 +24,10 @@ import (
 
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
-		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
+		"This option takes effect only when less than -remoteWrite.maxRowsPerBlock data points per -remoteWrite.flushInterval are pushed to -remoteWrite.url")
 	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
+	maxMetadataPerBlock  = flag.Int("remoteWrite.maxMetadataPerBlock", 5000, "The maximum number of metadata to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
 		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
@@ -60,9 +61,16 @@ func (ps *pendingSeries) MustStop() {
 	ps.periodicFlusherWG.Wait()
 }
 
-func (ps *pendingSeries) TryPush(tss []prompb.TimeSeries) bool {
+func (ps *pendingSeries) TryPushTimeSeries(tss []prompb.TimeSeries) bool {
 	ps.mu.Lock()
-	ok := ps.wr.tryPush(tss)
+	ok := ps.wr.tryPushTimeSeries(tss)
+	ps.mu.Unlock()
+	return ok
+}
+
+func (ps *pendingSeries) TryPushMetadata(mms []prompb.MetricMetadata) bool {
+	ps.mu.Lock()
+	ok := ps.wr.tryPushMetadata(mms)
 	ps.mu.Unlock()
 	return ok
 }
@@ -111,26 +119,34 @@ type writeRequest struct {
 	wr prompb.WriteRequest
 
 	tss     []prompb.TimeSeries
+	mms     []prompb.MetricMetadata
 	labels  []prompb.Label
 	samples []prompb.Sample
 
 	// buf holds labels data
 	buf []byte
+	// metadatabuf holds metadata data
+	metadatabuf []byte
 }
 
 func (wr *writeRequest) reset() {
 	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are reused.
 
 	wr.wr.Timeseries = nil
+	wr.wr.Metadata = nil
 
 	clear(wr.tss)
 	wr.tss = wr.tss[:0]
 
+	clear(wr.mms)
+	wr.mms = wr.mms[:0]
+
 	promrelabel.CleanLabels(wr.labels)
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
 	wr.buf = wr.buf[:0]
+	wr.metadatabuf = wr.metadatabuf[:0]
 }
 
 // mustFlushOnStop force pushes wr data into wr.fq
@@ -138,6 +154,7 @@ func (wr *writeRequest) reset() {
 // This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
 func (wr *writeRequest) mustFlushOnStop() {
 	wr.wr.Timeseries = wr.tss
+	wr.wr.Metadata = wr.mms
 	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
 		logger.Panicf("BUG: final flush must always return true")
 	}
@@ -151,6 +168,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
+	wr.wr.Metadata = wr.mms
 	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
 	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
 		return false
@@ -174,7 +192,49 @@ func adjustSampleValues(samples []prompb.Sample, significantFigures, roundDigits
 	}
 }
 
-func (wr *writeRequest) tryPush(src []prompb.TimeSeries) bool {
+func (wr *writeRequest) tryPushMetadata(mms []prompb.MetricMetadata) bool {
+	mmdDst := wr.mms
+	maxMetadataPerBlock := *maxMetadataPerBlock
+	for i := range mms {
+		if len(wr.mms) >= maxMetadataPerBlock {
+			if !wr.tryFlush() {
+				return false
+			}
+			mmdDst = wr.mms
+		}
+		mmSrc := &mms[i]
+		mmdDst = append(mmdDst, prompb.MetricMetadata{})
+		wr.copyMetadata(&mmdDst[len(mmdDst)-1], mmSrc)
+	}
+	wr.mms = mmdDst
+	return true
+}
+
+func (wr *writeRequest) copyMetadata(dst, src *prompb.MetricMetadata) {
+	// Direct copy for non-string fields, which are safe by value.
+	dst.Type = src.Type
+	dst.Unit = src.Unit
+
+	// Pre-allocate memory for all string fields.
+	neededBufLen := len(src.MetricFamilyName) + len(src.Help)
+	bufLen := len(wr.metadatabuf)
+	wr.metadatabuf = slicesutil.SetLength(wr.metadatabuf, bufLen+neededBufLen)
+	buf := wr.metadatabuf[:bufLen]
+
+	// Copy MetricFamilyName
+	bufLen = len(buf)
+	buf = append(buf, src.MetricFamilyName...)
+	dst.MetricFamilyName = bytesutil.ToUnsafeString(buf[bufLen:])
+
+	// Copy Help
+	bufLen = len(buf)
+	buf = append(buf, src.Help...)
+	dst.Help = bytesutil.ToUnsafeString(buf[bufLen:])
+
+	wr.metadatabuf = buf
+}
+
+func (wr *writeRequest) tryPushTimeSeries(src []prompb.TimeSeries) bool {
 	tssDst := wr.tss
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
@@ -241,7 +301,7 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompb.TimeSeries) {
 var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
 
 func tryPushWriteRequest(wr *prompb.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
-	if len(wr.Timeseries) == 0 {
+	if wr.IsEmpty() {
 		// Nothing to push
 		return true
 	}
@@ -267,6 +327,7 @@ func tryPushWriteRequest(wr *prompb.WriteRequest, tryPushBlock func(block []byte
 			compressBufPool.Put(zb)
 			if ok {
 				blockSizeRows.Update(float64(len(wr.Timeseries)))
+				blockMetadataRows.Update(float64(len(wr.Metadata)))
 				blockSizeBytes.Update(float64(zbLen))
 			}
 			return ok
@@ -278,47 +339,86 @@ func tryPushWriteRequest(wr *prompb.WriteRequest, tryPushBlock func(block []byte
 		<-marshalConcurrencyCh
 	}
 
-	// Too big block. Recursively split it into smaller parts if possible.
-	if len(wr.Timeseries) == 1 {
-		// A single time series left. Recursively split its samples into smaller parts if possible.
+	// Split timeseries or metadata into two smaller blocks
+	switch len(wr.Timeseries) {
+	case 0:
+		if len(wr.Metadata) == 1 {
+			logger.Warnf("dropping a metadata exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
+			return true
+		}
+		metadata := wr.Metadata
+		n := len(metadata) / 2
+		wr.Metadata = metadata[:n]
+		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+			wr.Metadata = metadata
+			return false
+		}
+		wr.Metadata = metadata[n:]
+		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+			wr.Metadata = metadata
+			return false
+		}
+		wr.Metadata = metadata
+		return true
+
+	case 1:
+		// A single time series left. Recursively split its samples and metadata into smaller parts if possible.
 		samples := wr.Timeseries[0].Samples
-		if len(samples) == 1 {
-			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
+		metaData := wr.Metadata
+		if len(samples) == 1 && len(metaData) <= 1 {
+			logger.Warnf("dropping a sample for metric and %d metadata which are exceeding -remoteWrite.maxBlockSize=%d bytes", len(metaData), maxUnpackedBlockSize.N)
 			return true
 		}
 		n := len(samples) / 2
+		m := len(metaData) / 2
 		wr.Timeseries[0].Samples = samples[:n]
+		wr.Metadata = metaData[:m]
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
+			wr.Metadata = metaData
 			return false
 		}
 		wr.Timeseries[0].Samples = samples[n:]
+		wr.Metadata = metaData[m:]
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
+			wr.Metadata = metaData
 			return false
 		}
 		wr.Timeseries[0].Samples = samples
+		wr.Metadata = metaData
+		return true
+
+	default:
+		// Split both timeseries and metadata.
+		timeseries := wr.Timeseries
+		metaData := wr.Metadata
+		n := len(timeseries) / 2
+		m := len(metaData) / 2
+		wr.Timeseries = timeseries[:n]
+		wr.Metadata = metaData[:m]
+		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+			wr.Timeseries = timeseries
+			wr.Metadata = metaData
+			return false
+		}
+		wr.Timeseries = timeseries[n:]
+		wr.Metadata = metaData[m:]
+		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+			wr.Timeseries = timeseries
+			wr.Metadata = metaData
+			return false
+		}
+		wr.Timeseries = timeseries
+		wr.Metadata = metaData
 		return true
 	}
-	timeseries := wr.Timeseries
-	n := len(timeseries) / 2
-	wr.Timeseries = timeseries[:n]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
-	wr.Timeseries = timeseries[n:]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
-	wr.Timeseries = timeseries
-	return true
 }
 
 var (
-	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
-	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
+	blockSizeBytes    = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows     = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
+	blockMetadataRows = metrics.NewHistogram(`vmagent_remotewrite_block_metadata_rows`)
 )
 
 var (

app/vmagent/remotewrite/remotewrite.go

@@ -209,7 +209,7 @@ func Init() {
 	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
 	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
 	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
-	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
+	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(*remoteWriteURLs) > 1
 
 	dropDanglingQueues()
 
@@ -388,13 +388,7 @@ func TryPush(at *auth.Token, wr *prompb.WriteRequest) bool {
 
 func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure bool) bool {
 	tss := wr.Timeseries
-
-	var tenantRctx *relabelCtx
-	if at != nil {
-		// Convert at to (vm_account_id, vm_project_id) labels.
-		tenantRctx = getRelabelCtx()
-		defer putRelabelCtx(tenantRctx)
-	}
+	mms := wr.Metadata
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
@@ -411,6 +405,23 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 		return true
 	}
 
+	// Push metadata separately from time series, since it doesn't need sharding,
+	// relabeling, stream aggregation, deduplication, etc.
+	if !tryPushMetadataToRemoteStorages(rwctxs, mms, forceDropSamplesOnFailure) {
+		return false
+	}
+
+	if len(tss) == 0 {
+		return true
+	}
+
+	var tenantRctx *relabelCtx
+	if at != nil {
+		// Convert at to (vm_account_id, vm_project_id) labels.
+		tenantRctx = getRelabelCtx()
+		defer putRelabelCtx(tenantRctx)
+	}
+
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
@@ -481,7 +492,7 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
-		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushTimeSeriesToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
@@ -520,18 +531,49 @@ func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailu
 	return rwctxs, true
 }
 
-func pushToRemoteStoragesTrackDropped(tss []prompb.TimeSeries) {
+func pushTimeSeriesToRemoteStoragesTrackDropped(tss []prompb.TimeSeries) {
 	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
 	if len(rwctxs) == 0 {
 		return
 	}
 
-	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
-		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
+	if !tryPushTimeSeriesToRemoteStorages(rwctxs, tss, true) {
+		logger.Panicf("BUG: tryPushTimeSeriesToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
 	}
 }
 
-func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.MetricMetadata, forceDropSamplesOnFailure bool) bool {
+	if len(mms) == 0 {
+		// Nothing to push
+		return true
+	}
+	// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
+	// Since metadata is usually small and there is no guarantee that metadata can be sent to
+	// the same remote storage with the corresponding metrics.
+	//
+	// Push metadata to remote storage systems in parallel to reduce
+	// the time needed for sending the data to multiple remote storage systems.
+	var wg sync.WaitGroup
+	wg.Add(len(rwctxs))
+	var anyPushFailed atomic.Bool
+	for _, rwctx := range rwctxs {
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
+			if !rwctx.tryPushMetadataInternal(mms) {
+				rwctx.pushFailures.Inc()
+				if forceDropSamplesOnFailure {
+					rwctx.metadataDroppedOnPushFailure.Add(len(mms))
+					return
+				}
+				anyPushFailed.Store(true)
+			}
+		}(rwctx)
+	}
+	wg.Wait()
+	return !anyPushFailed.Load()
+}
+
+func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -539,7 +581,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.Ti
 
 	if len(rwctxs) == 1 {
 		// Fast path - just push data to the configured single remote storage
-		return rwctxs[0].TryPush(tssBlock, forceDropSamplesOnFailure)
+		return rwctxs[0].TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure)
 	}
 
 	// We need to push tssBlock to multiple remote storages.
@@ -550,11 +592,11 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.Ti
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingTimeSeriesAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
-	// Push tssBlock to remote storage systems in parallel in order to reduce
+	// Push tssBlock to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
 	wg.Add(len(rwctxs))
@@ -562,7 +604,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.Ti
 	for _, rwctx := range rwctxs {
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
-			if !rwctx.TryPush(tssBlock, forceDropSamplesOnFailure) {
+			if !rwctx.TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
 		}(rwctx)
@@ -571,7 +613,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.Ti
 	return !anyPushFailed.Load()
 }
 
-func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -590,7 +632,7 @@ func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []pr
 		wg.Add(1)
 		go func(rwctx *remoteWriteCtx, tss []prompb.TimeSeries) {
 			defer wg.Done()
-			if !rwctx.TryPush(tss, forceDropSamplesOnFailure) {
+			if !rwctx.TryPushTimeSeries(tss, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
 		}(rwctx, shard)
@@ -797,8 +839,9 @@ type remoteWriteCtx struct {
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
 
-	pushFailures             *metrics.Counter
-	rowsDroppedOnPushFailure *metrics.Counter
+	pushFailures                 *metrics.Counter
+	metadataDroppedOnPushFailure *metrics.Counter
+	rowsDroppedOnPushFailure     *metrics.Counter
 }
 
 func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
@@ -862,8 +905,9 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		pushFailures:                 metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		metadataDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_metadata_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure:     metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
 	rwctx.initStreamAggrConfig()
 
@@ -897,10 +941,10 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.rowsDroppedByRelabel = nil
 }
 
-// TryPush sends tss series to the configured remote write endpoint
+// TryPushTimeSeries sends tss series to the configured remote write endpoint
 //
-// TryPush doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
-func (rwctx *remoteWriteCtx) TryPush(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
+// TryPushTimeSeries doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
+func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	var rctx *relabelCtx
 	var v *[]prompb.TimeSeries
 	defer func() {
@@ -953,7 +997,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompb.TimeSeries, forceDropSamplesOn
 	}
 
 	// Try pushing tss to remote storage
-	if rwctx.tryPushInternal(tss) {
+	if rwctx.tryPushTimeSeriesInternal(tss) {
 		return true
 	}
 
@@ -985,7 +1029,7 @@ func dropAggregatedSeries(src []prompb.TimeSeries, matchIdxs []byte, dropInput b
 }
 
 func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompb.TimeSeries) {
-	if rwctx.tryPushInternal(tss) {
+	if rwctx.tryPushTimeSeriesInternal(tss) {
 		return
 	}
 	if !rwctx.fq.IsPersistentQueueDisabled() {
@@ -996,7 +1040,14 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompb.TimeSeries) {
 	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 }
 
-func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompb.TimeSeries) bool {
+func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata) bool {
+	pss := rwctx.pss
+	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
+
+	return pss[idx].TryPushMetadata(mms)
+}
+
+func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompb.TimeSeries
 	defer func() {
@@ -1020,7 +1071,7 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompb.TimeSeries) bool {
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
-	return pss[idx].TryPush(tss)
+	return pss[idx].TryPushTimeSeries(tss)
 }
 
 var tssPool = &sync.Pool{

app/vmagent/remotewrite/remotewrite_test.go

@@ -106,7 +106,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 
 		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		if !rwctx.TryPush(inputTss, false) {
+		if !rwctx.TryPushTimeSeries(inputTss, false) {
 			t.Fatalf("cannot push samples to rwctx")
 		}
 

app/vmagent/remotewrite/streamaggr.go

@@ -141,7 +141,7 @@ func initStreamAggrConfigGlobal() {
 	}
 	dedupInterval := *streamAggrGlobalDedupInterval
 	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushTimeSeriesToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 	}
 }
 
@@ -216,7 +216,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 		EnableWindows:        *streamAggrGlobalEnableWindows,
 	}
 
-	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
+	sas, err := streamaggr.LoadFromFile(path, pushTimeSeriesToRemoteStoragesTrackDropped, opts, "global")
 	if err != nil {
 		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
 	}

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -15,6 +15,10 @@ groups:
       - alert: SameAlertNameWithDifferentGroup
         expr: absent(test)
         for: 1m
+      - alert: AlertWithTemplate
+        expr: test
+        annotations:
+          queryAnno: '{{ query "foo" | first | value }}'
 
   - name: group2
     rules:

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -10,7 +10,9 @@ tests:
     input_series:
       - series: "test"
         values: "_x5 1x5 _ stale"
-
+      - series: "foo"
+        values: "1x20"
+  
     alert_rule_test:
       - eval_time: 1m
         groupname: group1
@@ -32,6 +34,14 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts: []
+      - eval_time: 6m
+        groupname: group1
+        alertname: AlertWithTemplate
+        exp_alerts:
+          - exp_labels:
+              cluster: prod
+            exp_annotations:
+              queryAnno: '1'
 
     metricsql_expr_test:
       - expr: test
@@ -50,6 +60,8 @@ tests:
         values: "0+0x1440"
       - series: "test"
         values: "0+1x1440"
+      - series: "foo"
+        values: "1x20"
 
     metricsql_expr_test:
       - expr: count(ALERTS) by (alertgroup, alertname, alertstate)
@@ -59,6 +71,8 @@ tests:
             value: 1
           - labels: '{alertgroup="group1", alertname="InstanceDown", alertstate="pending"}'
             value: 1
+          - labels: '{alertgroup="group1", alertname="AlertWithTemplate", alertstate="firing"}'
+            value: 1
       - expr: t1
         eval_time: 4m
         exp_samples:

app/vmalert-tool/unittest/unittest.go

@@ -34,6 +34,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -84,7 +85,8 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 		defer server.Close()
 	} else {
 		httpListenAddr = httpListenPort
-		ln, err := net.Listen("tcp", fmt.Sprintf(":%s", httpListenPort))
+
+		ln, err := net.Listen(netutil.GetTCPNetwork(), fmt.Sprintf(":%s", httpListenPort))
 		if err != nil {
 			logger.Fatalf("cannot listen on port %s: %v", httpListenPort, err)
 		}
@@ -130,7 +132,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 		}
 		labels[s[:n]] = s[n+1:]
 	}
-	_, err = notifier.Init(nil, labels, externalURL)
+	_, err = notifier.Init(labels, externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init notifier: %v", err)
 	}
@@ -366,6 +368,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 			mergedExternalLabels[k] = v
 		}
 		ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
+		ng.Init()
 		groups = append(groups, ng)
 	}
 

app/vmalert/config/config.go

@@ -31,7 +31,7 @@ type Group struct {
 	// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
 	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155
 	EvalDelay   *promutil.Duration `yaml:"eval_delay,omitempty"`
-	Limit       int                `yaml:"limit,omitempty"`
+	Limit       *int               `yaml:"limit,omitempty"`
 	Rules       []Rule             `yaml:"rules"`
 	Concurrency int                `yaml:"concurrency"`
 	// Labels is a set of label value pairs, that will be added to every rule.
@@ -91,8 +91,8 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 	if g.EvalOffset != nil && g.EvalDelay != nil {
 		return fmt.Errorf("eval_offset cannot be used with eval_delay")
 	}
-	if g.Limit < 0 {
-		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", g.Limit)
+	if g.Limit != nil && *g.Limit < 0 {
+		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", *g.Limit)
 	}
 	if g.Concurrency < 0 {
 		return fmt.Errorf("invalid concurrency %d, shouldn't be less than 0", g.Concurrency)
@@ -295,10 +295,7 @@ func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpress
 }
 
 func parseConfig(data []byte) ([]Group, error) {
-	data, err := envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
-	}
+	data = envtemplate.ReplaceBytes(data)
 
 	var result []Group
 	type cfgFile struct {
@@ -310,13 +307,13 @@ func parseConfig(data []byte) ([]Group, error) {
 	decoder := yaml.NewDecoder(bytes.NewReader(data))
 	for {
 		var cf cfgFile
-		if err = decoder.Decode(&cf); err != nil {
+		if err := decoder.Decode(&cf); err != nil {
 			if err == io.EOF { // EOF indicates no more documents to read
 				break
 			}
 			return nil, err
 		}
-		if err = checkOverflow(cf.XXX, "config"); err != nil {
+		if err := checkOverflow(cf.XXX, "config"); err != nil {
 			return nil, err
 		}
 		result = append(result, cf.Groups...)

app/vmalert/config/config_test.go

@@ -181,9 +181,10 @@ func TestGroupValidate_Failure(t *testing.T) {
 		EvalOffset: promutil.NewDuration(2 * time.Minute),
 	}, false, "eval_offset should be smaller than interval")
 
+	limit := -1
 	f(&Group{
 		Name:  "wrong limit",
-		Limit: -1,
+		Limit: &limit,
 	}, false, "invalid limit")
 
 	f(&Group{

app/vmalert/datasource/datasource.go

@@ -132,10 +132,7 @@ func (ls Labels) String() string {
 // a=[]Label{{Name: "a", Value: "2"}},b=[]Label{{Name: "a", Value: "1"}}, return 1
 // a=[]Label{{Name: "a", Value: "1"}},b=[]Label{{Name: "a", Value: "1"}}, return 0
 func LabelCompare(a, b Labels) int {
-	l := len(a)
-	if len(b) < l {
-		l = len(b)
-	}
+	l := min(len(b), len(a))
 
 	for i := 0; i < l; i++ {
 		if a[i].Name != b[i].Name {

app/vmalert/main.go

@@ -7,7 +7,6 @@ import (
 	"net/url"
 	"os"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -83,8 +82,7 @@ absolute path to all .tpl files in root.
 )
 
 var (
-	alertURLGeneratorFn notifier.AlertURLGenerator
-	extURL              *url.URL
+	extURL *url.URL
 )
 
 func main() {
@@ -121,7 +119,7 @@ func main() {
 		return
 	}
 
-	alertURLGeneratorFn, err = getAlertURLGenerator(extURL, *externalAlertSource, *validateTemplates)
+	err = notifier.InitAlertURLGeneratorFn(extURL, *externalAlertSource, *validateTemplates)
 	if err != nil {
 		logger.Fatalf("failed to init `external.alert.source`: %s", err)
 	}
@@ -228,7 +226,7 @@ func newManager(ctx context.Context) (*manager, error) {
 		labels[s[:n]] = s[n+1:]
 	}
 
-	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
+	nts, err := notifier.Init(labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
@@ -292,35 +290,6 @@ func getHostnameAsExternalURL(addr string, isSecure bool) (*url.URL, error) {
 	return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port))
 }
 
-func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) {
-	if externalAlertSource == "" {
-		return func(a notifier.Alert) string {
-			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
-			return fmt.Sprintf("%s/vmalert/alert?%s=%s&%s=%s", externalURL, paramGroupID, gID, paramAlertID, aID)
-		}, nil
-	}
-	if validateTemplate {
-		if err := notifier.ValidateTemplates(map[string]string{
-			"tpl": externalAlertSource,
-		}); err != nil {
-			return nil, fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
-		}
-	}
-	m := map[string]string{
-		"tpl": externalAlertSource,
-	}
-	return func(alert notifier.Alert) string {
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
-		}
-		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
-		if err != nil {
-			logger.Errorf("cannot template alert source: %s", err)
-		}
-		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
-	}, nil
-}
-
 func usage() {
 	const s = `
 vmalert processes alerts and recording rules.

app/vmalert/main_test.go

@@ -49,30 +49,6 @@ func TestGetExternalURL(t *testing.T) {
 	}
 }
 
-func TestGetAlertURLGenerator(t *testing.T) {
-	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	fn, err := getAlertURLGenerator(u, "", false)
-	if err != nil {
-		t.Fatalf("unexpected error %s", err)
-	}
-	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
-	if exp != fn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
-	}
-	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
-	if err == nil {
-		t.Fatalf("expected template validation error got nil")
-	}
-	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
-	if err != nil {
-		t.Fatalf("unexpected error %s", err)
-	}
-	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
-	}
-}
-
 func TestConfigReload(t *testing.T) {
 	originalRulePath := *rulePath
 	originalExternalURL := extURL

app/vmalert/manager.go

@@ -29,25 +29,37 @@ type manager struct {
 	groups   map[uint64]*rule.Group
 }
 
-// ruleAPI generates apiRule object from alert by its ID(hash)
-func (m *manager) ruleAPI(gID, rID uint64) (apiRule, error) {
+// groupAPI generates apiGroup object from group by its ID(hash)
+func (m *manager) groupAPI(gID uint64) (*rule.ApiGroup, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
 	g, ok := m.groups[gID]
 	if !ok {
-		return apiRule{}, fmt.Errorf("can't find group with id %d", gID)
+		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
-	for _, rule := range g.Rules {
-		if rule.ID() == rID {
-			return ruleToAPI(rule), nil
+	return g.ToAPI(), nil
+}
+
+// ruleAPI generates apiRule object from alert by its ID(hash)
+func (m *manager) ruleAPI(gID, rID uint64) (rule.ApiRule, error) {
+	m.groupsMu.RLock()
+	defer m.groupsMu.RUnlock()
+
+	g, ok := m.groups[gID]
+	if !ok {
+		return rule.ApiRule{}, fmt.Errorf("can't find group with id %d", gID)
+	}
+	for _, r := range g.Rules {
+		if r.ID() == rID {
+			return r.ToAPI(), nil
 		}
 	}
-	return apiRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
+	return rule.ApiRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
 }
 
 // alertAPI generates apiAlert object from alert by its ID(hash)
-func (m *manager) alertAPI(gID, aID uint64) (*apiAlert, error) {
+func (m *manager) alertAPI(gID, aID uint64) (*rule.ApiAlert, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
@@ -60,7 +72,7 @@ func (m *manager) alertAPI(gID, aID uint64) (*apiAlert, error) {
 		if !ok {
 			continue
 		}
-		if apiAlert := alertToAPI(ar, aID); apiAlert != nil {
+		if apiAlert := ar.AlertToAPI(aID); apiAlert != nil {
 			return apiAlert, nil
 		}
 	}

app/vmalert/notifier/alert_test.go

@@ -20,7 +20,7 @@ func TestAlertExecTemplate(t *testing.T) {
 	)
 	extLabels["cluster"] = extCluster
 	extLabels["dc"] = extDC
-	_, err := Init(nil, extLabels, extURL)
+	_, err := Init(extLabels, extURL)
 	checkErr(t, err)
 
 	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {

app/vmalert/notifier/alertmanager.go

@@ -22,10 +22,11 @@ import (
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
-	addr    *url.URL
-	argFunc AlertURLGenerator
-	client  *http.Client
-	timeout time.Duration
+	addr      *url.URL
+	argFunc   AlertURLGenerator
+	client    *http.Client
+	timeout   time.Duration
+	lastError string
 
 	authCfg *promauth.Config
 	// stores already parsed RelabelConfigs object
@@ -71,6 +72,10 @@ func (am AlertManager) Addr() string {
 	return am.addr.Redacted()
 }
 
+func (am *AlertManager) LastError() string {
+	return am.lastError
+}
+
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	am.metrics.alertsSent.Add(len(alerts))
@@ -79,6 +84,9 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[st
 	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
 		am.metrics.alertsSendErrors.Add(len(alerts))
+		am.lastError = err.Error()
+	} else {
+		am.lastError = ""
 	}
 	return err
 }

app/vmalert/notifier/faker.go

@@ -18,6 +18,11 @@ type FakeNotifier struct {
 // Close does nothing
 func (*FakeNotifier) Close() {}
 
+// LastError returns last error message
+func (*FakeNotifier) LastError() string {
+	return ""
+}
+
 // Addr returns ""
 func (*FakeNotifier) Addr() string { return "" }
 

app/vmalert/notifier/init.go

@@ -4,10 +4,13 @@ import (
 	"flag"
 	"fmt"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 )
@@ -57,6 +60,42 @@ var (
 	sendTimeout = flagutil.NewArrayDuration("notifier.sendTimeout", 10*time.Second, "Timeout when sending alerts to the corresponding -notifier.url")
 )
 
+// AlertURLGeneratorFn returns a URL to the passed alert object.
+// Call InitAlertURLGeneratorFn before using this function.
+var AlertURLGeneratorFn AlertURLGenerator
+
+// InitAlertURLGeneratorFn populates AlertURLGeneratorFn
+func InitAlertURLGeneratorFn(externalURL *url.URL, externalAlertSource string, validateTemplate bool) error {
+	if externalAlertSource == "" {
+		AlertURLGeneratorFn = func(a Alert) string {
+			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
+			return fmt.Sprintf("%s/vmalert/alert?%s=%s&%s=%s", externalURL, "group_id", gID, "alert_id", aID)
+		}
+		return nil
+	}
+	if validateTemplate {
+		if err := ValidateTemplates(map[string]string{
+			"tpl": externalAlertSource,
+		}); err != nil {
+			return fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
+		}
+	}
+	m := map[string]string{
+		"tpl": externalAlertSource,
+	}
+	AlertURLGeneratorFn = func(alert Alert) string {
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
+		}
+		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
+		if err != nil {
+			logger.Errorf("cannot template alert source: %s", err)
+		}
+		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
+	}
+	return nil
+}
+
 // cw holds a configWatcher for configPath configuration file
 // configWatcher provides a list of Notifier objects discovered
 // from static config or via service discovery.
@@ -90,7 +129,7 @@ var (
 //   - configuration via file. Supports live reloads and service discovery.
 //
 // Init returns an error if both mods are used.
-func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
+func Init(extLabels map[string]string, extURL string) (func() []Notifier, error) {
 	externalURL = extURL
 	externalLabels = extLabels
 	_, err := url.Parse(externalURL)
@@ -117,7 +156,7 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 	}
 
 	if len(*addrs) > 0 {
-		notifiers, err := notifiersFromFlags(gen)
+		notifiers, err := notifiersFromFlags(AlertURLGeneratorFn)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create notifier from flag values: %w", err)
 		}
@@ -127,7 +166,7 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 		return staticNotifiersFn, nil
 	}
 
-	cw, err = newWatcher(*configPath, gen)
+	cw, err = newWatcher(*configPath, AlertURLGeneratorFn)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init config watcher: %w", err)
 	}

app/vmalert/notifier/init_test.go

@@ -1,6 +1,8 @@
 package notifier
 
 import (
+	"fmt"
+	"net/url"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -12,7 +14,7 @@ func TestInit(t *testing.T) {
 
 	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
 
-	fn, err := Init(nil, nil, "")
+	fn, err := Init(nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
@@ -52,7 +54,7 @@ func TestInitNegative(t *testing.T) {
 		*configPath = path
 		*addrs = flagutil.ArrayString{addr}
 		*blackHole = bh
-		if _, err := Init(nil, nil, ""); err == nil {
+		if _, err := Init(nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
 		}
 	}
@@ -69,7 +71,7 @@ func TestBlackHole(t *testing.T) {
 
 	*blackHole = true
 
-	fn, err := Init(nil, nil, "")
+	fn, err := Init(nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
@@ -91,3 +93,30 @@ func TestBlackHole(t *testing.T) {
 		t.Fatalf("expected to get \"blackhole\"; got %q instead", nf1.Addr())
 	}
 }
+
+func TestGetAlertURLGenerator(t *testing.T) {
+	oldAlertURLGeneratorFn := AlertURLGeneratorFn
+	defer func() { AlertURLGeneratorFn = oldAlertURLGeneratorFn }()
+
+	testAlert := Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	err := InitAlertURLGeneratorFn(u, "", false)
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", "group_id", "alert_id")
+	if exp != AlertURLGeneratorFn(testAlert) {
+		t.Fatalf("unexpected url want %s, got %s", exp, AlertURLGeneratorFn(testAlert))
+	}
+	err = InitAlertURLGeneratorFn(nil, "foo?{{invalid}}", true)
+	if err == nil {
+		t.Fatalf("expected template validation error got nil")
+	}
+	err = InitAlertURLGeneratorFn(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != AlertURLGeneratorFn(testAlert) {
+		t.Fatalf("unexpected url want %s, got %s", exp, AlertURLGeneratorFn(testAlert))
+	}
+}

app/vmalert/notifier/notifier.go

@@ -10,6 +10,8 @@ type Notifier interface {
 	Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error
 	// Addr returns address where alerts are sent.
 	Addr() string
+	// LastError returns error, that occured during last attempt to send data
+	LastError() string
 	// Close is a destructor for the Notifier
 	Close()
 }

app/vmalert/notifier/notifier_blackhole.go

@@ -25,6 +25,11 @@ func (bh *blackHoleNotifier) Close() {
 	bh.metrics.close()
 }
 
+// LastError return last notifier's error
+func (bh *blackHoleNotifier) LastError() string {
+	return ""
+}
+
 // newBlackHoleNotifier creates a new blackHoleNotifier
 func newBlackHoleNotifier() *blackHoleNotifier {
 	address := "blackhole"

app/vmalert/notifier/web.go

@@ -0,0 +1,19 @@
+package notifier
+
+// ApiNotifier represents a Notifier configuration for WEB view
+type ApiNotifier struct {
+	// Kind is a Notifier type
+	Kind TargetType `json:"kind"`
+	// Targets is a list of Notifier targets
+	Targets []*ApiTarget `json:"targets"`
+}
+
+// ApiTarget represents a specific Notifier target for WEB view
+type ApiTarget struct {
+	// Address is a URL for sending notifications
+	Address string `json:"address"`
+	// Labels is a list of labels to add to each sent notification
+	Labels map[string]string `json:"labels"`
+	// LastError contains the error faced while sending to notifier.
+	LastError string `json:"lastError"`
+}

app/vmalert/remotewrite/client_test.go

@@ -182,7 +182,7 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("decode err: %w", err))
 		return
 	}
-	wru := &prompb.WriteRequestUnmarshaller{}
+	wru := &prompb.WriteRequestUnmarshaler{}
 	wr, err := wru.UnmarshalProtobuf(b)
 	if err != nil {
 		rw.err(w, fmt.Errorf("unmarhsal err: %w", err))

app/vmalert/replay.go

@@ -28,8 +28,8 @@ var (
 		"Defines how many retries to make before giving up on rule if request for it returns an error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
-	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent `/query_range` requests for a single rule. "+
-		"Increasing this value when replaying for a long time and a single request range is limited by `-replay.maxDatapointsPerQuery`.")
+	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
+		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {

app/vmalert/replay_test.go

@@ -246,24 +246,33 @@ func TestReplay(t *testing.T) {
 
 	// multiple rules + rule concurrency + group concurrency
 	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 3, 0, []config.Group{
-		{Rules: []config.Rule{{Alert: "foo-group-single-concurrent", Expr: "sum(up) > 1"}, {Alert: "bar-group-single-concurrent", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
+		{Rules: []config.Rule{{Alert: "foo-group-single-concurrent", For: promutil.NewDuration(30 * time.Second), Expr: "sum(up) > 1"}, {Alert: "bar-group-single-concurrent", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
 		registry: map[string]map[string][]datasource.Metric{
 			"sum(up) > 1": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {{
-					Timestamps: []int64{1},
+				"12:00:00+12:01:00": {{
+					Timestamps: []int64{1609502460},
+					Values:     []float64{1},
+				}},
+				"12:01:00+12:02:00": {{
+					Timestamps: []int64{1609502520},
+					Values:     []float64{1},
+				}},
+				"12:02:00+12:02:30": {{
+					Timestamps: []int64{1609502580},
 					Values:     []float64{1},
 				}},
-				"12:02:00+12:02:30": {},
 			},
 			"max(up) < 1": {
-				"12:00:00+12:01:00": {},
+				"12:00:00+12:01:00": {{
+					Timestamps: []int64{1609502460},
+					Values:     []float64{1},
+				}},
 				"12:01:00+12:02:00": {{
-					Timestamps: []int64{1},
+					Timestamps: []int64{1609502520},
 					Values:     []float64{1},
 				}},
 				"12:02:00+12:02:30": {},
 			},
 		},
-	}, 4)
+	}, 10)
 }

app/vmalert/rule/alerting.go

@@ -187,6 +187,54 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
+// ToAPI returns ApiRule representation of ar
+func (ar *AlertingRule) ToAPI() ApiRule {
+	state := ar.state
+	lastState := state.getLast()
+	r := ApiRule{
+		Type:              TypeAlerting,
+		DatasourceType:    ar.Type.String(),
+		Name:              ar.Name,
+		Query:             ar.Expr,
+		Duration:          ar.For.Seconds(),
+		KeepFiringFor:     ar.KeepFiringFor.Seconds(),
+		Labels:            ar.Labels,
+		Annotations:       ar.Annotations,
+		LastEvaluation:    lastState.Time,
+		EvaluationTime:    lastState.Duration.Seconds(),
+		Health:            "ok",
+		State:             "inactive",
+		Alerts:            ar.AlertsToAPI(),
+		LastSamples:       lastState.Samples,
+		LastSeriesFetched: lastState.SeriesFetched,
+		MaxUpdates:        state.size(),
+		Updates:           state.getAll(),
+		Debug:             ar.Debug,
+
+		// encode as strings to avoid rounding in JSON
+		ID:        fmt.Sprintf("%d", ar.ID()),
+		GroupID:   fmt.Sprintf("%d", ar.GroupID),
+		GroupName: ar.GroupName,
+		File:      ar.File,
+	}
+	if lastState.Err != nil {
+		r.LastError = lastState.Err.Error()
+		r.Health = "err"
+	}
+	// satisfy apiRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
+	}
+	return r
+}
+
 // GetAlerts returns active alerts of rule
 func (ar *AlertingRule) GetAlerts() []*notifier.Alert {
 	ar.alertsMu.RLock()
@@ -341,11 +389,15 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 		return []datasource.Metric{{Timestamps: []int64{0}, Values: []float64{math.NaN()}}}, nil
 	}
 	for _, s := range res.Data {
-		ls, as, err := ar.expandTemplates(s, qFn, time.Time{})
+		ls, err := ar.expandLabelTemplates(s)
 		if err != nil {
-			return nil, fmt.Errorf("failed to expand templates: %s", err)
+			return nil, err
 		}
 		alertID := hash(ls.processed)
+		as, err := ar.expandAnnotationTemplates(s, qFn, time.Time{}, ls)
+		if err != nil {
+			return nil, err
+		}
 		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
 
 		prevT := time.Time{}
@@ -363,7 +415,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				a.State = notifier.StatePending
 				a.ActiveAt = at
 				// re-template the annotations as active timestamp is changed
-				_, a.Annotations, _ = ar.expandTemplates(s, qFn, at)
+				a.Annotations, _ = ar.expandAnnotationTemplates(s, qFn, at, ls)
 				a.Start = time.Time{}
 			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
 				a.State = notifier.StateFiring
@@ -376,13 +428,15 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
 
-			// save alert's state on last iteration, so it can be used on the next execRange call
-			if at.Equal(end) {
+			// if for>0, save alert's state on last iteration, so it can be used on the next execRange call
+			if ar.For > 0 && at.Equal(end) {
 				holdAlertState[alertID] = a
 			}
 		}
 	}
-	ar.alerts = holdAlertState
+	if len(holdAlertState) > 0 {
+		ar.alerts = holdAlertState
+	}
 	return result, nil
 }
 
@@ -428,9 +482,22 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	expandedLabels := make([]*labelSet, len(res.Data))
 	expandedAnnotations := make([]map[string]string, len(res.Data))
 	for i, m := range res.Data {
-		ls, as, err := ar.expandTemplates(m, qFn, ts)
+		ls, err := ar.expandLabelTemplates(m)
 		if err != nil {
-			curState.Err = fmt.Errorf("failed to expand templates: %w", err)
+			curState.Err = err
+			return nil, curState.Err
+		}
+		at := ts
+		alertID := hash(ls.processed)
+		if a, ok := ar.alerts[alertID]; ok {
+			// modify activeAt for annotation templating if the alert has already triggered(in state Pending or Firing)
+			if a.State != notifier.StateInactive {
+				at = a.ActiveAt
+			}
+		}
+		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
+		if err != nil {
+			curState.Err = err
 			return nil, curState.Err
 		}
 		expandedLabels[i] = ls
@@ -473,6 +540,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
+
 		a := ar.newAlert(m, ts, labels.processed, annotations)
 		a.ID = alertID
 		a.State = notifier.StatePending
@@ -536,12 +604,18 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	return append(tss, ar.toTimeSeries(ts.Unix())...), nil
 }
 
-func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
+func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric) (*labelSet, error) {
+	qFn := func(_ string) ([]datasource.Metric, error) {
+		return nil, fmt.Errorf("`query` template isn't supported in rule label")
+	}
 	ls, err := ar.toLabels(m, qFn)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to expand labels: %w", err)
+		return nil, fmt.Errorf("failed to expand label templates: %s", err)
 	}
+	return ls, nil
+}
 
+func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet) (map[string]string, error) {
 	tplData := notifier.AlertTplData{
 		Value:    m.Values[0],
 		Type:     ar.Type.String(),
@@ -549,14 +623,14 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 		Expr:     ar.Expr,
 		AlertID:  hash(ls.processed),
 		GroupID:  ar.GroupID,
-		ActiveAt: ts,
+		ActiveAt: activeAt,
 		For:      ar.For,
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to template annotations: %w", err)
+		return nil, fmt.Errorf("failed to expand annotation templates: %s", err)
 	}
-	return ls, as, nil
+	return as, nil
 }
 
 // toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts

app/vmalert/rule/alerting_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -267,8 +268,15 @@ func TestAlertingRule_Exec(t *testing.T) {
 				if got.State != exp.State {
 					t.Fatalf("evalIndex %d: expected state %d; got %d", i, exp.State, got.State)
 				}
+				if rule.Annotations != nil && exp.Annotations != nil {
+					if !reflect.DeepEqual(got.Annotations, exp.Annotations) {
+						t.Fatalf("evalIndex %d: expected annotations %v; got %v", i, exp.Annotations, got.Annotations)
+					}
+				}
 			}
 		}
+		// reset ts for next test
+		ts, _ = time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
 	}
 
 	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil, nil)
@@ -522,7 +530,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		},
 	})
 
-	f(newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep), [][]datasource.Metric{
+	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>inactive=>pending=>firing", defaultStep, 0, 0, map[string]string{"activeAt": "{{ $activeAt.UnixMilli }}"}), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
 		{metricWithLabels(t, "name", "foo")},
 		// empty step to set alert inactive
@@ -530,11 +538,11 @@ func TestAlertingRule_Exec(t *testing.T) {
 		{metricWithLabels(t, "name", "foo")},
 		{metricWithLabels(t, "name", "foo")},
 	}, map[int][]testAlert{
-		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
-		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
-		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
-		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.UnixMilli(), 10)}}}},
+		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.UnixMilli(), 10)}}}},
+		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.UnixMilli(), 10)}}}},
+		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.Add(defaultStep*3).UnixMilli(), 10)}}}},
+		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.Add(defaultStep*3).UnixMilli(), 10)}}}},
 	}, nil)
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>firing", defaultStep, 0, defaultStep, nil), [][]datasource.Metric{

app/vmalert/rule/group.go

@@ -2,7 +2,6 @@ package rule
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -25,6 +24,10 @@ import (
 )
 
 var (
+	ruleResultsLimit = flag.Int("rule.resultsLimit", 0, "Limits the number of alerts or recording results a single rule can produce. "+
+		"Can be overridden by the limit option under group if specified. "+
+		"If exceeded, the rule will be marked with an error and all its results will be discarded. "+
+		"0 means no limit.")
 	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
 		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
 	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
@@ -112,7 +115,6 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Name:            cfg.Name,
 		File:            cfg.File,
 		Interval:        cfg.Interval.Duration(),
-		Limit:           cfg.Limit,
 		Concurrency:     cfg.Concurrency,
 		checksum:        cfg.Checksum,
 		Params:          cfg.Params,
@@ -129,6 +131,11 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 	if g.Interval == 0 {
 		g.Interval = defaultInterval
 	}
+	if cfg.Limit != nil {
+		g.Limit = *cfg.Limit
+	} else {
+		g.Limit = *ruleResultsLimit
+	}
 	if g.Concurrency < 1 {
 		g.Concurrency = 1
 	}
@@ -289,7 +296,7 @@ func (g *Group) InterruptEval() {
 	}
 }
 
-// Close stops the group and it's rules, unregisters group metrics
+// Close stops the group and its rules, unregisters group metrics
 func (g *Group) Close() {
 	if g.doneCh == nil {
 		return
@@ -298,10 +305,6 @@ func (g *Group) Close() {
 	g.InterruptEval()
 	<-g.finishedCh
 
-	g.closeGroupMetrics()
-}
-
-func (g *Group) closeGroupMetrics() {
 	metrics.UnregisterSet(g.metrics.set, true)
 }
 
@@ -331,7 +334,7 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 	defer func() { close(g.finishedCh) }()
 	evalTS := time.Now()
 	// sleep random duration to spread group rules evaluation
-	// over time in order to reduce load on datasource.
+	// over time to reduce the load on datasource.
 	if !SkipRandSleepOnGroupStart {
 		sleepBeforeStart := delayBeforeStart(evalTS, g.GetID(), g.Interval, g.EvalOffset)
 		g.infof("will start in %v", sleepBeforeStart)
@@ -472,18 +475,6 @@ func (g *Group) UpdateWith(newGroup *Group) {
 	g.updateCh <- newGroup
 }
 
-// DeepCopy returns a deep copy of group
-func (g *Group) DeepCopy() *Group {
-	g.mu.RLock()
-	data, _ := json.Marshal(g)
-	g.mu.RUnlock()
-	newG := Group{}
-	_ = json.Unmarshal(data, &newG)
-	newG.Rules = g.Rules
-	newG.id = g.id
-	return &newG
-}
-
 // if offset is specified, delayBeforeStart returns a duration to help aligning timestamp with offset;
 // otherwise, it returns a random duration between [0..interval] based on group key.
 func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *time.Duration) time.Duration {
@@ -587,6 +578,11 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 
 func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
 	fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
+	// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
+	if r, ok := r.(*AlertingRule); ok && r.For > 0 {
+		ruleEvaluationConcurrency = 1
+	}
 	sem := make(chan struct{}, ruleEvaluationConcurrency)
 	wg := sync.WaitGroup{}
 	res := make(chan int, int(ri.end.Sub(ri.start)/ri.step)+1)

app/vmalert/rule/recording.go

@@ -81,6 +81,37 @@ func (rr *RecordingRule) ID() uint64 {
 	return rr.RuleID
 }
 
+// ToAPI returns ApiRule representation of rr
+func (rr *RecordingRule) ToAPI() ApiRule {
+	state := rr.state
+	lastState := state.getLast()
+	r := ApiRule{
+		Type:              TypeRecording,
+		DatasourceType:    rr.Type.String(),
+		Name:              rr.Name,
+		Query:             rr.Expr,
+		Labels:            rr.Labels,
+		LastEvaluation:    lastState.Time,
+		EvaluationTime:    lastState.Duration.Seconds(),
+		Health:            "ok",
+		LastSamples:       lastState.Samples,
+		LastSeriesFetched: lastState.SeriesFetched,
+		MaxUpdates:        state.size(),
+		Updates:           state.getAll(),
+
+		// encode as strings to avoid rounding
+		ID:        fmt.Sprintf("%d", rr.ID()),
+		GroupID:   fmt.Sprintf("%d", rr.GroupID),
+		GroupName: rr.GroupName,
+		File:      rr.File,
+	}
+	if lastState.Err != nil {
+		r.LastError = lastState.Err.Error()
+		r.Health = "err"
+	}
+	return r
+}
+
 // NewRecordingRule creates a new RecordingRule
 func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *RecordingRule {
 	debug := group.Debug

app/vmalert/rule/recording_test.go

@@ -437,7 +437,7 @@ func TestRecordingRuleExec_Negative(t *testing.T) {
 
 	_, err = rr.exec(context.TODO(), time.Now(), 0)
 	if err != nil {
-		t.Fatalf("cannot execute recroding rule: %s", err)
+		t.Fatalf("cannot execute recording rule: %s", err)
 	}
 }
 

app/vmalert/rule/rule.go

@@ -21,6 +21,8 @@ type Rule interface {
 	// ID returns unique ID that may be used for
 	// identifying this Rule among others.
 	ID() uint64
+	// ToAPI returns ApiRule representation of Rule
+	ToAPI() ApiRule
 	// exec executes the rule with given context at the given timestamp and limit.
 	// returns an err if number of resulting time series exceeds the limit.
 	exec(ctx context.Context, ts time.Time, limit int) ([]prompb.TimeSeries, error)
@@ -68,39 +70,6 @@ type StateEntry struct {
 	Curl string `json:"curl"`
 }
 
-// GetLastEntry returns latest stateEntry of rule
-func GetLastEntry(r Rule) StateEntry {
-	if rule, ok := r.(*AlertingRule); ok {
-		return rule.state.getLast()
-	}
-	if rule, ok := r.(*RecordingRule); ok {
-		return rule.state.getLast()
-	}
-	return StateEntry{}
-}
-
-// GetRuleStateSize returns size of rule stateEntry
-func GetRuleStateSize(r Rule) int {
-	if rule, ok := r.(*AlertingRule); ok {
-		return rule.state.size()
-	}
-	if rule, ok := r.(*RecordingRule); ok {
-		return rule.state.size()
-	}
-	return 0
-}
-
-// GetAllRuleState returns rule entire stateEntries
-func GetAllRuleState(r Rule) []StateEntry {
-	if rule, ok := r.(*AlertingRule); ok {
-		return rule.state.getAll()
-	}
-	if rule, ok := r.(*RecordingRule); ok {
-		return rule.state.getAll()
-	}
-	return []StateEntry{}
-}
-
 func (s *ruleState) size() int {
 	s.RLock()
 	defer s.RUnlock()

app/vmalert/rule/web.go

@@ -1,4 +1,4 @@
-package main
+package rule
 
 import (
 	"fmt"
@@ -8,79 +8,28 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 )
 
 const (
 	// ParamGroupID is group id key in url parameter
-	paramGroupID = "group_id"
+	ParamGroupID = "group_id"
 	// ParamAlertID is alert id key in url parameter
-	paramAlertID = "alert_id"
+	ParamAlertID = "alert_id"
 	// ParamRuleID is rule id key in url parameter
-	paramRuleID = "rule_id"
+	ParamRuleID = "rule_id"
+
+	// TypeRecording is a RecordingRule type
+	TypeRecording = "recording"
+	// TypeAlerting is an AlertingRule type
+	TypeAlerting = "alerting"
 )
 
-type apiNotifier struct {
-	Kind    string       `json:"kind"`
-	Targets []*apiTarget `json:"targets"`
-}
-
-type apiTarget struct {
-	Address string            `json:"address"`
-	Labels  map[string]string `json:"labels"`
-}
-
-// apiAlert represents a notifier.AlertingRule state
-// for WEB view
-// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
-type apiAlert struct {
-	State       string            `json:"state"`
-	Name        string            `json:"name"`
-	Value       string            `json:"value"`
-	Labels      map[string]string `json:"labels,omitempty"`
-	Annotations map[string]string `json:"annotations"`
-	ActiveAt    time.Time         `json:"activeAt"`
-
-	// Additional fields
-
-	// ID is an unique Alert's ID within a group
-	ID string `json:"id"`
-	// RuleID is an unique Rule's ID within a group
-	RuleID string `json:"rule_id"`
-	// GroupID is an unique Group's ID
-	GroupID string `json:"group_id"`
-	// Expression contains the PromQL/MetricsQL expression
-	// for Rule's evaluation
-	Expression string `json:"expression"`
-	// SourceLink contains a link to a system which should show
-	// why Alert was generated
-	SourceLink string `json:"source"`
-	// Restored shows whether Alert's state was restored on restart
-	Restored bool `json:"restored"`
-	// Stabilizing shows when firing state is kept because of
-	// `keep_firing_for` instead of real alert
-	Stabilizing bool `json:"stabilizing"`
-}
-
-// WebLink returns a link to the alert which can be used in UI.
-func (aa *apiAlert) WebLink() string {
-	return fmt.Sprintf("alert?%s=%s&%s=%s",
-		paramGroupID, aa.GroupID, paramAlertID, aa.ID)
-}
-
-// APILink returns a link to the alert's JSON representation.
-func (aa *apiAlert) APILink() string {
-	return fmt.Sprintf("api/v1/alert?%s=%s&%s=%s",
-		paramGroupID, aa.GroupID, paramAlertID, aa.ID)
-}
-
-// apiGroup represents Group for web view
-// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
-type apiGroup struct {
+// ApiGroup represents a Group for web view
+type ApiGroup struct {
 	// Name is the group name as present in the config
 	Name string `json:"name"`
 	// Rules contains both recording and alerting rules
-	Rules []apiRule `json:"rules"`
+	Rules []ApiRule `json:"rules"`
 	// Interval is the Group's evaluation interval in float seconds as present in the file.
 	Interval float64 `json:"interval"`
 	// LastEvaluation is the timestamp of the last time the Group was executed
@@ -116,15 +65,20 @@ type apiGroup struct {
 	NoMatch int
 }
 
-// groupAlerts represents a group of alerts for WEB view
-type groupAlerts struct {
-	Group  *apiGroup
-	Alerts []*apiAlert
+// APILink returns a link to the group's JSON representation.
+func (ag *ApiGroup) APILink() string {
+	return fmt.Sprintf("api/v1/group?%s=%s", ParamGroupID, ag.ID)
 }
 
-// apiRule represents a Rule for web view
+// GroupAlerts represents a Group with its Alerts for web view
+type GroupAlerts struct {
+	Group  *ApiGroup
+	Alerts []*ApiAlert
+}
+
+// ApiRule represents a Rule for web view
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
-type apiRule struct {
+type ApiRule struct {
 	// State must be one of these under following scenarios
 	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing ruleState.
 	//  "firing": at least 1 alert in the rule in firing state.
@@ -146,7 +100,7 @@ type apiRule struct {
 	// LastEvaluation is the timestamp of the last time the rule was executed
 	LastEvaluation time.Time `json:"lastEvaluation"`
 	// Alerts  is the list of all the alerts in this rule that are currently pending or firing
-	Alerts []*apiAlert `json:"alerts,omitempty"`
+	Alerts []*ApiAlert `json:"alerts,omitempty"`
 	// Health is the health of rule evaluation.
 	// It MUST be one of "ok", "err", "unknown"
 	Health string `json:"health"`
@@ -177,143 +131,96 @@ type apiRule struct {
 	// MaxUpdates is the max number of recorded ruleStateEntry objects
 	MaxUpdates int `json:"max_updates_entries"`
 	// Updates contains the ordered list of recorded ruleStateEntry objects
-	Updates []rule.StateEntry `json:"-"`
+	Updates []StateEntry `json:"-"`
 }
 
-// apiRuleWithUpdates represents apiRule but with extra fields for marshalling
-type apiRuleWithUpdates struct {
-	apiRule
-	// Updates contains the ordered list of recorded ruleStateEntry objects
-	StateUpdates []rule.StateEntry `json:"updates,omitempty"`
-}
+// ApiAlert represents a notifier.AlertingRule state
+// for WEB view
+// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
+type ApiAlert struct {
+	State       string            `json:"state"`
+	Name        string            `json:"name"`
+	Value       string            `json:"value"`
+	Labels      map[string]string `json:"labels,omitempty"`
+	Annotations map[string]string `json:"annotations"`
+	ActiveAt    time.Time         `json:"activeAt"`
 
-// APILink returns a link to the rule's JSON representation.
-func (ar apiRule) APILink() string {
-	return fmt.Sprintf("api/v1/rule?%s=%s&%s=%s",
-		paramGroupID, ar.GroupID, paramRuleID, ar.ID)
+	// Additional fields
+
+	// ID is an unique Alert's ID within a group
+	ID string `json:"id"`
+	// RuleID is an unique Rule's ID within a group
+	RuleID string `json:"rule_id"`
+	// GroupID is an unique Group's ID
+	GroupID string `json:"group_id"`
+	// Expression contains the PromQL/MetricsQL expression
+	// for Rule's evaluation
+	Expression string `json:"expression"`
+	// SourceLink contains a link to a system which should show
+	// why Alert was generated
+	SourceLink string `json:"source"`
+	// Restored shows whether Alert's state was restored on restart
+	Restored bool `json:"restored"`
+	// Stabilizing shows when firing state is kept because of
+	// `keep_firing_for` instead of real alert
+	Stabilizing bool `json:"stabilizing"`
 }
 
 // WebLink returns a link to the alert which can be used in UI.
-func (ar apiRule) WebLink() string {
+func (aa *ApiAlert) WebLink() string {
+	return fmt.Sprintf("alert?%s=%s&%s=%s",
+		ParamGroupID, aa.GroupID, ParamAlertID, aa.ID)
+}
+
+// APILink returns a link to the alert's JSON representation.
+func (aa *ApiAlert) APILink() string {
+	return fmt.Sprintf("api/v1/alert?%s=%s&%s=%s",
+		ParamGroupID, aa.GroupID, ParamAlertID, aa.ID)
+}
+
+// ApiRuleWithUpdates represents ApiRule but with extra fields for marshalling
+type ApiRuleWithUpdates struct {
+	ApiRule
+	// Updates contains the ordered list of recorded ruleStateEntry objects
+	StateUpdates []StateEntry `json:"updates,omitempty"`
+}
+
+// APILink returns a link to the rule's JSON representation.
+func (ar ApiRule) APILink() string {
+	return fmt.Sprintf("api/v1/rule?%s=%s&%s=%s",
+		ParamGroupID, ar.GroupID, ParamRuleID, ar.ID)
+}
+
+// WebLink returns a link to the alert which can be used in UI.
+func (ar ApiRule) WebLink() string {
 	return fmt.Sprintf("rule?%s=%s&%s=%s",
-		paramGroupID, ar.GroupID, paramRuleID, ar.ID)
+		ParamGroupID, ar.GroupID, ParamRuleID, ar.ID)
 }
 
-func ruleToAPI(r any) apiRule {
-	if ar, ok := r.(*rule.AlertingRule); ok {
-		return alertingToAPI(ar)
-	}
-	if rr, ok := r.(*rule.RecordingRule); ok {
-		return recordingToAPI(rr)
-	}
-	return apiRule{}
-}
-
-const (
-	ruleTypeRecording = "recording"
-	ruleTypeAlerting  = "alerting"
-)
-
-func recordingToAPI(rr *rule.RecordingRule) apiRule {
-	lastState := rule.GetLastEntry(rr)
-	r := apiRule{
-		Type:              ruleTypeRecording,
-		DatasourceType:    rr.Type.String(),
-		Name:              rr.Name,
-		Query:             rr.Expr,
-		Labels:            rr.Labels,
-		LastEvaluation:    lastState.Time,
-		EvaluationTime:    lastState.Duration.Seconds(),
-		Health:            "ok",
-		LastSamples:       lastState.Samples,
-		LastSeriesFetched: lastState.SeriesFetched,
-		MaxUpdates:        rule.GetRuleStateSize(rr),
-		Updates:           rule.GetAllRuleState(rr),
-
-		// encode as strings to avoid rounding
-		ID:        fmt.Sprintf("%d", rr.ID()),
-		GroupID:   fmt.Sprintf("%d", rr.GroupID),
-		GroupName: rr.GroupName,
-		File:      rr.File,
-	}
-	if lastState.Err != nil {
-		r.LastError = lastState.Err.Error()
-		r.Health = "err"
-	}
-	return r
-}
-
-// alertingToAPI returns Rule representation in form of apiRule
-func alertingToAPI(ar *rule.AlertingRule) apiRule {
-	lastState := rule.GetLastEntry(ar)
-	r := apiRule{
-		Type:              ruleTypeAlerting,
-		DatasourceType:    ar.Type.String(),
-		Name:              ar.Name,
-		Query:             ar.Expr,
-		Duration:          ar.For.Seconds(),
-		KeepFiringFor:     ar.KeepFiringFor.Seconds(),
-		Labels:            ar.Labels,
-		Annotations:       ar.Annotations,
-		LastEvaluation:    lastState.Time,
-		EvaluationTime:    lastState.Duration.Seconds(),
-		Health:            "ok",
-		State:             "inactive",
-		Alerts:            ruleToAPIAlert(ar),
-		LastSamples:       lastState.Samples,
-		LastSeriesFetched: lastState.SeriesFetched,
-		MaxUpdates:        rule.GetRuleStateSize(ar),
-		Updates:           rule.GetAllRuleState(ar),
-		Debug:             ar.Debug,
-
-		// encode as strings to avoid rounding in JSON
-		ID:        fmt.Sprintf("%d", ar.ID()),
-		GroupID:   fmt.Sprintf("%d", ar.GroupID),
-		GroupName: ar.GroupName,
-		File:      ar.File,
-	}
-	if lastState.Err != nil {
-		r.LastError = lastState.Err.Error()
-		r.Health = "err"
-	}
-	// satisfy apiRule.State logic
-	if len(r.Alerts) > 0 {
-		r.State = notifier.StatePending.String()
-		stateFiring := notifier.StateFiring.String()
-		for _, a := range r.Alerts {
-			if a.State == stateFiring {
-				r.State = stateFiring
-				break
-			}
-		}
-	}
-	return r
-}
-
-// ruleToAPIAlert generates list of apiAlert objects from existing alerts
-func ruleToAPIAlert(ar *rule.AlertingRule) []*apiAlert {
-	var alerts []*apiAlert
+// AlertsToAPI returns list of ApiAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*ApiAlert {
+	var alerts []*ApiAlert
 	for _, a := range ar.GetAlerts() {
 		if a.State == notifier.StateInactive {
 			continue
 		}
-		alerts = append(alerts, newAlertAPI(ar, a))
+		alerts = append(alerts, NewAlertAPI(ar, a))
 	}
 	return alerts
 }
 
-// alertToAPI generates apiAlert object from alert by its id(hash)
-func alertToAPI(ar *rule.AlertingRule, id uint64) *apiAlert {
+// AlertToAPI generates apiAlert object from alert by its id(hash)
+func (ar *AlertingRule) AlertToAPI(id uint64) *ApiAlert {
 	a := ar.GetAlert(id)
 	if a == nil {
 		return nil
 	}
-	return newAlertAPI(ar, a)
+	return NewAlertAPI(ar, a)
 }
 
 // NewAlertAPI creates apiAlert for notifier.Alert
-func newAlertAPI(ar *rule.AlertingRule, a *notifier.Alert) *apiAlert {
-	aa := &apiAlert{
+func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
+	aa := &ApiAlert{
 		// encode as strings to avoid rounding
 		ID:      fmt.Sprintf("%d", a.ID),
 		GroupID: fmt.Sprintf("%d", a.GroupID),
@@ -328,8 +235,8 @@ func newAlertAPI(ar *rule.AlertingRule, a *notifier.Alert) *apiAlert {
 		Restored:    a.Restored,
 		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
 	}
-	if alertURLGeneratorFn != nil {
-		aa.SourceLink = alertURLGeneratorFn(*a)
+	if notifier.AlertURLGeneratorFn != nil {
+		aa.SourceLink = notifier.AlertURLGeneratorFn(*a)
 	}
 	if a.State == notifier.StateFiring && !a.KeepFiringSince.IsZero() {
 		aa.Stabilizing = true
@@ -337,9 +244,11 @@ func newAlertAPI(ar *rule.AlertingRule, a *notifier.Alert) *apiAlert {
 	return aa
 }
 
-func groupToAPI(g *rule.Group) *apiGroup {
-	g = g.DeepCopy()
-	ag := apiGroup{
+// ToAPI returns ApiGroup representation of g
+func (g *Group) ToAPI() *ApiGroup {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+	ag := ApiGroup{
 		// encode as string to avoid rounding
 		ID:              strconv.FormatUint(g.GetID(), 10),
 		Name:            g.Name,
@@ -359,9 +268,9 @@ func groupToAPI(g *rule.Group) *apiGroup {
 	if g.EvalDelay != nil {
 		ag.EvalDelay = g.EvalDelay.Seconds()
 	}
-	ag.Rules = make([]apiRule, 0)
+	ag.Rules = make([]ApiRule, 0)
 	for _, r := range g.Rules {
-		ag.Rules = append(ag.Rules, ruleToAPI(r))
+		ag.Rules = append(ag.Rules, r.ToAPI())
 	}
 	return &ag
 }

app/vmalert/rule/web_test.go

@@ -1,4 +1,4 @@
-package main
+package rule
 
 import (
 	"fmt"
@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 )
 
 func TestRecordingToApi(t *testing.T) {
@@ -17,7 +16,7 @@ func TestRecordingToApi(t *testing.T) {
 		Values: []float64{1}, Timestamps: []int64{0},
 	})
 	entriesLimit := 44
-	g := rule.NewGroup(config.Group{
+	g := NewGroup(config.Group{
 		Name:        "group",
 		File:        "rules.yaml",
 		Concurrency: 1,
@@ -31,24 +30,24 @@ func TestRecordingToApi(t *testing.T) {
 			},
 		},
 	}, fq, 1*time.Minute, nil)
-	rr := g.Rules[0].(*rule.RecordingRule)
+	rr := g.Rules[0].(*RecordingRule)
 
-	expectedRes := apiRule{
+	expectedRes := ApiRule{
 		Name:           "record_name",
 		Query:          "up",
 		Labels:         map[string]string{"label": "value"},
 		Health:         "ok",
-		Type:           ruleTypeRecording,
+		Type:           TypeRecording,
 		DatasourceType: "prometheus",
 		ID:             "1248",
 		GroupID:        fmt.Sprintf("%d", g.CreateID()),
 		GroupName:      "group",
 		File:           "rules.yaml",
 		MaxUpdates:     44,
-		Updates:        make([]rule.StateEntry, 0),
+		Updates:        make([]StateEntry, 0),
 	}
 
-	res := recordingToAPI(rr)
+	res := rr.ToAPI()
 
 	if !reflect.DeepEqual(res, expectedRes) {
 		t.Fatalf("expected to have: \n%v;\ngot: \n%v", expectedRes, res)

app/vmalert/web.go

@@ -29,7 +29,9 @@ var (
 		{"api/v1/rules", "list all loaded groups and rules"},
 		{"api/v1/alerts", "list all active alerts"},
 		{"api/v1/notifiers", "list all notifiers"},
-		{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", paramGroupID, paramAlertID), "get alert status by group and alert ID"},
+		{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", rule.ParamGroupID, rule.ParamAlertID), "get alert status by group and alert ID"},
+		{fmt.Sprintf("api/v1/rule?%s=<int>&%s=<int>", rule.ParamGroupID, rule.ParamRuleID), "get rule status by group and rule ID"},
+		{fmt.Sprintf("api/v1/group?%s=<int>", rule.ParamGroupID), "get group status by group ID"},
 	}
 	systemLinks = [][2]string{
 		{"vmalert/groups", "UI"},
@@ -45,8 +47,8 @@ var (
 		{Name: "Docs", URL: "https://docs.victoriametrics.com/victoriametrics/vmalert/"},
 	}
 	ruleTypeMap = map[string]string{
-		"alert":  ruleTypeAlerting,
-		"record": ruleTypeRecording,
+		"alert":  rule.TypeAlerting,
+		"record": rule.TypeRecording,
 	}
 )
 
@@ -112,7 +114,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/rules":
 		// Grafana makes an extra request to `/rules`
 		// handler in addition to `/api/v1/rules` calls in alerts UI
-		var data []*apiGroup
+		var data []*rule.ApiGroup
 		rf, err := newRulesFilter(r)
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
@@ -178,14 +180,14 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/vmalert/api/v1/rule", "/api/v1/rule":
-		rule, err := rh.getRule(r)
+		apiRule, err := rh.getRule(r)
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		rwu := apiRuleWithUpdates{
-			apiRule:      rule,
-			StateUpdates: rule.Updates,
+		rwu := rule.ApiRuleWithUpdates{
+			ApiRule:      apiRule,
+			StateUpdates: apiRule.Updates,
 		}
 		data, err := json.Marshal(rwu)
 		if err != nil {
@@ -195,6 +197,20 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		w.Write(data)
 		return true
+	case "/vmalert/api/v1/group", "/api/v1/group":
+		group, err := rh.getGroup(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		data, err := json.Marshal(group)
+		if err != nil {
+			httpserver.Errorf(w, r, "failed to marshal group: %s", err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(data)
+		return true
 	case "/-/reload":
 		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
@@ -209,30 +225,42 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
-func (rh *requestHandler) getRule(r *http.Request) (apiRule, error) {
-	groupID, err := strconv.ParseUint(r.FormValue(paramGroupID), 10, 64)
+func (rh *requestHandler) getGroup(r *http.Request) (*rule.ApiGroup, error) {
+	groupID, err := strconv.ParseUint(r.FormValue(rule.ParamGroupID), 10, 64)
 	if err != nil {
-		return apiRule{}, fmt.Errorf("failed to read %q param: %w", paramGroupID, err)
+		return nil, fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err)
 	}
-	ruleID, err := strconv.ParseUint(r.FormValue(paramRuleID), 10, 64)
+	obj, err := rh.m.groupAPI(groupID)
 	if err != nil {
-		return apiRule{}, fmt.Errorf("failed to read %q param: %w", paramRuleID, err)
-	}
-	obj, err := rh.m.ruleAPI(groupID, ruleID)
-	if err != nil {
-		return apiRule{}, errResponse(err, http.StatusNotFound)
+		return nil, errResponse(err, http.StatusNotFound)
 	}
 	return obj, nil
 }
 
-func (rh *requestHandler) getAlert(r *http.Request) (*apiAlert, error) {
-	groupID, err := strconv.ParseUint(r.FormValue(paramGroupID), 10, 64)
+func (rh *requestHandler) getRule(r *http.Request) (rule.ApiRule, error) {
+	groupID, err := strconv.ParseUint(r.FormValue(rule.ParamGroupID), 10, 64)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read %q param: %w", paramGroupID, err)
+		return rule.ApiRule{}, fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err)
 	}
-	alertID, err := strconv.ParseUint(r.FormValue(paramAlertID), 10, 64)
+	ruleID, err := strconv.ParseUint(r.FormValue(rule.ParamRuleID), 10, 64)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read %q param: %w", paramAlertID, err)
+		return rule.ApiRule{}, fmt.Errorf("failed to read %q param: %w", rule.ParamRuleID, err)
+	}
+	obj, err := rh.m.ruleAPI(groupID, ruleID)
+	if err != nil {
+		return rule.ApiRule{}, errResponse(err, http.StatusNotFound)
+	}
+	return obj, nil
+}
+
+func (rh *requestHandler) getAlert(r *http.Request) (*rule.ApiAlert, error) {
+	groupID, err := strconv.ParseUint(r.FormValue(rule.ParamGroupID), 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err)
+	}
+	alertID, err := strconv.ParseUint(r.FormValue(rule.ParamAlertID), 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %q param: %w", rule.ParamAlertID, err)
 	}
 	a, err := rh.m.alertAPI(groupID, alertID)
 	if err != nil {
@@ -244,7 +272,7 @@ func (rh *requestHandler) getAlert(r *http.Request) (*apiAlert, error) {
 type listGroupsResponse struct {
 	Status string `json:"status"`
 	Data   struct {
-		Groups []*apiGroup `json:"groups"`
+		Groups []*rule.ApiGroup `json:"groups"`
 	} `json:"data"`
 }
 
@@ -310,19 +338,19 @@ func (rf *rulesFilter) matchesGroup(group *rule.Group) bool {
 	return true
 }
 
-func (rh *requestHandler) groups(rf *rulesFilter) []*apiGroup {
+func (rh *requestHandler) groups(rf *rulesFilter) []*rule.ApiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	groups := make([]*apiGroup, 0)
+	groups := make([]*rule.ApiGroup, 0)
 	for _, group := range rh.m.groups {
 		if !rf.matchesGroup(group) {
 			continue
 		}
-		g := groupToAPI(group)
+		g := group.ToAPI()
 		// the returned list should always be non-nil
 		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
-		filteredRules := make([]apiRule, 0)
+		filteredRules := make([]rule.ApiRule, 0)
 		for _, rule := range g.Rules {
 			if rf.ruleType != "" && rf.ruleType != rule.Type {
 				continue
@@ -350,7 +378,7 @@ func (rh *requestHandler) groups(rf *rulesFilter) []*apiGroup {
 		groups = append(groups, g)
 	}
 	// sort list of groups for deterministic output
-	slices.SortFunc(groups, func(a, b *apiGroup) int {
+	slices.SortFunc(groups, func(a, b *rule.ApiGroup) int {
 		if a.Name != b.Name {
 			return strings.Compare(a.Name, b.Name)
 		}
@@ -375,32 +403,32 @@ func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, error) {
 type listAlertsResponse struct {
 	Status string `json:"status"`
 	Data   struct {
-		Alerts []*apiAlert `json:"alerts"`
+		Alerts []*rule.ApiAlert `json:"alerts"`
 	} `json:"data"`
 }
 
-func (rh *requestHandler) groupAlerts() []groupAlerts {
+func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	var gAlerts []groupAlerts
+	var gAlerts []rule.GroupAlerts
 	for _, g := range rh.m.groups {
-		var alerts []*apiAlert
+		var alerts []*rule.ApiAlert
 		for _, r := range g.Rules {
 			a, ok := r.(*rule.AlertingRule)
 			if !ok {
 				continue
 			}
-			alerts = append(alerts, ruleToAPIAlert(a)...)
+			alerts = append(alerts, a.AlertsToAPI()...)
 		}
 		if len(alerts) > 0 {
-			gAlerts = append(gAlerts, groupAlerts{
-				Group:  groupToAPI(g),
+			gAlerts = append(gAlerts, rule.GroupAlerts{
+				Group:  g.ToAPI(),
 				Alerts: alerts,
 			})
 		}
 	}
-	slices.SortFunc(gAlerts, func(a, b groupAlerts) int {
+	slices.SortFunc(gAlerts, func(a, b rule.GroupAlerts) int {
 		return strings.Compare(a.Group.Name, b.Group.Name)
 	})
 	return gAlerts
@@ -411,7 +439,7 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
 	defer rh.m.groupsMu.RUnlock()
 
 	lr := listAlertsResponse{Status: "success"}
-	lr.Data.Alerts = make([]*apiAlert, 0)
+	lr.Data.Alerts = make([]*rule.ApiAlert, 0)
 	for _, group := range rh.m.groups {
 		if !rf.matchesGroup(group) {
 			continue
@@ -421,12 +449,12 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
 			if !ok {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, ruleToAPIAlert(a)...)
+			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsToAPI()...)
 		}
 	}
 
 	// sort list of alerts for deterministic output
-	slices.SortFunc(lr.Data.Alerts, func(a, b *apiAlert) int {
+	slices.SortFunc(lr.Data.Alerts, func(a, b *rule.ApiAlert) int {
 		return strings.Compare(a.ID, b.ID)
 	})
 
@@ -443,7 +471,7 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
 type listNotifiersResponse struct {
 	Status string `json:"status"`
 	Data   struct {
-		Notifiers []*apiNotifier `json:"notifiers"`
+		Notifiers []*notifier.ApiNotifier `json:"notifiers"`
 	} `json:"data"`
 }
 
@@ -451,19 +479,20 @@ func (rh *requestHandler) listNotifiers() ([]byte, error) {
 	targets := notifier.GetTargets()
 
 	lr := listNotifiersResponse{Status: "success"}
-	lr.Data.Notifiers = make([]*apiNotifier, 0)
+	lr.Data.Notifiers = make([]*notifier.ApiNotifier, 0)
 	for protoName, protoTargets := range targets {
-		notifier := &apiNotifier{
-			Kind:    string(protoName),
-			Targets: make([]*apiTarget, 0, len(protoTargets)),
+		nr := &notifier.ApiNotifier{
+			Kind:    protoName,
+			Targets: make([]*notifier.ApiTarget, 0, len(protoTargets)),
 		}
 		for _, target := range protoTargets {
-			notifier.Targets = append(notifier.Targets, &apiTarget{
-				Address: target.Addr(),
-				Labels:  target.Labels.ToMap(),
+			nr.Targets = append(nr.Targets, &notifier.ApiTarget{
+				Address:   target.Addr(),
+				Labels:    target.Labels.ToMap(),
+				LastError: target.LastError(),
 			})
 		}
-		lr.Data.Notifiers = append(lr.Data.Notifiers, notifier)
+		lr.Data.Notifiers = append(lr.Data.Notifiers, nr)
 	}
 
 	b, err := json.Marshal(lr)

app/vmalert/web.qtpl

@@ -8,6 +8,7 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 ) %}
 
 {% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
@@ -93,7 +94,7 @@
     {%= tpl.Footer(r) %}
 {% endfunc %}
 
-{% func ListGroups(r *http.Request, groups []*apiGroup, filter string) %}
+{% func ListGroups(r *http.Request, groups []*rule.ApiGroup, filter string) %}
     {%code
         prefix := vmalertutil.Prefix(r.URL.Path)
         filters := map[string]string{
@@ -222,7 +223,7 @@
 {% endfunc %}
 
 
-{% func ListAlerts(r *http.Request, groupAlerts []groupAlerts) %}
+{% func ListAlerts(r *http.Request, groupAlerts []rule.GroupAlerts) %}
     {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
     {%= Controls(prefix, "", "", nil, nil, true) %}
@@ -231,7 +232,7 @@
              {%code
                  g := ga.Group
                  var keys []string
-                 alertsByRule := make(map[string][]*apiAlert)
+                 alertsByRule := make(map[string][]*rule.ApiAlert)
                  for _, alert := range ga.Alerts {
                      if len(alertsByRule[alert.RuleID]) < 1 {
                          keys = append(keys, alert.RuleID)
@@ -378,7 +379,7 @@
     {%= tpl.Footer(r) %}
 {% endfunc %}
 
-{% func Alert(r *http.Request, alert *apiAlert) %}
+{% func Alert(r *http.Request, alert *rule.ApiAlert) %}
     {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "", getLastConfigError()) %}
     {%code
@@ -464,7 +465,7 @@
 {% endfunc %}
 
 
-{% func RuleDetails(r *http.Request, rule apiRule) %}
+{% func RuleDetails(r *http.Request, rule rule.ApiRule) %}
     {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "", getLastConfigError()) %}
     {%code
@@ -649,7 +650,7 @@
 <span class="badge bg-warning text-dark" title="This firing state is kept because of `keep_firing_for`">stabilizing</span>
 {% endfunc %}
 
-{% func seriesFetchedWarn(prefix string, r apiRule) %}
+{% func seriesFetchedWarn(prefix string, r rule.ApiRule) %}
 {% if isNoMatch(r) %}
 <svg
     data-bs-toggle="tooltip"
@@ -663,7 +664,7 @@
 {% endfunc %}
 
 {%code
-  func isNoMatch (r apiRule) bool {
+  func isNoMatch (r rule.ApiRule) bool {
     return r.LastSamples == 0 && r.LastSeriesFetched != nil && *r.LastSeriesFetched == 0
   }
 %}

app/vmalert/web_test.go

@@ -25,6 +25,7 @@ func TestHandler(t *testing.T) {
 	m := &manager{groups: map[uint64]*rule.Group{}}
 	var ar *rule.AlertingRule
 	var rr *rule.RecordingRule
+	var groupIDs []uint64
 	for _, dsType := range []string{"prometheus", "", "graphite"} {
 		g := rule.NewGroup(config.Group{
 			Name:        "group",
@@ -45,7 +46,9 @@ func TestHandler(t *testing.T) {
 		ar = g.Rules[0].(*rule.AlertingRule)
 		rr = g.Rules[1].(*rule.RecordingRule)
 		g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
-		m.groups[g.CreateID()] = g
+		id := g.CreateID()
+		m.groups[id] = g
+		groupIDs = append(groupIDs, id)
 	}
 	rh := &requestHandler{m: m}
 
@@ -82,22 +85,22 @@ func TestHandler(t *testing.T) {
 	})
 
 	t.Run("/vmalert/rule", func(t *testing.T) {
-		a := ruleToAPI(ar)
+		a := ar.ToAPI()
 		getResp(t, ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
-		r := ruleToAPI(rr)
+		r := rr.ToAPI()
 		getResp(t, ts.URL+"/vmalert/"+r.WebLink(), nil, 200)
 	})
 	t.Run("/vmalert/alert", func(t *testing.T) {
-		alerts := ruleToAPIAlert(ar)
+		alerts := ar.AlertsToAPI()
 		for _, a := range alerts {
 			getResp(t, ts.URL+"/vmalert/"+a.WebLink(), nil, 200)
 		}
 	})
 	t.Run("/vmalert/rule?badParam", func(t *testing.T) {
-		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramRuleID)
+		params := fmt.Sprintf("?%s=0&%s=1", rule.ParamGroupID, rule.ParamRuleID)
 		getResp(t, ts.URL+"/vmalert/rule"+params, nil, 404)
 
-		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramRuleID)
+		params = fmt.Sprintf("?%s=1&%s=0", rule.ParamGroupID, rule.ParamRuleID)
 		getResp(t, ts.URL+"/vmalert/rule"+params, nil, 404)
 	})
 
@@ -124,14 +127,14 @@ func TestHandler(t *testing.T) {
 		}
 	})
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
-		expAlert := newAlertAPI(ar, ar.GetAlerts()[0])
-		alert := &apiAlert{}
+		expAlert := rule.NewAlertAPI(ar, ar.GetAlerts()[0])
+		alert := &rule.ApiAlert{}
 		getResp(t, ts.URL+"/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
 			t.Fatalf("expected %v is equal to %v", alert, expAlert)
 		}
 
-		alert = &apiAlert{}
+		alert = &rule.ApiAlert{}
 		getResp(t, ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
 			t.Fatalf("expected %v is equal to %v", alert, expAlert)
@@ -139,16 +142,16 @@ func TestHandler(t *testing.T) {
 	})
 
 	t.Run("/api/v1/alert?badParams", func(t *testing.T) {
-		params := fmt.Sprintf("?%s=0&%s=1", paramGroupID, paramAlertID)
+		params := fmt.Sprintf("?%s=0&%s=1", rule.ParamGroupID, rule.ParamAlertID)
 		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 404)
 		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
 
-		params = fmt.Sprintf("?%s=1&%s=0", paramGroupID, paramAlertID)
+		params = fmt.Sprintf("?%s=1&%s=0", rule.ParamGroupID, rule.ParamAlertID)
 		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 404)
 		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 404)
 
 		// bad request, alertID is missing
-		params = fmt.Sprintf("?%s=1", paramGroupID)
+		params = fmt.Sprintf("?%s=1", rule.ParamGroupID)
 		getResp(t, ts.URL+"/api/v1/alert"+params, nil, 400)
 		getResp(t, ts.URL+"/vmalert/api/v1/alert"+params, nil, 400)
 	})
@@ -167,27 +170,42 @@ func TestHandler(t *testing.T) {
 		}
 	})
 	t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
-		expRule := ruleToAPI(ar)
-		gotRule := apiRule{}
+		expRule := ar.ToAPI()
+		gotRule := rule.ApiRule{}
 		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
 			t.Fatalf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
-		gotRule = apiRule{}
+		gotRule = rule.ApiRule{}
 		getResp(t, ts.URL+"/vmalert/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
 			t.Fatalf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
-		gotRuleWithUpdates := apiRuleWithUpdates{}
+		gotRuleWithUpdates := rule.ApiRuleWithUpdates{}
 		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRuleWithUpdates, 200)
 		if len(gotRuleWithUpdates.StateUpdates) < 1 {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})
+	t.Run("/api/v1/group?groupID", func(t *testing.T) {
+		id := groupIDs[0]
+		g := m.groups[id]
+		expGroup := g.ToAPI()
+		gotGroup := rule.ApiGroup{}
+		getResp(t, ts.URL+"/"+expGroup.APILink(), &gotGroup, 200)
+		if expGroup.ID != gotGroup.ID {
+			t.Fatalf("expected to get Group %q; got %q instead", expGroup.ID, gotGroup.ID)
+		}
+		gotGroup = rule.ApiGroup{}
+		getResp(t, ts.URL+"/vmalert/"+expGroup.APILink(), &gotGroup, 200)
+		if expGroup.ID != gotGroup.ID {
+			t.Fatalf("expected to get Group %q; got %q instead", expGroup.ID, gotGroup.ID)
+		}
+	})
 
 	t.Run("/api/v1/rules&filters", func(t *testing.T) {
 		check := func(url string, statusCode, expGroups, expRules int) {

app/vmauth/auth_config.go

@@ -41,6 +41,9 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#load-balancing for details")
 	defaultLoadBalancingPolicy = flag.String("loadBalancingPolicy", "least_loaded", "The default load balancing policy to use for backend urls specified inside url_prefix section. "+
 		"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/victoriametrics/vmauth/#load-balancing")
+	defaultMergeQueryArgs = flagutil.NewArrayString("mergeQueryArgs", "An optional list of client query arg names, which must be merged with args at backend urls. "+
+		"The rest of client query args are replaced by the corresponding query args from backend urls for security reasons; "+
+		"see https://docs.victoriametrics.com/victoriametrics/vmauth/#query-args-handling")
 	discoverBackendIPsGlobal = flag.Bool("discoverBackendIPs", false, "Whether to discover backend IPs via periodic DNS queries to hostnames specified in url_prefix. "+
 		"This may be useful when url_prefix points to a hostname with dynamically scaled instances behind it. See https://docs.victoriametrics.com/victoriametrics/vmauth/#discovering-backend-ips")
 	discoverBackendIPsInterval = flag.Duration("discoverBackendIPsInterval", 10*time.Second, "The interval for re-discovering backend IPs if -discoverBackendIPs command-line flag is set. "+
@@ -75,6 +78,7 @@ type UserInfo struct {
 	DefaultURL             *URLPrefix  `yaml:"default_url,omitempty"`
 	RetryStatusCodes       []int       `yaml:"retry_status_codes,omitempty"`
 	LoadBalancingPolicy    string      `yaml:"load_balancing_policy,omitempty"`
+	MergeQueryArgs         []string    `yaml:"merge_query_args,omitempty"`
 	DropSrcPathPrefixParts *int        `yaml:"drop_src_path_prefix_parts,omitempty"`
 	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 	TLSCertFile            string      `yaml:"tls_cert_file,omitempty"`
@@ -182,6 +186,11 @@ type URLMap struct {
 	// LoadBalancingPolicy is load balancing policy among UrlPrefix backends.
 	LoadBalancingPolicy string `yaml:"load_balancing_policy,omitempty"`
 
+	// MergeQueryArgs is a list of client query args, which must be merged with the existing backend query args.
+	//
+	// The rest of client query args are replaced with the corresponding backend query args for security reasons.
+	MergeQueryArgs []string `yaml:"merge_query_args,omitempty"`
+
 	// DropSrcPathPrefixParts is the number of `/`-delimited request path prefix parts to drop before proxying the request to backend.
 	DropSrcPathPrefixParts *int `yaml:"drop_src_path_prefix_parts,omitempty"`
 }
@@ -228,7 +237,7 @@ func (qa *QueryArg) MarshalYAML() (any, error) {
 	return qa.sOriginal, nil
 }
 
-// URLPrefix represents passed `url_prefix`
+// URLPrefix represents the `url_prefix` from auth config.
 type URLPrefix struct {
 	// requests are re-tried on other backend urls for these http response status codes
 	retryStatusCodes []int
@@ -236,6 +245,11 @@ type URLPrefix struct {
 	// load balancing policy used
 	loadBalancingPolicy string
 
+	// the list of client query args, which must be merged with backend query args.
+	//
+	// By default backend query args replace all the client query args for security reasons.
+	mergeQueryArgs []string
+
 	// how many request path prefix parts to drop before routing the request to backendURL
 	dropSrcPathPrefixParts int
 
@@ -723,14 +737,11 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 }
 
 func parseAuthConfig(data []byte) (*AuthConfig, error) {
-	data, err := envtemplate.ReplaceBytes(data)
-	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
-	}
+	data = envtemplate.ReplaceBytes(data)
 	ac := &AuthConfig{
 		ms: metrics.NewSet(),
 	}
-	if err = yaml.UnmarshalStrict(data, ac); err != nil {
+	if err := yaml.UnmarshalStrict(data, ac); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err)
 	}
 
@@ -859,6 +870,7 @@ func (ui *UserInfo) getMetricLabels() (string, error) {
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
+	mergeQueryArgs := *defaultMergeQueryArgs
 	dropSrcPathPrefixParts := 0
 	discoverBackendIPs := *discoverBackendIPsGlobal
 	if ui.RetryStatusCodes != nil {
@@ -867,6 +879,9 @@ func (ui *UserInfo) initURLs() error {
 	if ui.LoadBalancingPolicy != "" {
 		loadBalancingPolicy = ui.LoadBalancingPolicy
 	}
+	if len(ui.MergeQueryArgs) != 0 {
+		mergeQueryArgs = ui.MergeQueryArgs
+	}
 	if ui.DropSrcPathPrefixParts != nil {
 		dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts
 	}
@@ -874,16 +889,18 @@ func (ui *UserInfo) initURLs() error {
 		discoverBackendIPs = *ui.DiscoverBackendIPs
 	}
 
-	if ui.URLPrefix != nil {
-		if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil {
+	up := ui.URLPrefix
+	if up != nil {
+		if err := up.sanitizeAndInitialize(); err != nil {
 			return err
 		}
-		ui.URLPrefix.retryStatusCodes = retryStatusCodes
-		ui.URLPrefix.dropSrcPathPrefixParts = dropSrcPathPrefixParts
-		ui.URLPrefix.discoverBackendIPs = discoverBackendIPs
-		if err := ui.URLPrefix.setLoadBalancingPolicy(loadBalancingPolicy); err != nil {
+		up.retryStatusCodes = retryStatusCodes
+		up.dropSrcPathPrefixParts = dropSrcPathPrefixParts
+		up.discoverBackendIPs = discoverBackendIPs
+		if err := up.setLoadBalancingPolicy(loadBalancingPolicy); err != nil {
 			return err
 		}
+		up.mergeQueryArgs = mergeQueryArgs
 	}
 	if ui.DefaultURL != nil {
 		if err := ui.DefaultURL.sanitizeAndInitialize(); err != nil {
@@ -902,6 +919,7 @@ func (ui *UserInfo) initURLs() error {
 		}
 		rscs := retryStatusCodes
 		lbp := loadBalancingPolicy
+		mqa := mergeQueryArgs
 		dsp := dropSrcPathPrefixParts
 		dbd := discoverBackendIPs
 		if e.RetryStatusCodes != nil {
@@ -910,6 +928,9 @@ func (ui *UserInfo) initURLs() error {
 		if e.LoadBalancingPolicy != "" {
 			lbp = e.LoadBalancingPolicy
 		}
+		if len(e.MergeQueryArgs) != 0 {
+			mqa = e.MergeQueryArgs
+		}
 		if e.DropSrcPathPrefixParts != nil {
 			dsp = *e.DropSrcPathPrefixParts
 		}
@@ -920,6 +941,7 @@ func (ui *UserInfo) initURLs() error {
 		if err := e.URLPrefix.setLoadBalancingPolicy(lbp); err != nil {
 			return err
 		}
+		e.URLPrefix.mergeQueryArgs = mqa
 		e.URLPrefix.dropSrcPathPrefixParts = dsp
 		e.URLPrefix.discoverBackendIPs = dbd
 	}

app/vmauth/auth_config_test.go

@@ -280,7 +280,7 @@ users:
 }
 
 func TestParseAuthConfigSuccess(t *testing.T) {
-	f := func(s string, expectedAuthConfig map[string]*UserInfo) {
+	f := func(s string, expectedAuthConfig map[string]*UserInfo, expectedUnauthorizedUserConfig *UserInfo) {
 		t.Helper()
 		ac, err := parseAuthConfig([]byte(s))
 		if err != nil {
@@ -294,15 +294,19 @@ func TestParseAuthConfigSuccess(t *testing.T) {
 		if err := areEqualConfigs(m, expectedAuthConfig); err != nil {
 			t.Fatal(err)
 		}
+
+		if err := areEqualConfigs(ac.UnauthorizedUser, expectedUnauthorizedUserConfig); err != nil {
+			t.Fatal(err)
+		}
 	}
 
 	insecureSkipVerifyTrue := true
 
 	// Empty config
-	f(``, map[string]*UserInfo{})
+	f(``, map[string]*UserInfo{}, nil)
 
 	// Empty users
-	f(`users: []`, map[string]*UserInfo{})
+	f(`users: []`, map[string]*UserInfo{}, nil)
 
 	// Single user
 	f(`
@@ -320,7 +324,7 @@ users:
 			MaxConcurrentRequests: 5,
 			TLSInsecureSkipVerify: &insecureSkipVerifyTrue,
 		},
-	})
+	}, nil)
 
 	// Single user with auth_token
 	f(`
@@ -344,7 +348,7 @@ users:
 			TLSCertFile:           "foo/baz",
 			TLSKeyFile:            "foo/foo",
 		},
-	})
+	}, nil)
 
 	// Multiple url_prefix entries
 	insecureSkipVerifyFalse := false
@@ -359,6 +363,7 @@ users:
   tls_insecure_skip_verify: false
   retry_status_codes: [500, 501]
   load_balancing_policy: first_available
+  merge_query_args: [foo, bar]
   drop_src_path_prefix_parts: 1
   discover_backend_ips: true
 `, map[string]*UserInfo{
@@ -372,10 +377,11 @@ users:
 			TLSInsecureSkipVerify:  &insecureSkipVerifyFalse,
 			RetryStatusCodes:       []int{500, 501},
 			LoadBalancingPolicy:    "first_available",
+			MergeQueryArgs:         []string{"foo", "bar"},
 			DropSrcPathPrefixParts: intp(1),
 			DiscoverBackendIPs:     &discoverBackendIPsTrue,
 		},
-	})
+	}, nil)
 
 	// Multiple users
 	f(`
@@ -393,7 +399,7 @@ users:
 			Username:  "bar",
 			URLPrefix: mustParseURL("https://bar/x/"),
 		},
-	})
+	}, nil)
 
 	// non-empty URLMap
 	sharedUserInfo := &UserInfo{
@@ -443,7 +449,7 @@ users:
 `, map[string]*UserInfo{
 		getHTTPAuthBearerToken("foo"):    sharedUserInfo,
 		getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
-	})
+	}, nil)
 
 	// Multiple users with the same name - this should work, since these users have different passwords
 	f(`
@@ -465,7 +471,7 @@ users:
 			Password:  "bar",
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
-	})
+	}, nil)
 
 	// with default url
 	keepOriginalHost := true
@@ -481,6 +487,8 @@ users:
     - "foo: bar"
     - "xxx: y"
     keep_original_host: true
+    load_balancing_policy: first_available
+    merge_query_args: [foo, bar]
   default_url:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
@@ -505,6 +513,8 @@ users:
 						},
 						KeepOriginalHost: &keepOriginalHost,
 					},
+					LoadBalancingPolicy: "first_available",
+					MergeQueryArgs:      []string{"foo", "bar"},
 				},
 			},
 			DefaultURL: mustParseURLs([]string{
@@ -532,6 +542,8 @@ users:
 						},
 						KeepOriginalHost: &keepOriginalHost,
 					},
+					LoadBalancingPolicy: "first_available",
+					MergeQueryArgs:      []string{"foo", "bar"},
 				},
 			},
 			DefaultURL: mustParseURLs([]string{
@@ -539,7 +551,7 @@ users:
 				"http://default2/select/0/prometheus",
 			}),
 		},
-	})
+	}, nil)
 
 	// With metric_labels
 	f(`
@@ -591,6 +603,23 @@ users:
 				},
 			},
 		},
+	}, nil)
+
+	// unauthorized_user
+	f(`
+unauthorized_user:
+  merge_query_args: [extra_filters]
+  url_map:
+  - src_paths: ["/select/.+"]
+    url_prefix: 'http://victoria-logs:9428/?extra_filters={env="prod"}'
+`, nil, &UserInfo{
+		MergeQueryArgs: []string{"extra_filters"},
+		URLMaps: []URLMap{
+			{
+				SrcPaths:  getRegexs([]string{"/select/.+"}),
+				URLPrefix: mustParseURL(`http://victoria-logs:9428/?extra_filters={env="prod"}`),
+			},
+		},
 	})
 }
 
@@ -884,7 +913,7 @@ func removeMetrics(m map[string]*UserInfo) {
 	}
 }
 
-func areEqualConfigs(a, b map[string]*UserInfo) error {
+func areEqualConfigs(a, b any) error {
 	aData, err := yaml.Marshal(a)
 	if err != nil {
 		return fmt.Errorf("cannot marshal a: %w", err)

app/vmauth/main.go

@@ -269,7 +269,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			query.Set("request_path", u.String())
 			targetURL.RawQuery = query.Encode()
 		} else { // Update path for regular routes.
-			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
+			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		}
 
 		wasLocalRetry := false
@@ -372,20 +372,54 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	updateHeadersByConfig(w.Header(), hc.ResponseHeaders)
 	w.WriteHeader(res.StatusCode)
 
-	copyBuf := copyBufPool.Get()
-	copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
-	_, err = io.CopyBuffer(w, res.Body, copyBuf.B)
-	copyBufPool.Put(copyBuf)
+	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-	if err != nil && !netutil.IsTrivialNetworkError(err) {
+	if err != nil && !netutil.IsTrivialNetworkError(err) && !errors.Is(err, context.Canceled) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
+
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
 		return true, false
 	}
 	return true, false
 }
 
+func copyStreamToClient(client io.Writer, backend io.Reader) error {
+	copyBuf := copyBufPool.Get()
+	copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
+	defer copyBufPool.Put(copyBuf)
+	buf := copyBuf.B
+
+	flusher, ok := client.(http.Flusher)
+	if !ok {
+		logger.Panicf("BUG: client must implement net/http.Flusher interface; got %T", client)
+	}
+
+	for {
+		n, backendErr := backend.Read(buf)
+		if n > 0 {
+			data := buf[:n]
+			n, clientErr := client.Write(data)
+			if clientErr != nil {
+				return fmt.Errorf("cannot write data to client: %w", clientErr)
+			}
+			if n != len(data) {
+				logger.Panicf("BUG: unexpected number of bytes written returned by client.Write; got %d; want %d", n, len(data))
+			}
+			// Flush the read data from the backend to the client as fast as possible
+			// in order to reduce delays for data propagation.
+			// See https://github.com/VictoriaMetrics/VictoriaLogs/issues/667
+			flusher.Flush()
+		}
+		if backendErr != nil {
+			if backendErr == io.EOF {
+				return nil
+			}
+			return fmt.Errorf("cannot read data from backend: %w", backendErr)
+		}
+	}
+}
+
 var copyBufPool bytesutil.ByteBufferPool
 
 func copyHeader(dst, src http.Header) {

app/vmauth/main_test.go

@@ -514,6 +514,11 @@ func (w *fakeResponseWriter) getResponse() string {
 	return w.bb.String()
 }
 
+// Flush implements net/http.Flusher
+func (w *fakeResponseWriter) Flush() {
+	// Nothing to do.
+}
+
 func (w *fakeResponseWriter) Header() http.Header {
 	if w.h == nil {
 		w.h = http.Header{}

app/vmauth/target_url.go

@@ -8,29 +8,42 @@ import (
 	"strings"
 )
 
-func mergeURLs(uiURL, requestURI *url.URL, dropSrcPathPrefixParts int) *url.URL {
+func mergeURLs(uiURL, requestURI *url.URL, dropSrcPathPrefixParts int, mergeQueryArgs []string) *url.URL {
 	targetURL := *uiURL
+
 	srcPath := dropPrefixParts(requestURI.Path, dropSrcPathPrefixParts)
 	if strings.HasPrefix(srcPath, "/") {
 		targetURL.Path = strings.TrimSuffix(targetURL.Path, "/")
 	}
 	targetURL.Path += srcPath
 	requestParams := requestURI.Query()
-	// fast path
+
 	if len(requestParams) == 0 {
 		return &targetURL
 	}
-	// merge query parameters from requests.
-	uiParams := targetURL.Query()
+
+	// Merge client query args with backend query args
+	targetParams := targetURL.Query()
+	uiParams := url.Values{}
+
+	// Copy all the target query args
+	for k, v := range targetParams {
+		for i := range v {
+			uiParams.Add(k, v[i])
+		}
+	}
+
+	// Copy the client query args if they do not clash with target args.
 	for k, v := range requestParams {
-		// skip clashed query params from original request
-		if exist := uiParams.Get(k); len(exist) > 0 {
+		if targetParams.Has(k) && !slices.Contains(mergeQueryArgs, k) {
+			// Skip clashed client query params for security reasons
 			continue
 		}
 		for i := range v {
 			uiParams.Add(k, v[i])
 		}
 	}
+
 	targetURL.RawQuery = uiParams.Encode()
 	return &targetURL
 }

app/vmauth/target_url_test.go

@@ -101,7 +101,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			return
 		}
 		bu := up.getBackendURL()
-		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
+		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		bu.put()
 
 		gotTarget := target.String()
@@ -352,7 +352,7 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
 			return
 		}
 		bu := up.getBackendURL()
-		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
+		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		bu.put()
 
 		gotTarget := target.String()
@@ -528,3 +528,43 @@ func (r *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAdd
 func (r *fakeResolver) LookupMX(_ context.Context, _ string) ([]*net.MX, error) {
 	return nil, nil
 }
+
+func TestMergeURLs(t *testing.T) {
+	f := func(clientURL, backendURL string, dropSrcPathPrefixParts int, mergeQueryArgs []string, resultURLExpected string) {
+		t.Helper()
+
+		cu, err := url.Parse(clientURL)
+		if err != nil {
+			t.Fatalf("cannot parse client url %q: %s", clientURL, err)
+		}
+		cu = normalizeURL(cu)
+
+		bu, err := url.Parse(backendURL)
+		if err != nil {
+			t.Fatalf("cannot parse backend url %q: %s", backendURL, err)
+		}
+
+		ru := mergeURLs(bu, cu, dropSrcPathPrefixParts, mergeQueryArgs)
+		resultURL := ru.String()
+		if resultURL != resultURLExpected {
+			t.Fatalf("unexpected resultURL\ngot\n%s\nwant\n%s", resultURL, resultURLExpected)
+		}
+	}
+
+	f("http://foo:1234", "https://backend/foo/bar?baz=abc&de", 0, nil, "https://backend/foo/bar?baz=abc&de")
+	f("http://foo:1234", "https://backend/foo/bar/?baz=abc&de", 0, nil, "https://backend/foo/bar/?baz=abc&de")
+	f("https://foo:1234/", "https://backend/foo/bar?baz=abc&de", 0, nil, "https://backend/foo/bar?baz=abc&de")
+	f("https://foo:1234/", "http://backend:8888/foo/bar/?baz=abc&de", 0, nil, "http://backend:8888/foo/bar/?baz=abc&de")
+
+	// merge paths
+	f("http://foo:1234/x/y?z=xxx", "https://backend/foo/bar?baz=abc&de", 0, nil, "https://backend/foo/bar/x/y?baz=abc&de=&z=xxx")
+
+	// "hacky" url
+	f("http://foo:1234/../../x/../y?z=xxx", "https://backend/foo/bar?baz=abc&de", 0, nil, "https://backend/foo/bar/y?baz=abc&de=&z=xxx")
+
+	// make sure that the client args are overridden by server args by default
+	f("http://foo:1234/x/y?password=hack&qqq=www", "https://backend/foo/bar?password=abc", 0, nil, "https://backend/foo/bar/x/y?password=abc&qqq=www")
+
+	// allow overriding the selected query args
+	f("http://foo:1234/x/y?baz=xxx&qqq=www", "https://backend/foo/bar?baz=abc", 0, []string{"baz"}, "https://backend/foo/bar/x/y?baz=abc&baz=xxx&qqq=www")
+}

app/vmbackup/Makefile

@@ -1,106 +1,110 @@
 # All these commands must run from repository root.
 
+# special tag to reduce resulting binary size
+# See this issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8008
+VMBACKUP_GO_BUILD_TAGS=disable_grpc_modules
+
 vmbackup:
-	APP_NAME=vmbackup $(MAKE) app-local
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-local
 
 vmbackup-race:
-	APP_NAME=vmbackup RACE=-race $(MAKE) app-local
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) RACE=-race $(MAKE) app-local
 
 vmbackup-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker
 
 vmbackup-pure-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-pure
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-pure
 
 vmbackup-linux-amd64-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-linux-amd64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-amd64
 
 vmbackup-linux-arm-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-linux-arm
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-arm
 
 vmbackup-linux-arm64-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-linux-arm64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-arm64
 
 vmbackup-linux-ppc64le-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-linux-ppc64le
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-ppc64le
 
 vmbackup-linux-386-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-linux-386
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-386
 
 vmbackup-darwin-amd64-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-darwin-amd64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-darwin-amd64
 
 vmbackup-darwin-arm64-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-darwin-arm64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-darwin-arm64
 
 vmbackup-freebsd-amd64-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-freebsd-amd64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-freebsd-amd64
 
 vmbackup-openbsd-amd64-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-openbsd-amd64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-openbsd-amd64
 
 vmbackup-windows-amd64-prod:
-	APP_NAME=vmbackup $(MAKE) app-via-docker-windows-amd64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-windows-amd64
 
 package-vmbackup:
-	APP_NAME=vmbackup $(MAKE) package-via-docker
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) package-via-docker
 
 package-vmbackup-pure:
-	APP_NAME=vmbackup $(MAKE) package-via-docker-pure
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) package-via-docker-pure
 
 package-vmbackup-amd64:
-	APP_NAME=vmbackup $(MAKE) package-via-docker-amd64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) package-via-docker-amd64
 
 package-vmbackup-arm:
-	APP_NAME=vmbackup $(MAKE) package-via-docker-arm
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) package-via-docker-arm
 
 package-vmbackup-arm64:
-	APP_NAME=vmbackup $(MAKE) package-via-docker-arm64
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) package-via-docker-arm64
 
 package-vmbackup-ppc64le:
-	APP_NAME=vmbackup $(MAKE) package-via-docker-ppc64le
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) package-via-docker-ppc64le
 
 package-vmbackup-386:
-	APP_NAME=vmbackup $(MAKE) package-via-docker-386
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) package-via-docker-386
 
 publish-vmbackup:
-	APP_NAME=vmbackup $(MAKE) publish-via-docker
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) publish-via-docker
 
 vmbackup-linux-amd64:
-	APP_NAME=vmbackup CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmbackup-linux-arm:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
 vmbackup-linux-arm64:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
 vmbackup-linux-ppc64le:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
 vmbackup-linux-s390x:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
 vmbackup-linux-loong64:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
 
 vmbackup-linux-386:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
 vmbackup-darwin-amd64:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmbackup-darwin-arm64:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
 vmbackup-freebsd-amd64:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmbackup-openbsd-amd64:
-	APP_NAME=vmbackup CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmbackup-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmbackup $(MAKE) app-local-windows-goarch
+	GOARCH=amd64 APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-local-windows-goarch
 
 vmbackup-pure:
-	APP_NAME=vmbackup $(MAKE) app-local-pure
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-local-pure

app/vmbackup/main.go

@@ -115,7 +115,7 @@ func main() {
 	if err != nil {
 		logger.Fatalf("cannot create backup: %s", err)
 	}
-	pushmetrics.Stop()
+	pushmetrics.StopAndPush()
 
 	startTime := time.Now()
 	logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)

app/vmctl/vm/timeseries.go

@@ -63,10 +63,7 @@ func (ts *TimeSeries) write(w io.Writer) (int, error) {
 		// Split long lines with more than 10K samples into multiple JSON lines.
 		// This should limit memory usage at VictoriaMetrics during data ingestion,
 		// since it allocates memory for the whole JSON line and processes it in one go.
-		batchSize := 10000
-		if batchSize > len(timestamps) {
-			batchSize = len(timestamps)
-		}
+		batchSize := min(10000, len(timestamps))
 		timestampsBatch := timestamps[:batchSize]
 		valuesBatch := values[:batchSize]
 		timestamps = timestamps[batchSize:]

app/vmctl/vm_native.go

@@ -121,7 +121,7 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
 		pr := bar.NewProxyReader(reader)
 		if pr != nil {
 			reader = pr
-			fmt.Printf("Continue import process with filter %s:\n", f.String())
+			fmt.Fprintf(log.Writer(), "Continue import process with filter %s:\n", f.String())
 		}
 	}
 
@@ -191,7 +191,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		initParams = []any{srcURL, dstURL, p.filter.String(), tenantID}
 	}
 
-	fmt.Println("") // extra line for better output formatting
+	fmt.Fprintln(log.Writer(), "") // extra line for better output formatting
 	log.Printf(initMessage, initParams...)
 	if len(ranges) > 1 {
 		log.Printf("Selected time range will be split into %d ranges according to %q step", len(ranges), p.filter.Chunk)

app/vminsert/opentelemetry/request_handler.go

@@ -33,7 +33,7 @@ func InsertHandler(req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries) error {
+	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, _ []prompb.MetricMetadata) error {
 		return insertRows(tss, extraLabels)
 	})
 }

app/vminsert/prometheusimport/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
@@ -29,7 +30,7 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, func(rows []prometheus.Row) error {
+	return stream.Parse(req.Body, defaultTimestamp, encoding, true, prommetadata.IsEnabled(), func(rows []prometheus.Row, _ []prometheus.Metadata) error {
 		return insertRows(rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vminsert/promremotewrite/request_handler.go

@@ -23,7 +23,7 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, _ []prompb.MetricMetadata) error {
 		return insertRows(tss, extraLabels)
 	})
 }

app/vmrestore/Makefile

@@ -1,106 +1,110 @@
 # All these commands must run from repository root.
 
+# special tag to reduce resulting binary size
+# See this issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8008
+VMRESTORE_GO_BUILD_TAGS=disable_grpc_modules
+
 vmrestore:
-	APP_NAME=vmrestore $(MAKE) app-local
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-local
 
 vmrestore-race:
-	APP_NAME=vmrestore RACE=-race $(MAKE) app-local
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) RACE=-race $(MAKE) app-local
 
 vmrestore-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker
 
 vmrestore-pure-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-pure
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-pure
 
 vmrestore-linux-amd64-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-linux-amd64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-amd64
 
 vmrestore-linux-arm-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-linux-arm
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-arm
 
 vmrestore-linux-arm64-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-linux-arm64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-arm64
 
 vmrestore-linux-ppc64le-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-linux-ppc64le
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-ppc64le
 
 vmrestore-linux-386-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-linux-386
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-386
 
 vmrestore-darwin-amd64-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-darwin-amd64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-darwin-amd64
 
 vmrestore-darwin-arm64-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-darwin-arm64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-darwin-arm64
 
 vmrestore-freebsd-amd64-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-freebsd-amd64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-freebsd-amd64
 
 vmrestore-openbsd-amd64-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-openbsd-amd64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-openbsd-amd64
 
 vmrestore-windows-amd64-prod:
-	APP_NAME=vmrestore $(MAKE) app-via-docker-windows-amd64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-windows-amd64
 
 package-vmrestore:
-	APP_NAME=vmrestore $(MAKE) package-via-docker
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) package-via-docker
 
 package-vmrestore-pure:
-	APP_NAME=vmrestore $(MAKE) package-via-docker-pure
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) package-via-docker-pure
 
 package-vmrestore-amd64:
-	APP_NAME=vmrestore $(MAKE) package-via-docker-amd64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) package-via-docker-amd64
 
 package-vmrestore-arm:
-	APP_NAME=vmrestore $(MAKE) package-via-docker-arm
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) package-via-docker-arm
 
 package-vmrestore-arm64:
-	APP_NAME=vmrestore $(MAKE) package-via-docker-arm64
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) package-via-docker-arm64
 
 package-vmrestore-ppc64le:
-	APP_NAME=vmrestore $(MAKE) package-via-docker-ppc64le
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) package-via-docker-ppc64le
 
 package-vmrestore-386:
-	APP_NAME=vmrestore $(MAKE) package-via-docker-386
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) package-via-docker-386
 
 publish-vmrestore:
-	APP_NAME=vmrestore $(MAKE) publish-via-docker
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) publish-via-docker
 
 vmrestore-linux-amd64:
-	APP_NAME=vmrestore CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmrestore-linux-arm:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
 
 vmrestore-linux-arm64:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
 vmrestore-linux-ppc64le:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
 vmrestore-linux-s390x:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
 vmrestore-linux-loong64:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
 
 vmrestore-linux-386:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
 vmrestore-darwin-amd64:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmrestore-darwin-arm64:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
 
 vmrestore-freebsd-amd64:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmrestore-openbsd-amd64:
-	APP_NAME=vmrestore CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
 vmrestore-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmrestore $(MAKE) app-local-windows-goarch
+	GOARCH=amd64 APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-local-windows-goarch
 
 vmrestore-pure:
-	APP_NAME=vmrestore $(MAKE) app-local-pure
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-local-pure

app/vmrestore/main.go

@@ -68,7 +68,7 @@ func main() {
 	if err := a.Run(ctx); err != nil {
 		logger.Fatalf("cannot restore from backup: %s", err)
 	}
-	pushmetrics.Stop()
+	pushmetrics.StopAndPush()
 	srcFS.MustStop()
 	dstFS.MustStop()
 

app/vmselect/graphite/eval.go

@@ -17,6 +17,9 @@ import (
 var maxGraphiteSeries = flag.Int("search.maxGraphiteSeries", 300e3, "The maximum number of time series, which can be scanned during queries to Graphite Render API. "+
 	"See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#render-api")
 
+var maxGraphitePathExpressionLen = flag.Int("search.maxGraphitePathExpressionLen", 1024, "The maximum length of pathExpression field in Graphite series. "+
+	"Longer expressions are truncated to prevent memory exhaustion on complex nested queries. Set to 0 to disable truncation.")
+
 type evalConfig struct {
 	startTime   int64
 	endTime     int64
@@ -53,6 +56,21 @@ func (ec *evalConfig) newTimestamps(step int64) []int64 {
 	return timestamps
 }
 
+// safePathExpression creates a pathExpression string from the given expression,
+// truncating it if it exceeds the maximum allowed length to prevent memory exhaustion.
+func safePathExpression(expr graphiteql.Expr) string {
+	if expr == nil {
+		return ""
+	}
+
+	pathExpr := string(expr.AppendString(nil))
+	maxLen := *maxGraphitePathExpressionLen
+	if maxLen > 0 && len(pathExpr) > maxLen {
+		return pathExpr[:maxLen] + "..."
+	}
+	return pathExpr
+}
+
 type series struct {
 	Name       string
 	Tags       map[string]string
@@ -124,6 +142,12 @@ func (s *series) summarize(aggrFunc aggrFunc, startTime, endTime, step int64, xF
 }
 
 func execExpr(ec *evalConfig, query string) (nextSeriesFunc, error) {
+	// Validate query length to prevent memory exhaustion
+	maxLen := searchutil.GetMaxQueryLen()
+	if len(query) > maxLen {
+		return nil, fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen)
+	}
+
 	expr, err := graphiteql.Parse(query)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse %q: %w", query, err)
@@ -169,17 +193,17 @@ func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr g
 				Timestamps:     append([]int64{}, rs.Timestamps...),
 				Values:         append([]float64{}, rs.Values...),
 				expr:           expr,
-				pathExpression: string(expr.AppendString(nil)),
+				pathExpression: safePathExpression(expr),
 			}
 			s.summarize(aggrAvg, ec.startTime, ec.endTime, ec.storageStep, 0)
 			t := timerpool.Get(30 * time.Second)
+			defer timerpool.Put(t)
 			select {
 			case seriesCh <- s:
 			case <-t.C:
 				logger.Errorf("resource leak when processing the %s (full query: %s); please report this error to VictoriaMetrics developers",
 					expr.AppendString(nil), ec.originalQuery)
 			}
-			timerpool.Put(t)
 			return nil
 		})
 		close(seriesCh)

app/vmselect/graphite/eval_test.go

@@ -4070,6 +4070,9 @@ func TestExecExprFailure(t *testing.T) {
 
 	f(`holtWintersConfidenceArea(group(time("foo.baz",15),time("foo.baz",15)))`)
 	f(`holtWintersConfidenceArea()`)
+
+	// too long query
+	f(`sumSeries(` + strings.Repeat("metric.very.long.name.that.takes.space,", 500) + `metric.final)`)
 }
 
 func compareSeries(ss, ssExpected []*series, expr graphiteql.Expr) error {
@@ -4178,3 +4181,170 @@ func formatTimestamps(tss []int64) string {
 	fmt.Fprintf(&sb, " ]")
 	return sb.String()
 }
+
+func TestSafePathExpression(t *testing.T) {
+	// Save original value and restore after test
+	originalMaxLen := *maxGraphitePathExpressionLen
+	defer func() {
+		*maxGraphitePathExpressionLen = originalMaxLen
+	}()
+
+	t.Run("nil expression", func(t *testing.T) {
+		result := safePathExpression(nil)
+		if result != "" {
+			t.Fatalf("expected empty string for nil expression, got: %q", result)
+		}
+	})
+
+	t.Run("short expression - no truncation", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 50
+		expr := &graphiteql.MetricExpr{Query: "metric.cpu.usage"}
+		result := safePathExpression(expr)
+		expected := "metric.cpu.usage"
+		if result != expected {
+			t.Fatalf("expected %q, got %q", expected, result)
+		}
+	})
+
+	t.Run("long expression - with truncation", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 20
+		longQuery := "vertica.metrics.fr4.verticamultitenant-eon.request_resource_consumption.very_long_metric_name"
+		expr := &graphiteql.MetricExpr{Query: longQuery}
+		result := safePathExpression(expr)
+		expectedPrefix := longQuery[:20]
+		expectedSuffix := "..."
+		expected := expectedPrefix + expectedSuffix
+
+		if result != expected {
+			t.Fatalf("expected %q, got %q", expected, result)
+		}
+		if len(result) != 23 { // 20 + 3 for "..."
+			t.Fatalf("expected result length 23, got %d", len(result))
+		}
+		if !strings.HasSuffix(result, "...") {
+			t.Fatalf("expected result to end with '...', got %q", result)
+		}
+	})
+
+	t.Run("truncation disabled", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 0 // Disable truncation
+		longQuery := "very.long.metric.name.that.would.normally.be.truncated.but.should.not.be"
+		expr := &graphiteql.MetricExpr{Query: longQuery}
+		result := safePathExpression(expr)
+
+		if result != longQuery {
+			t.Fatalf("expected full string %q when truncation disabled, got %q", longQuery, result)
+		}
+	})
+
+	t.Run("function expression", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 30
+		// Create a function expression: sum(metric.cpu.usage)
+		args := []*graphiteql.ArgExpr{
+			{Expr: &graphiteql.MetricExpr{Query: "metric.cpu.usage"}},
+		}
+		funcExpr := &graphiteql.FuncExpr{
+			FuncName: "sum",
+			Args:     args,
+		}
+
+		result := safePathExpression(funcExpr)
+		expected := "sum(metric.cpu.usage)"
+
+		if result != expected {
+			t.Fatalf("expected %q, got %q", expected, result)
+		}
+	})
+
+	t.Run("complex nested function - truncated", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 15
+		// Create nested functions: sum(avg(metric.cpu.usage))
+		innerArgs := []*graphiteql.ArgExpr{
+			{Expr: &graphiteql.MetricExpr{Query: "metric.cpu.usage"}},
+		}
+		innerFunc := &graphiteql.FuncExpr{
+			FuncName: "avg",
+			Args:     innerArgs,
+		}
+		outerArgs := []*graphiteql.ArgExpr{
+			{Expr: innerFunc},
+		}
+		outerFunc := &graphiteql.FuncExpr{
+			FuncName: "sum",
+			Args:     outerArgs,
+		}
+
+		result := safePathExpression(outerFunc)
+
+		if len(result) != 18 { // 15 + 3 for "..."
+			t.Fatalf("expected result length 18, got %d", len(result))
+		}
+		if !strings.HasSuffix(result, "...") {
+			t.Fatalf("expected result to end with '...', got %q", result)
+		}
+		if !strings.HasPrefix(result, "sum(avg(metric") {
+			t.Fatalf("expected result to start with 'sum(avg(metric', got %q", result)
+		}
+	})
+
+	t.Run("boundary case - exact length", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 10
+		expr := &graphiteql.MetricExpr{Query: "metric.cpu"} // Exactly 10 characters
+		result := safePathExpression(expr)
+		expected := "metric.cpu"
+
+		if result != expected {
+			t.Fatalf("expected %q, got %q", expected, result)
+		}
+	})
+
+	t.Run("boundary case - one character over", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 10
+		expr := &graphiteql.MetricExpr{Query: "metric.cpu.x"} // 11 characters
+		result := safePathExpression(expr)
+		expected := "metric.cpu..."
+
+		if result != expected {
+			t.Fatalf("expected %q, got %q", expected, result)
+		}
+	})
+}
+
+func TestSafePathExpressionFromString(t *testing.T) {
+	// Save original value and restore after test
+	originalMaxLen := *maxGraphitePathExpressionLen
+	defer func() {
+		*maxGraphitePathExpressionLen = originalMaxLen
+	}()
+
+	t.Run("short string - no truncation", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 50
+		input := "sumSeries(metric1,metric2)"
+		result := safePathExpressionFromString(input)
+
+		if result != input {
+			t.Fatalf("expected %q, got %q", input, result)
+		}
+	})
+
+	t.Run("long string - with truncation", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 20
+		input := "sumSeries(very.long.metric.name.that.exceeds.limit,another.metric)"
+		result := safePathExpressionFromString(input)
+		expected := "sumSeries(very.long...."
+
+		if result != expected {
+			t.Fatalf("expected %q, got %q", expected, result)
+		}
+	})
+
+	t.Run("truncation disabled", func(t *testing.T) {
+		*maxGraphitePathExpressionLen = 0
+		input := "very.long.string.that.would.normally.be.truncated"
+		result := safePathExpressionFromString(input)
+
+		if result != input {
+			t.Fatalf("expected full string when truncation disabled, got %q", result)
+		}
+	})
+}

app/vmselect/graphite/render_api_test.go

@@ -10,7 +10,7 @@ func TestParseIntervalSuccess(t *testing.T) {
 		t.Helper()
 		interval, err := parseInterval(s)
 		if err != nil {
-			t.Fatalf("unexpected error in parseInterva(%q): %s", s, err)
+			t.Fatalf("unexpected error in parseInterval(%q): %s", s, err)
 		}
 		if interval != intervalExpected {
 			t.Fatalf("unexpected result for parseInterval(%q); got %d; want %d", s, interval, intervalExpected)

app/vmselect/graphite/transform.go

@@ -17,6 +17,16 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 )
 
+// safePathExpressionFromString truncates a pathExpression string if it exceeds
+// the maximum allowed length to prevent memory exhaustion.
+func safePathExpressionFromString(pathExpr string) string {
+	maxLen := *maxGraphitePathExpressionLen
+	if maxLen > 0 && len(pathExpr) > maxLen {
+		return pathExpr[:maxLen] + "..."
+	}
+	return pathExpr
+}
+
 // nextSeriesFunc must return the next series to process.
 //
 // nextSeriesFunc must release all the occupied resources before returning non-nil error.
@@ -319,7 +329,7 @@ func aggregateSeries(ec *evalConfig, expr graphiteql.Expr, nextSeries nextSeries
 		Tags:           tags,
 		Timestamps:     ec.newTimestamps(step),
 		Values:         as.Finalize(xFilesFactor),
-		pathExpression: name,
+		pathExpression: safePathExpressionFromString(name),
 		expr:           expr,
 		step:           step,
 	}
@@ -1124,7 +1134,7 @@ func constantLine(ec *evalConfig, expr graphiteql.Expr, n float64) nextSeriesFun
 		Timestamps:     []int64{ec.startTime, ec.startTime + step, ec.startTime + 2*step},
 		Values:         []float64{n, n, n},
 		expr:           expr,
-		pathExpression: string(expr.AppendString(nil)),
+		pathExpression: safePathExpression(expr),
 		step:           step,
 	}
 	return singleSeriesFunc(s)
@@ -1208,10 +1218,7 @@ func transformDelay(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, er
 		values := s.Values
 		stepsLocal := steps
 		if stepsLocal < 0 {
-			stepsLocal = -stepsLocal
-			if stepsLocal > len(values) {
-				stepsLocal = len(values)
-			}
+			stepsLocal = min(-stepsLocal, len(values))
 			copy(values, values[stepsLocal:])
 			for i := len(values) - 1; i >= len(values)-stepsLocal; i-- {
 				values[i] = nan
@@ -4653,20 +4660,14 @@ func transformSubstr(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, e
 		if start > len(splitName) {
 			start = len(splitName)
 		} else if start < 0 {
-			start = len(splitName) + start
-			if start < 0 {
-				start = 0
-			}
+			start = max(len(splitName)+start, 0)
 		}
 		if stop == 0 {
 			stop = len(splitName)
 		} else if stop > len(splitName) {
 			stop = len(splitName)
 		} else if stop < 0 {
-			stop = len(splitName) + stop
-			if stop < 0 {
-				stop = 0
-			}
+			stop = max(len(splitName)+stop, 0)
 		}
 		if stop < start {
 			stop = start

app/vmselect/graphiteql/lexer_test.go

@@ -17,7 +17,7 @@ func TestScanStringSuccess(t *testing.T) {
 			t.Fatalf("unexpected string scanned from %s; got %s; want %s", s, result, sExpected)
 		}
 		if !strings.HasPrefix(s, result) {
-			t.Fatalf("invalid prefix for scanne string %s: %s", s, result)
+			t.Fatalf("invalid prefix for scanned string %s: %s", s, result)
 		}
 	}
 	f(`""`, `""`)

app/vmselect/graphiteql/parser.go

@@ -210,7 +210,7 @@ func (p *parser) parseMetricExprOrFuncCall() (Expr, error) {
 		}
 		return fe, nil
 	default:
-		// Metric epxression or bool expression or None.
+		// Metric expression or bool expression or None.
 		if isBool(ident) {
 			be := &BoolExpr{
 				B: strings.EqualFold(ident, "true"),

app/vmselect/main.go

@@ -2,6 +2,7 @@ package vmselect
 
 import (
 	"embed"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"net/http"
@@ -10,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
@@ -17,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/stats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -26,7 +30,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -48,13 +51,10 @@ var (
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
 
 func getDefaultMaxConcurrentRequests() int {
-	n := cgroup.AvailableCPUs() * 2
-	if n > 16 {
-		// A single request can saturate all the CPU cores, so there is no sense
-		// in allowing higher number of concurrent requests - they will just contend
-		// for unavailable CPU time.
-		n = 16
-	}
+	// A single request can saturate all the CPU cores, so there is no sense
+	// in allowing higher number of concurrent requests - they will just contend
+	// for unavailable CPU time.
+	n := min(cgroup.AvailableCPUs()*2, 16)
 	return n
 }
 
@@ -67,6 +67,7 @@ func Init() {
 	prometheus.InitMaxUniqueTimeseries(*maxConcurrentRequests)
 
 	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
+	initVMUIConfig()
 	initVMAlertProxy()
 }
 
@@ -128,10 +129,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
-		d := searchutil.GetMaxQueryDuration(r)
-		if d > *maxQueueDuration {
-			d = *maxQueueDuration
-		}
+		d := min(searchutil.GetMaxQueryDuration(r), *maxQueueDuration)
 		t := timerpool.Get(d)
 		select {
 		case concurrencyLimitCh <- struct{}{}:
@@ -460,6 +458,11 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	}
 	if strings.HasPrefix(path, "/vmui/") {
+		if path == "/vmui/config.json" {
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprint(w, vmuiConfig)
+			return true
+		}
 		if strings.HasPrefix(path, "/vmui/static/") {
 			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
 			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
@@ -734,8 +737,40 @@ func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
 var (
 	vmalertProxyHost string
 	vmalertProxy     *nethttputil.ReverseProxy
+	vmuiConfig       string
 )
 
+func initVMUIConfig() {
+	var cfg struct {
+		Version string `json:"version"`
+		License struct {
+			Type string `json:"type"`
+		} `json:"license"`
+		VMAlert struct {
+			Enabled bool `json:"enabled"`
+		} `json:"vmalert"`
+	}
+	data, err := vmuiFiles.ReadFile("vmui/config.json")
+	if err != nil {
+		logger.Fatalf("cannot read vmui default config: %s", err)
+	}
+	err = json.Unmarshal(data, &cfg)
+	if err != nil {
+		logger.Fatalf("cannot parse vmui default config: %s", err)
+	}
+	cfg.Version = buildinfo.ShortVersion()
+	if cfg.Version == "" {
+		// buildinfo.ShortVersion() may return empty result for builds without tags
+		cfg.Version = buildinfo.Version
+	}
+	cfg.VMAlert.Enabled = len(*vmalertProxyURL) != 0
+	data, err = json.Marshal(&cfg)
+	if err != nil {
+		logger.Fatalf("cannot create vmui config: %s", err)
+	}
+	vmuiConfig = string(data)
+}
+
 // initVMAlertProxy must be called after flag.Parse(), since it uses command-line flags.
 func initVMAlertProxy() {
 	if len(*vmalertProxyURL) == 0 {

app/vmselect/netstorage/netstorage.go

@@ -203,10 +203,7 @@ var defaultMaxWorkersPerQuery = func() int {
 	// for processing an average query, without significant impact on inter-CPU communications.
 	const maxWorkersLimit = 32
 
-	n := gomaxprocs
-	if n > maxWorkersLimit {
-		n = maxWorkersLimit
-	}
+	n := min(gomaxprocs, maxWorkersLimit)
 	return n
 }()
 
@@ -269,7 +266,7 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	}
 
 	// Slow path - spin up multiple local workers for parallel data processing.
-	// Do not use global workers pool, since it increases inter-CPU memory ping-poing,
+	// Do not use global workers pool, since it increases inter-CPU memory ping-pong,
 	// which reduces the scalability on systems with many CPU cores.
 
 	// Prepare the work for workers.
@@ -279,10 +276,7 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	}
 
 	// Prepare worker channels.
-	workers := len(tsws)
-	if workers > maxWorkers {
-		workers = maxWorkers
-	}
+	workers := min(len(tsws), maxWorkers)
 	itemsPerWorker := (len(tsws) + workers - 1) / workers
 	workChs := make([]chan *timeseriesWork, workers)
 	for i := range workChs {
@@ -485,7 +479,7 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 	}
 
 	// Slow path - spin up multiple local workers for parallel data unpacking.
-	// Do not use global workers pool, since it increases inter-CPU memory ping-poing,
+	// Do not use global workers pool, since it increases inter-CPU memory ping-pong,
 	// which reduces the scalability on systems with many CPU cores.
 
 	// Prepare the work for workers.
@@ -497,10 +491,7 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 	}
 
 	// Prepare worker channels.
-	workers := len(upws)
-	if workers > gomaxprocs {
-		workers = gomaxprocs
-	}
+	workers := min(len(upws), gomaxprocs)
 	if workers < 1 {
 		workers = 1
 	}
@@ -1153,10 +1144,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	// metricNamesBuf is used for holding all the loaded unique metric names at m and orderedMetricNames.
 	// It should reduce pressure on Go GC by reducing the number of string allocations
 	// when constructing metricName string from byte slice.
-	metricNamesBufCap := maxSeriesCount * 100
-	if metricNamesBufCap > maxFastAllocBlockSize {
-		metricNamesBufCap = maxFastAllocBlockSize
-	}
+	metricNamesBufCap := min(maxSeriesCount*100, maxFastAllocBlockSize)
 	metricNamesBuf := make([]byte, 0, metricNamesBufCap)
 
 	// brssPool is used for holding all the blockRefs objects across all the loaded time series.
@@ -1165,10 +1153,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 
 	// brsPool is used for holding the most of blockRefs.brs slices across all the loaded time series.
 	// It should reduce pressure on Go GC by reducing the number of allocations for blockRefs.brs slices.
-	brsPoolCap := uintptr(maxSeriesCount)
-	if brsPoolCap > maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}) {
-		brsPoolCap = maxFastAllocBlockSize / unsafe.Sizeof(blockRef{})
-	}
+	brsPoolCap := min(uintptr(maxSeriesCount), maxFastAllocBlockSize/unsafe.Sizeof(blockRef{}))
 	brsPool := make([]blockRef, 0, brsPoolCap)
 
 	// m maps from metricName to the index of blockRefs inside brssPool

app/vmselect/netstorage/tmp_blocks_file.go

@@ -135,7 +135,7 @@ func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	return addr, nil
 }
 
-// Len() returnt tbf size in bytes.
+// Len() return tbf size in bytes.
 func (tbf *tmpBlocksFile) Len() uint64 {
 	return tbf.offset
 }

app/vmselect/prometheus/prometheus.go

@@ -24,7 +24,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -38,7 +37,6 @@ var (
 	latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the collection. "+
 		"It can be overridden on per-query basis via latency_offset arg. "+
 		"Too small value can result in incomplete last points for query results")
-	maxQueryLen = flagutil.NewBytes("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
 	maxLookback = flag.Duration("search.maxLookback", 0, "Synonym to -query.lookback-delta from Prometheus. "+
 		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. "+
 		"See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons")
@@ -733,8 +731,9 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		step = defaultStep
 	}
 
-	if len(query) > maxQueryLen.IntN() {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
+	maxLen := searchutil.GetMaxQueryLen()
+	if len(query) > maxLen {
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen)
 	}
 	etfs, err := searchutil.GetExtraTagFilters(r)
 	if err != nil {
@@ -904,8 +903,9 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	}
 
 	// Validate input args.
-	if len(query) > maxQueryLen.IntN() {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
+	maxLen := searchutil.GetMaxQueryLen()
+	if len(query) > maxLen {
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen)
 	}
 	if start > end {
 		end = start + defaultStep
@@ -1089,12 +1089,9 @@ func getRoundDigits(r *http.Request) int {
 }
 
 func getLatencyOffsetMilliseconds(r *http.Request) (int64, error) {
-	d := latencyOffset.Milliseconds()
-	if d < 0 {
-		// Zero latency offset may be useful for some use cases.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2061#issuecomment-1299109836
-		d = 0
-	}
+	// Zero latency offset may be useful for some use cases.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2061#issuecomment-1299109836
+	d := max(latencyOffset.Milliseconds(), 0)
 	return httputil.GetDuration(r, "latency_offset", d)
 }
 

app/vmselect/promql/aggr.go

@@ -161,11 +161,8 @@ func aggrFuncAny(afa *aggrFuncArg) ([]*timeseries, error) {
 	afe := func(tss []*timeseries, _ *metricsql.ModifierExpr) []*timeseries {
 		return tss[:1]
 	}
-	limit := afa.ae.Limit
-	if limit > 1 {
-		// Only a single time series per group must be returned
-		limit = 1
-	}
+	// Only a single time series per group must be returned
+	limit := min(afa.ae.Limit, 1)
 	return aggrFuncExt(afe, tss, &afa.ae.Modifier, limit, true)
 }
 

app/vmselect/promql/binary_op.go

@@ -188,7 +188,7 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			rightValues := right[i].Values
 			dstValues := dst[i].Values
 			if len(leftValues) != len(rightValues) || len(leftValues) != len(dstValues) {
-				logger.Panicf("BUG: len(leftVaues) must match len(rightValues) and len(dstValues); got %d vs %d vs %d",
+				logger.Panicf("BUG: len(leftValues) must match len(rightValues) and len(dstValues); got %d vs %d vs %d",
 					len(leftValues), len(rightValues), len(dstValues))
 			}
 			for j, a := range leftValues {

app/vmselect/promql/eval.go

@@ -1002,10 +1002,7 @@ func getKeepMetricNames(expr metricsql.Expr) bool {
 }
 
 func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64, workerID uint) ([]float64, []int64)) {
-	workers := netstorage.MaxWorkers()
-	if workers > len(tss) {
-		workers = len(tss)
-	}
+	workers := min(netstorage.MaxWorkers(), len(tss))
 	seriesPerWorker := (len(tss) + workers - 1) / workers
 	workChs := make([]chan *timeseries, workers)
 	for i := range workChs {
@@ -1079,10 +1076,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		return evalRollupFuncNoCache(qt, ecCopy, funcName, rf, expr, me, iafc, window, pointsPerSeries)
 	}
 	tooBigOffset := func(offset int64) bool {
-		maxOffset := window / 2
-		if maxOffset > 1800*1000 {
-			maxOffset = 1800 * 1000
-		}
+		maxOffset := min(window/2, 1800*1000)
 		return offset >= maxOffset
 	}
 	deleteCachedSeries := func(qt *querytracer.Tracer) {
@@ -1156,15 +1150,23 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		}
 		qt.Printf("optimized calculation for instant rollup avg_over_time(m[d]) as (sum_over_time(m[d]) / count_over_time(m[d]))")
 		fe := expr.(*metricsql.FuncExpr)
-		feSum := *fe
-		feSum.Name = "sum_over_time"
-		feCount := *fe
-		feCount.Name = "count_over_time"
+		// copy RollupExpr to drop possible offset,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
+		newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
+		newArg.Offset = nil
 		be := &metricsql.BinaryOpExpr{
 			Op:              "/",
 			KeepMetricNames: fe.KeepMetricNames,
-			Left:            &feSum,
-			Right:           &feCount,
+			Left: &metricsql.FuncExpr{
+				Name:            "sum_over_time",
+				Args:            []metricsql.Expr{newArg},
+				KeepMetricNames: fe.KeepMetricNames,
+			},
+			Right: &metricsql.FuncExpr{
+				Name:            "count_over_time",
+				Args:            []metricsql.Expr{newArg},
+				KeepMetricNames: fe.KeepMetricNames,
+			},
 		}
 		return evalExpr(qt, ec, be)
 	case "rate":
@@ -1178,8 +1180,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			fe := afe.Args[0].(*metricsql.FuncExpr)
 			feIncrease := *fe
 			feIncrease.Name = "increase"
-			re := fe.Args[0].(*metricsql.RollupExpr)
-			d := re.Window.Duration(ec.Step)
+			// copy RollupExpr to drop possible offset,
+			// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
+			newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
+			newArg.Offset = nil
+			feIncrease.Args = []metricsql.Expr{newArg}
+			d := newArg.Window.Duration(ec.Step)
 			if d == 0 {
 				d = ec.Step
 			}
@@ -1199,8 +1205,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		fe := expr.(*metricsql.FuncExpr)
 		feIncrease := *fe
 		feIncrease.Name = "increase"
-		re := fe.Args[0].(*metricsql.RollupExpr)
-		d := re.Window.Duration(ec.Step)
+		// copy RollupExpr to drop possible offset,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
+		newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
+		newArg.Offset = nil
+		feIncrease.Args = []metricsql.Expr{newArg}
+		d := newArg.Window.Duration(ec.Step)
 		if d == 0 {
 			d = ec.Step
 		}
@@ -2005,3 +2015,23 @@ func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]flo
 	}
 	return dstValues, dstTimestamps
 }
+
+func copyRollupExpr(re *metricsql.RollupExpr) *metricsql.RollupExpr {
+	var newRe metricsql.RollupExpr
+	newRe.Expr = re.Expr
+	newRe.InheritStep = re.InheritStep
+	newRe.At = re.At
+	if re.Window != nil {
+		newRe.Window = &metricsql.DurationExpr{}
+		*newRe.Window = *re.Window
+	}
+	if re.Offset != nil {
+		newRe.Offset = &metricsql.DurationExpr{}
+		*newRe.Offset = *re.Offset
+	}
+	if re.Step != nil {
+		newRe.Step = &metricsql.DurationExpr{}
+		*newRe.Step = *re.Step
+	}
+	return &newRe
+}

app/vmselect/promql/eval_test.go

@@ -55,7 +55,7 @@ func TestValidateMaxPointsPerSeriesFailure(t *testing.T) {
 	f := func(start, end, step int64, maxPoints int) {
 		t.Helper()
 		if err := ValidateMaxPointsPerSeries(start, end, step, maxPoints); err == nil {
-			t.Fatalf("expecint non-nil error for ValidateMaxPointsPerSeries(start=%d, end=%d, step=%d, maxPoints=%d)", start, end, step, maxPoints)
+			t.Fatalf("expecting non-nil error for ValidateMaxPointsPerSeries(start=%d, end=%d, step=%d, maxPoints=%d)", start, end, step, maxPoints)
 		}
 	}
 	// zero step

app/vmselect/promql/rollup.go

@@ -820,17 +820,11 @@ func seekFirstTimestampIdxAfter(timestamps []int64, seekTimestamp int64, nHint i
 	if len(timestamps) == 0 || timestamps[0] > seekTimestamp {
 		return 0
 	}
-	startIdx := nHint - 2
-	if startIdx < 0 {
-		startIdx = 0
-	}
+	startIdx := max(nHint-2, 0)
 	if startIdx >= len(timestamps) {
 		startIdx = len(timestamps) - 1
 	}
-	endIdx := nHint + 2
-	if endIdx > len(timestamps) {
-		endIdx = len(timestamps)
-	}
+	endIdx := min(nHint+2, len(timestamps))
 	if startIdx > 0 && timestamps[startIdx] <= seekTimestamp {
 		timestamps = timestamps[startIdx:]
 		endIdx -= startIdx
@@ -2443,13 +2437,14 @@ func rollupFake(_ *rollupFuncArg) float64 {
 	return 0
 }
 
+// getScalar expects result from a [scalar](https://prometheus.io/docs/prometheus/latest/querying/basics/#expression-language-data-types).
 func getScalar(arg any, argNum int) ([]float64, error) {
 	ts, ok := arg.([]*timeseries)
 	if !ok {
-		return nil, fmt.Errorf(`unexpected type for arg #%d; got %T; want %T`, argNum+1, arg, ts)
+		return nil, fmt.Errorf(`arg #%d must be a scalar`, argNum+1)
 	}
 	if len(ts) != 1 {
-		return nil, fmt.Errorf(`arg #%d must contain a single timeseries; got %d timeseries`, argNum+1, len(ts))
+		return nil, fmt.Errorf(`arg #%d must be a scalar`, argNum+1)
 	}
 	return ts[0].Values, nil
 }
@@ -2466,14 +2461,15 @@ func getIntNumber(arg any, argNum int) (int, error) {
 	return n, nil
 }
 
+// getString expects result from a string expression, which contains a single timeseries with only NaN values.
 func getString(tss []*timeseries, argNum int) (string, error) {
 	if len(tss) != 1 {
-		return "", fmt.Errorf(`arg #%d must contain a single timeseries; got %d timeseries`, argNum+1, len(tss))
+		return "", fmt.Errorf(`arg #%d must be a string`, argNum+1)
 	}
 	ts := tss[0]
 	for _, v := range ts.Values {
 		if !math.IsNaN(v) {
-			return "", fmt.Errorf(`arg #%d contains non-string timeseries`, argNum+1)
+			return "", fmt.Errorf(`arg #%d must be a string`, argNum+1)
 		}
 	}
 	return string(ts.MetricName.MetricGroup), nil

app/vmselect/promql/transform.go

@@ -903,7 +903,6 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 
 	// Convert buckets with `vmrange` labels to buckets with `le` labels.
 	tss := vmrangeBucketsToLE(args[1])
-
 	// Parse boundsLabel. See https://github.com/prometheus/prometheus/issues/5706 for details.
 	var boundsLabel string
 	if len(args) > 2 {
@@ -1050,9 +1049,15 @@ func fixBrokenBuckets(i int, xss []leTimeseries) {
 		return
 	}
 
+	vNext := xss[0].ts.Values[i]
+	// Set the lowest bucket to 0 if its value is NaN, so it can be properly
+	// compared with upper buckets in the loop below.
+	if math.IsNaN(vNext) {
+		vNext = 0
+		xss[0].ts.Values[i] = vNext
+	}
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
 	// or are bigger than the lower bucket values.
-	vNext := xss[0].ts.Values[i]
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
 		if math.IsNaN(v) || vNext > v {

app/vmselect/promql/transform_test.go

@@ -37,6 +37,9 @@ func TestFixBrokenBuckets(t *testing.T) {
 	f([]float64{5, 1, 2, 3, nan}, []float64{5, 5, 5, 5, 5})
 	f([]float64{1, 5, 2, nan, 6, 3}, []float64{1, 5, 5, 5, 6, 6})
 	f([]float64{5, 10, 4, 3}, []float64{5, 10, 10, 10})
+	f([]float64{nan, 2, nan, 5}, []float64{0, 2, 2, 5})
+	f([]float64{nan, nan, 4, 5}, []float64{0, 0, 4, 5})
+	f([]float64{nan, nan, nan, 4}, []float64{0, 0, 0, 4})
 }
 
 func TestFixBrokenBucketsMultipleValues(t *testing.T) {
@@ -44,12 +47,11 @@ func TestFixBrokenBucketsMultipleValues(t *testing.T) {
 		t.Helper()
 		xss := make([]leTimeseries, len(values))
 		for i, v := range values {
-
 			xss[i].ts = &timeseries{
 				Values: v,
 			}
 		}
-		for i := range len(values) - 1 {
+		for i := range len(values[0]) {
 			fixBrokenBuckets(i, xss)
 		}
 		result := make([][]float64, len(values))
@@ -61,6 +63,8 @@ func TestFixBrokenBucketsMultipleValues(t *testing.T) {
 		}
 	}
 	f([][]float64{{10, 1}, {11, 2}, {13, 3}}, [][]float64{{10, 1}, {11, 2}, {13, 3}})
+	f([][]float64{{nan, nan}, {11, 2}, {13, 3}}, [][]float64{{0, 0}, {11, 2}, {13, 3}})
+	f([][]float64{{nan, nan, nan}, {11, 2, 3}, {13, 3, 4}}, [][]float64{{0, 0, 0}, {11, 2, 3}, {13, 3, 4}})
 }
 
 func TestVmrangeBucketsToLE(t *testing.T) {

app/vmselect/searchutil/searchutil.go

@@ -7,10 +7,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metricsql"
 )
 
 var (
@@ -20,6 +22,7 @@ var (
 	maxStatusRequestDuration = flag.Duration("search.maxStatusRequestDuration", time.Minute*5, "The maximum duration for /api/v1/status/* requests")
 	maxLabelsAPIDuration     = flag.Duration("search.maxLabelsAPIDuration", time.Second*5, "The maximum duration for /api/v1/labels, /api/v1/label/.../values and /api/v1/series requests. "+
 		"See also -search.maxLabelsAPISeries and -search.ignoreExtraFiltersAtLabelsAPI")
+	maxQueryLen = flagutil.NewBytes("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
 )
 
 // GetMaxQueryDuration returns the maximum duration for query from r.
@@ -227,3 +230,8 @@ func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) {
 	dst.IsRegexp = src.IsRegexp
 	dst.IsNegative = src.IsNegative
 }
+
+// GetMaxQueryLen returns the current value of the search.maxQueryLen flag.
+func GetMaxQueryLen() int {
+	return maxQueryLen.IntN()
+}

app/vmselect/vmui/assets/MetricsQL-BhDn4Z6i.md

@@ -21,7 +21,7 @@ However, there are some [intentional differences](https://medium.com/@romanhavro
 
 [Standalone MetricsQL package](https://godoc.org/github.com/VictoriaMetrics/metricsql) can be used for parsing MetricsQL in external apps.
 
-If you are unfamiliar with PromQL, then it is suggested reading [this tutorial for beginners](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085)
+If you are unfamiliar with PromQL, we suggest reading [this tutorial for beginners](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085)
 and introduction into [basic querying via MetricsQL](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#metricsql).
 
 The following functionality is implemented differently in MetricsQL compared to PromQL. This improves user experience:
@@ -69,13 +69,13 @@ The list of MetricsQL features on top of PromQL:
   See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#selecting-graphite-metrics).
   VictoriaMetrics can be used as Graphite datasource in Grafana. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#graphite-api-usage) for details.
   See also [label_graphite_group](#label_graphite_group) function, which can be used for extracting the given groups from Graphite metric name.
-* Lookbehind window in square brackets for [rollup functions](#rollup-functions) may be omitted. VictoriaMetrics automatically selects the lookbehind window
+* The lookbehind window in square brackets for [rollup functions](#rollup-functions) may be omitted. VictoriaMetrics automatically selects the lookbehind window
   depending on the `step` query arg passed to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query)
   and the real interval between [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) (aka `scrape_interval`).
   For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`.
   It is roughly equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
   The difference is documented in [rate() docs](#rate).
-* Numeric values can contain `_` delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
+* Numeric values may include underscore delimiters for better readability. For example, `1_234_567_890` can be used in queries instead of `1234567890`.
 * [Series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) accept multiple `or` filters. For example, `{env="prod",job="a" or env="dev",job="b"}`
   selects series with `{env="prod",job="a"}` or `{env="dev",job="b"}` labels.
   See [these docs](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering-by-multiple-or-filters) for details.
@@ -111,8 +111,8 @@ The list of MetricsQL features on top of PromQL:
 * Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
   It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
   Additionally, the following escape sequences are supported:
-  - `\xXX`, where `XX` is hexadecimal representation of the escaped ascii char.
-  - `\uXXXX`, where `XXXX` is a hexadecimal representation of the escaped unicode char.
+  * `\xXX`, where `XX` is hexadecimal representation of the escaped ascii char.
+  * `\uXXXX`, where `XXXX` is a hexadecimal representation of the escaped unicode char.
 * Aggregate functions support optional `limit N` suffix in order to limit the number of output series.
   For example, `sum(x) by (y) limit 3` limits the number of output time series after the aggregation to 3.
   All the other time series are dropped.
@@ -138,8 +138,9 @@ This may result in `duplicate time series` error when the function is applied to
 This error can be fixed by applying `keep_metric_names` modifier to the function or binary operator.
 
 For example:
-- `rate({__name__=~"foo|bar"}) keep_metric_names` leaves `foo` and `bar` metric names in the returned time series.
-- `({__name__=~"foo|bar"} / 10) keep_metric_names` leaves `foo` and `bar` metric names in the returned time series.
+
+* `rate({__name__=~"foo|bar"}) keep_metric_names` leaves `foo` and `bar` metric names in the returned time series.
+* `({__name__=~"foo|bar"} / 10) keep_metric_names` leaves `foo` and `bar` metric names in the returned time series.
 
 ## MetricsQL functions
 
@@ -166,10 +167,10 @@ Additional details:
 * If the given [series selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) returns multiple time series,
   then rollups are calculated individually per each returned series.
 * If lookbehind window in square brackets is missing, then it is automatically set to the following value:
-  - To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query)
+  * To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query)
     for all the [rollup functions](#rollup-functions) except of [default_rollup](#default_rollup) and [rate](#rate). This value is known as `$__interval` in Grafana or `1i` in MetricsQL.
     For example, `avg_over_time(temperature)` is automatically transformed to `avg_over_time(temperature[1i])`.
-  - To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+  * To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
     for [default_rollup](#default_rollup) and [rate](#rate) functions. This allows avoiding unexpected gaps on the graph when `step` is smaller than `scrape_interval`.
 * Every [series selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) in MetricsQL must be wrapped into a rollup function.
   Otherwise, it is automatically wrapped into [default_rollup](#default_rollup). For example, `foo{bar="baz"}`
@@ -666,8 +667,9 @@ This function is usually applied to [gauges](https://docs.victoriametrics.com/vi
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
 if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iqr` where:
-- `iqr` is an [Interquartile range](https://en.wikipedia.org/wiki/Interquartile_range) over [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on the lookbehind window `d`
-- `q25` and `q75` are 25th and 75th [percentiles](https://en.wikipedia.org/wiki/Percentile) over [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on the lookbehind window `d`.
+
+* `iqr` is an [Interquartile range](https://en.wikipedia.org/wiki/Interquartile_range) over [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on the lookbehind window `d`
+* `q25` and `q75` are 25th and 75th [percentiles](https://en.wikipedia.org/wiki/Percentile) over [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on the lookbehind window `d`.
 
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
@@ -759,7 +761,6 @@ This function is usually applied to [counters](https://docs.victoriametrics.com/
 
 See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
 
-
 #### rate_over_sum
 
 `rate_over_sum(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second rate over the sum of [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
@@ -1106,7 +1107,6 @@ This function is usually applied to [gauges](https://docs.victoriametrics.com/vi
 
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
-
 ### Transform functions
 
 **Transform functions** calculate transformations over [rollup results](#rollup-functions).
@@ -1851,7 +1851,6 @@ The list of supported label manipulation functions:
 `alias(q, "name")` is [label manipulation function](#label-manipulation-functions), which sets the given `name` to all the time series returned by `q`.
 For example, `alias(up, "foobar")` would rename `up` series to `foobar` series.
 
-
 #### drop_common_labels
 
 `drop_common_labels(q1, ...., qN)` is [label manipulation function](#label-manipulation-functions), which drops common `label="value"` pairs
@@ -1877,7 +1876,7 @@ For example, `label_graphite_group({__graphite__="foo*.bar.*"}, 0, 2)` would sub
 
 This function is useful for aggregating Graphite metrics with [aggregate functions](#aggregate-functions). For example, the following query would return per-app memory usage:
 
-```
+```metricsql
 sum by (__name__) (
     label_graphite_group({__graphite__="app*.host*.memory_usage"}, 0)
 )
@@ -2003,7 +2002,6 @@ would return series in the following order of `bar` label values: `101`, `15`, `
 
 See also [sort_by_label_numeric](#sort_by_label_numeric) and [sort_by_label_desc](#sort_by_label_desc).
 
-
 ### Aggregate functions
 
 **Aggregate functions** calculate aggregates over groups of [rollup results](#rollup-functions).
@@ -2179,8 +2177,9 @@ per each `group_labels` for all the time series returned by `q`. The aggregate i
 `outliers_iqr(q)` is [aggregate function](#aggregate-functions), which returns time series from `q` with at least a single point
 outside e.g. [Interquartile range outlier bounds](https://en.wikipedia.org/wiki/Interquartile_range) `[q25-1.5*iqr .. q75+1.5*iqr]`
 comparing to other time series at the given point, where:
-- `iqr` is an [Interquartile range](https://en.wikipedia.org/wiki/Interquartile_range) calculated independently per each point on the graph across `q` series.
-- `q25` and `q75` are 25th and 75th [percentiles](https://en.wikipedia.org/wiki/Percentile) calculated independently per each point on the graph across `q` series.
+
+* `iqr` is an [Interquartile range](https://en.wikipedia.org/wiki/Interquartile_range) calculated independently per each point on the graph across `q` series.
+* `q25` and `q75` are 25th and 75th [percentiles](https://en.wikipedia.org/wiki/Percentile) calculated independently per each point on the graph across `q` series.
 
 The `outliers_iqr()` is useful for detecting anomalous series in the group of series. For example, `outliers_iqr(temperature) by (country)` returns
 per-country series with anomalous outlier values comparing to the rest of per-country series.
@@ -2349,10 +2348,10 @@ VictoriaMetrics performs subqueries in the following way:
 VictoriaMetrics performs the following implicit conversions for incoming queries before starting the calculations:
 
 * If lookbehind window in square brackets is missing inside [rollup function](#rollup-functions), then it is automatically set to the following value:
-  - To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query)
+  * To `step` value passed to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query)
     for all the [rollup functions](#rollup-functions) except of [default_rollup](#default_rollup) and [rate](#rate). This value is known as `$__interval` in Grafana or `1i` in MetricsQL.
     For example, `avg_over_time(temperature)` is automatically transformed to `avg_over_time(temperature[1i])`.
-  - To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
+  * To the `max(step, scrape_interval)`, where `scrape_interval` is the interval between [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
     for [default_rollup](#default_rollup) and [rate](#rate) functions. This allows avoiding unexpected gaps on the graph when `step` is smaller than `scrape_interval`.
 * All the [series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering),
   which aren't wrapped into [rollup functions](#rollup-functions), are automatically wrapped into [default_rollup](#default_rollup) function.

app/vmselect/vmui/index.html

@@ -36,10 +36,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-BT5pWGkz.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-BVRvRxZ2.js">
+  <script type="module" crossorigin src="./assets/index-DQcPcJrn.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DY9kCvzk.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BHg4iVVe.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-dWApsAEM.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -683,7 +683,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricIDs", reason="miss_percentage"}`, m.MetricIDCacheMissEvictionBytes)
 	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricIDs", reason="expiration"}`, m.MetricIDCacheExpireEvictionBytes)
 
-	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, idbm.DeletedMetricsCount)
+	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
 
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/tsid"}`, m.TSIDCacheCollisions)
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/metricName"}`, m.MetricNameCacheCollisions)

app/vmui/Dockerfile-build

@@ -1,4 +1,4 @@
-FROM node:20-alpine3.19
+FROM node:22-alpine3.22
 
 # Sets a custom location for the npm cache, preventing access errors in system directories
 ENV NPM_CONFIG_CACHE=/build/.npm

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.24.5 AS build-web-stage
+FROM golang:1.25.1 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/index.html

@@ -6,6 +6,7 @@
   <link rel="apple-touch-icon" href="/favicon.svg"/>
   <link rel="mask-icon" href="/favicon.svg" color="#000000">
 
+  <meta name="robots" content="noindex">
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
   <meta name="theme-color" content="#000000"/>
   <meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/>

app/vmui/packages/vmui/package-lock.json

@@ -17,7 +17,7 @@
         "react-input-mask": "^2.0.4",
         "react-router-dom": "^7.6.3",
         "uplot": "^1.6.32",
-        "vite": "^7.0.4",
+        "vite": "^7.1.5",
         "web-vitals": "^5.0.3"
       },
       "devDependencies": {
@@ -1177,7 +1177,7 @@
       "version": "0.3.12",
       "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.12.tgz",
       "integrity": "sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "@jridgewell/sourcemap-codec": "^1.5.0",
@@ -1188,24 +1188,36 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
       "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "engines": {
         "node": ">=6.0.0"
       }
     },
+    "node_modules/@jridgewell/source-map": {
+      "version": "0.3.10",
+      "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.10.tgz",
+      "integrity": "sha512-0pPkgz9dY+bijgistcTTJ5mR+ocqRXLuhXHYdzoMmmoJ2C9S46RCm2GMUbatPEUK9Yjy26IrAy8D/M00lLkv+Q==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/gen-mapping": "^0.3.5",
+        "@jridgewell/trace-mapping": "^0.3.25"
+      }
+    },
     "node_modules/@jridgewell/sourcemap-codec": {
       "version": "1.5.4",
       "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz",
       "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT"
     },
     "node_modules/@jridgewell/trace-mapping": {
       "version": "0.3.29",
       "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.29.tgz",
       "integrity": "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "@jridgewell/resolve-uri": "^3.1.0",
@@ -1250,6 +1262,316 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@parcel/watcher": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.1.tgz",
+      "integrity": "sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "detect-libc": "^1.0.3",
+        "is-glob": "^4.0.3",
+        "micromatch": "^4.0.5",
+        "node-addon-api": "^7.0.0"
+      },
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      },
+      "optionalDependencies": {
+        "@parcel/watcher-android-arm64": "2.5.1",
+        "@parcel/watcher-darwin-arm64": "2.5.1",
+        "@parcel/watcher-darwin-x64": "2.5.1",
+        "@parcel/watcher-freebsd-x64": "2.5.1",
+        "@parcel/watcher-linux-arm-glibc": "2.5.1",
+        "@parcel/watcher-linux-arm-musl": "2.5.1",
+        "@parcel/watcher-linux-arm64-glibc": "2.5.1",
+        "@parcel/watcher-linux-arm64-musl": "2.5.1",
+        "@parcel/watcher-linux-x64-glibc": "2.5.1",
+        "@parcel/watcher-linux-x64-musl": "2.5.1",
+        "@parcel/watcher-win32-arm64": "2.5.1",
+        "@parcel/watcher-win32-ia32": "2.5.1",
+        "@parcel/watcher-win32-x64": "2.5.1"
+      }
+    },
+    "node_modules/@parcel/watcher-android-arm64": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-android-arm64/-/watcher-android-arm64-2.5.1.tgz",
+      "integrity": "sha512-KF8+j9nNbUN8vzOFDpRMsaKBHZ/mcjEjMToVMJOhTozkDonQFFrRcfdLWn6yWKCmJKmdVxSgHiYvTCef4/qcBA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-darwin-arm64": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-arm64/-/watcher-darwin-arm64-2.5.1.tgz",
+      "integrity": "sha512-eAzPv5osDmZyBhou8PoF4i6RQXAfeKL9tjb3QzYuccXFMQU0ruIc/POh30ePnaOyD1UXdlKguHBmsTs53tVoPw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-darwin-x64": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-x64/-/watcher-darwin-x64-2.5.1.tgz",
+      "integrity": "sha512-1ZXDthrnNmwv10A0/3AJNZ9JGlzrF82i3gNQcWOzd7nJ8aj+ILyW1MTxVk35Db0u91oD5Nlk9MBiujMlwmeXZg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-freebsd-x64": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-freebsd-x64/-/watcher-freebsd-x64-2.5.1.tgz",
+      "integrity": "sha512-SI4eljM7Flp9yPuKi8W0ird8TI/JK6CSxju3NojVI6BjHsTyK7zxA9urjVjEKJ5MBYC+bLmMcbAWlZ+rFkLpJQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-linux-arm-glibc": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-glibc/-/watcher-linux-arm-glibc-2.5.1.tgz",
+      "integrity": "sha512-RCdZlEyTs8geyBkkcnPWvtXLY44BCeZKmGYRtSgtwwnHR4dxfHRG3gR99XdMEdQ7KeiDdasJwwvNSF5jKtDwdA==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-linux-arm-musl": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.1.tgz",
+      "integrity": "sha512-6E+m/Mm1t1yhB8X412stiKFG3XykmgdIOqhjWj+VL8oHkKABfu/gjFj8DvLrYVHSBNC+/u5PeNrujiSQ1zwd1Q==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-linux-arm64-glibc": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.1.tgz",
+      "integrity": "sha512-LrGp+f02yU3BN9A+DGuY3v3bmnFUggAITBGriZHUREfNEzZh/GO06FF5u2kx8x+GBEUYfyTGamol4j3m9ANe8w==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-linux-arm64-musl": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.1.tgz",
+      "integrity": "sha512-cFOjABi92pMYRXS7AcQv9/M1YuKRw8SZniCDw0ssQb/noPkRzA+HBDkwmyOJYp5wXcsTrhxO0zq1U11cK9jsFg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-linux-x64-glibc": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.1.tgz",
+      "integrity": "sha512-GcESn8NZySmfwlTsIur+49yDqSny2IhPeZfXunQi48DMugKeZ7uy1FX83pO0X22sHntJ4Ub+9k34XQCX+oHt2A==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-linux-x64-musl": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.1.tgz",
+      "integrity": "sha512-n0E2EQbatQ3bXhcH2D1XIAANAcTZkQICBPVaxMeaCVBtOpBZpWJuf7LwyWPSBDITb7In8mqQgJ7gH8CILCURXg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-win32-arm64": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.1.tgz",
+      "integrity": "sha512-RFzklRvmc3PkjKjry3hLF9wD7ppR4AKcWNzH7kXR7GUe0Igb3Nz8fyPwtZCSquGrhU5HhUNDr/mKBqj7tqA2Vw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-win32-ia32": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-ia32/-/watcher-win32-ia32-2.5.1.tgz",
+      "integrity": "sha512-c2KkcVN+NJmuA7CGlaGD1qJh1cLfDnQsHjE89E60vUEMlqduHGCdCLJCID5geFVM0dOtA3ZiIO8BoEQmzQVfpQ==",
+      "cpu": [
+        "ia32"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/watcher-win32-x64": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-x64/-/watcher-win32-x64-2.5.1.tgz",
+      "integrity": "sha512-9lHBdJITeNR++EvSQVUcaZoWupyHfXe1jZvGZ06O/5MflPcuPLtEphScIBL+AiCWBO46tDSHzWyD0uDmmZqsgA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "peer": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
     "node_modules/@preact/preset-vite": {
       "version": "2.10.2",
       "resolved": "https://registry.npmjs.org/@preact/preset-vite/-/preset-vite-2.10.2.tgz",
@@ -1750,9 +2072,9 @@
       }
     },
     "node_modules/@types/node": {
-      "version": "24.0.12",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.12.tgz",
-      "integrity": "sha512-LtOrbvDf5ndC9Xi+4QZjVL0woFymF/xSTKZKPgrrl7H7XoeDvnD+E2IclKVDyaK9UM756W/3BXqSU+JEHopA9g==",
+      "version": "24.1.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz",
+      "integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==",
       "devOptional": true,
       "license": "MIT",
       "dependencies": {
@@ -2221,7 +2543,7 @@
       "version": "8.15.0",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
       "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "bin": {
         "acorn": "bin/acorn"
@@ -2523,7 +2845,7 @@
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
       "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "fill-range": "^7.1.1"
@@ -2572,6 +2894,14 @@
       "devOptional": true,
       "license": "MIT/X11"
     },
+    "node_modules/buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true
+    },
     "node_modules/cac": {
       "version": "6.7.14",
       "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz",
@@ -2702,6 +3032,23 @@
         "node": ">= 16"
       }
     },
+    "node_modules/chokidar": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz",
+      "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "readdirp": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 14.16.0"
+      },
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
     "node_modules/classnames": {
       "version": "2.5.1",
       "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz",
@@ -2750,6 +3097,14 @@
       "devOptional": true,
       "license": "MIT"
     },
+    "node_modules/commander": {
+      "version": "2.20.3",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3060,6 +3415,20 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/detect-libc": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
+      "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==",
+      "license": "Apache-2.0",
+      "optional": true,
+      "peer": true,
+      "bin": {
+        "detect-libc": "bin/detect-libc.js"
+      },
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "node_modules/doctrine": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
@@ -3808,7 +4177,7 @@
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
       "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "to-regex-range": "^5.0.1"
@@ -4522,7 +4891,7 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
       "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "engines": {
         "node": ">=0.10.0"
@@ -4577,7 +4946,7 @@
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
       "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "is-extglob": "^2.1.1"
@@ -4616,7 +4985,7 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
       "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "engines": {
         "node": ">=0.12.0"
@@ -5119,7 +5488,7 @@
       "version": "4.0.8",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
       "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "braces": "^3.0.3",
@@ -5184,6 +5553,14 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/node-addon-api": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz",
+      "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true
+    },
     "node_modules/node-html-parser": {
       "version": "6.1.13",
       "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.13.tgz",
@@ -5512,7 +5889,7 @@
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
       "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "engines": {
         "node": ">=8.6"
@@ -5754,6 +6131,21 @@
         "react-dom": ">=18"
       }
     },
+    "node_modules/readdirp": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz",
+      "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true,
+      "engines": {
+        "node": ">= 14.18.0"
+      },
+      "funding": {
+        "type": "individual",
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
     "node_modules/redent": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz",
@@ -6054,6 +6446,28 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/sass": {
+      "version": "1.89.2",
+      "resolved": "https://registry.npmjs.org/sass/-/sass-1.89.2.tgz",
+      "integrity": "sha512-xCmtksBKd/jdJ9Bt9p7nPKiuqrlBMBuuGkQlkhZjjQk3Ty48lv93k5Dq6OPkKt4XwxDJ7tvlfrTa1MPA9bf+QA==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "chokidar": "^4.0.0",
+        "immutable": "^5.0.2",
+        "source-map-js": ">=0.6.2 <2.0.0"
+      },
+      "bin": {
+        "sass": "sass.js"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      },
+      "optionalDependencies": {
+        "@parcel/watcher": "^2.4.1"
+      }
+    },
     "node_modules/sass-embedded": {
       "version": "1.89.2",
       "resolved": "https://registry.npmjs.org/sass-embedded/-/sass-embedded-1.89.2.tgz",
@@ -6583,6 +6997,29 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/source-map-support": {
+      "version": "0.5.21",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz",
+      "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==",
+      "license": "MIT",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "source-map": "^0.6.0"
+      }
+    },
+    "node_modules/source-map-support/node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+      "license": "BSD-3-Clause",
+      "optional": true,
+      "peer": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/stack-trace": {
       "version": "1.0.0-pre2",
       "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-1.0.0-pre2.tgz",
@@ -6849,6 +7286,26 @@
         "node": ">=16.0.0"
       }
     },
+    "node_modules/terser": {
+      "version": "5.43.1",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.43.1.tgz",
+      "integrity": "sha512-+6erLbBm0+LROX2sPXlUYx/ux5PyE9K/a92Wrt6oA+WDAoFTdpHE5tCYCI5PNzq2y8df4rA+QgHLJuR4jNymsg==",
+      "license": "BSD-2-Clause",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/source-map": "^0.3.3",
+        "acorn": "^8.14.0",
+        "commander": "^2.20.0",
+        "source-map-support": "~0.5.20"
+      },
+      "bin": {
+        "terser": "bin/terser"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/tinybench": {
       "version": "2.9.0",
       "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz",
@@ -6864,13 +7321,13 @@
       "license": "MIT"
     },
     "node_modules/tinyglobby": {
-      "version": "0.2.14",
-      "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.14.tgz",
-      "integrity": "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==",
+      "version": "0.2.15",
+      "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz",
+      "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==",
       "license": "MIT",
       "dependencies": {
-        "fdir": "^6.4.4",
-        "picomatch": "^4.0.2"
+        "fdir": "^6.5.0",
+        "picomatch": "^4.0.3"
       },
       "engines": {
         "node": ">=12.0.0"
@@ -6880,10 +7337,13 @@
       }
     },
     "node_modules/tinyglobby/node_modules/fdir": {
-      "version": "6.4.6",
-      "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz",
-      "integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==",
+      "version": "6.5.0",
+      "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
+      "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==",
       "license": "MIT",
+      "engines": {
+        "node": ">=12.0.0"
+      },
       "peerDependencies": {
         "picomatch": "^3 || ^4"
       },
@@ -6894,9 +7354,9 @@
       }
     },
     "node_modules/tinyglobby/node_modules/picomatch": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
-      "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
+      "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "license": "MIT",
       "engines": {
         "node": ">=12"
@@ -6959,7 +7419,7 @@
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
       "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "is-number": "^7.0.0"
@@ -7200,17 +7660,17 @@
       "license": "MIT"
     },
     "node_modules/vite": {
-      "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-7.0.4.tgz",
-      "integrity": "sha512-SkaSguuS7nnmV7mfJ8l81JGBFV7Gvzp8IzgE8A8t23+AxuNX61Q5H1Tpz5efduSN7NHC8nQXD3sKQKZAu5mNEA==",
+      "version": "7.1.5",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.5.tgz",
+      "integrity": "sha512-4cKBO9wR75r0BeIWWWId9XK9Lj6La5X846Zw9dFfzMRw38IlTk2iCcUt6hsyiDRcPidc55ZParFYDXi0nXOeLQ==",
       "license": "MIT",
       "dependencies": {
         "esbuild": "^0.25.0",
-        "fdir": "^6.4.6",
-        "picomatch": "^4.0.2",
+        "fdir": "^6.5.0",
+        "picomatch": "^4.0.3",
         "postcss": "^8.5.6",
-        "rollup": "^4.40.0",
-        "tinyglobby": "^0.2.14"
+        "rollup": "^4.43.0",
+        "tinyglobby": "^0.2.15"
       },
       "bin": {
         "vite": "bin/vite.js"
@@ -7315,10 +7775,13 @@
       }
     },
     "node_modules/vite/node_modules/fdir": {
-      "version": "6.4.6",
-      "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz",
-      "integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==",
+      "version": "6.5.0",
+      "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
+      "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==",
       "license": "MIT",
+      "engines": {
+        "node": ">=12.0.0"
+      },
       "peerDependencies": {
         "picomatch": "^3 || ^4"
       },
@@ -7329,9 +7792,9 @@
       }
     },
     "node_modules/vite/node_modules/picomatch": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
-      "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
+      "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "license": "MIT",
       "engines": {
         "node": ">=12"

app/vmui/packages/vmui/package.json

@@ -29,7 +29,7 @@
     "react-input-mask": "^2.0.4",
     "react-router-dom": "^7.6.3",
     "uplot": "^1.6.32",
-    "vite": "^7.0.4",
+    "vite": "^7.1.5",
     "web-vitals": "^5.0.3"
   },
   "devDependencies": {