feature: [logger] update comments

.github/ISSUE_TEMPLATE/question.yml

@@ -5,7 +5,7 @@ body:
   - type: textarea
     id: describe-the-component
     attributes:
-      label: Is your question related to a specific component?
+      label: Is your question request related to a specific component?
       placeholder: |
         VictoriaMetrics, vmagent, vmalert, vmui, etc...
     validations:

.github/copilot-instructions.md

@@ -1,23 +0,0 @@
-# Project Overview
-
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
-
-## Folder Structure
-
-- `/app`: Contains the compilable binaries.
-- `/lib`: Contains the golang reusable libraries
-- `/docs/victoriametrics`: Contains documentation for the project.
-- `/apptest/tests`: Contains integration tests.
-
-## Libraries and Frameworks
-
-- Backend: Golang, no framework. Use third-party libraries sparingly.
-- Frontend: React.
-
-## Code review guidelines
-
-Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
-Verify the entry is under the ## tip section and matches the structure and style of existing entries.
-Chore-only changes may be omitted from the changelog.
-
-

.github/scripts/lint-changelog-tip.sh

@@ -1,48 +0,0 @@
-#!/usr/bin/env sh
-
-set -e
-
-CHANGELOG_FILE="docs/victoriametrics/changelog/CHANGELOG.md"
-
-GITHUB_BASE_REF=${GITHUB_BASE_REF:-"master"}
-GIT_REMOTE=${GIT_REMOTE:-"origin"}
-
-git diff "${GIT_REMOTE}/${GITHUB_BASE_REF}"...HEAD -- $CHANGELOG_FILE > diff.txt
-if ! grep -q "^+" diff.txt; then
-  echo "No additions in CHANGELOG.md"
-  exit 0
-fi
-
-ADDED_LINES=$(grep "^+\S" diff.txt | sed 's/^+//')
-
-START_TIP=$(grep -n "^## tip" "$CHANGELOG_FILE" | head -1 | cut -d: -f1)
-if [ -z "$START_TIP" ]; then
-  echo "ERROR: ${CHANGELOG_FILE} does not contain a ## tip section"
-  exit 1
-fi
-
-END_TIP=$(awk "NR>$START_TIP && /^## / {print NR; exit}" "${CHANGELOG_FILE}")
-if [ -z "$END_TIP" ]; then
-  END_TIP=$(wc -l < "$CHANGELOG_FILE")
-fi
-
-BAD=0
-while IFS= read -r line; do
-  # Grep exact line inside the file and get line numbers
-  MATCHES=$(grep -n -F "$line" "$CHANGELOG_FILE" | cut -d: -f1)
-  for m in $MATCHES; do
-    if [ "$m" -lt "$START_TIP" ] || [ "$m" -gt "$END_TIP" ]; then
-      echo "'$line' on line ${m} is outside ## tip section (lines ${START_TIP}-${END_TIP})"
-      BAD=1
-    fi
-  done
-done << EOF
-$ADDED_LINES
-EOF
-
-if [ "$BAD" -ne 0 ]; then
-  echo "CHANGELOG modifications must be placed inside the ## tip section."
-  exit 1
-fi
-
-echo "CHANGELOG modifications are valid."

.github/workflows/build.yml

@@ -7,20 +7,16 @@ on:
       - master
     paths:
       - '**.go'
-      - '**/Dockerfile'
+      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
       - '**/Makefile'
-      - '!app/vmui/**'
-      - '.github/workflows/build.yml'
   pull_request:
     branches:
       - cluster
       - master
     paths:
       - '**.go'
-      - '**/Dockerfile'
+      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
       - '**/Makefile'
-      - '!app/vmui/**'
-      - '.github/workflows/build.yml'
 
 permissions:
   contents: read
@@ -31,50 +27,28 @@ concurrency:
 
 jobs:
   build:
-    name: ${{ matrix.os }}-${{ matrix.arch }}
+    name: Build
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: linux
-            arch: 386
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-          - os: linux
-            arch: arm
-          - os: linux
-            arch: ppc64le
-          - os: linux
-            arch: s390x
-          - os: darwin
-            arch: amd64
-          - os: darwin
-            arch: arm64
-          - os: freebsd
-            arch: amd64
-          - os: openbsd
-            arch: amd64
-          - os: windows
-            arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v5
         with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
           go-version: stable
+          cache: false
 
-      - name: Build victoria-metrics for ${{ matrix.os }}-${{ matrix.arch }}
-        run: make victoria-metrics-${{ matrix.os }}-${{ matrix.arch }}
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
 
-      - name: Build vmutils for ${{ matrix.os }}-${{ matrix.arch }}
-        run: make vmutils-${{ matrix.os }}-${{ matrix.arch }}
+      - name: Run crossbuild
+        run: make crossbuild

.github/workflows/changelog-linter.yml

@@ -1,19 +0,0 @@
-name: 'changelog-linter'
-
-on:
-  pull_request:
-    paths:
-      - "docs/victoriametrics/changelog/CHANGELOG.md"
-
-jobs:
-  tip-lint:
-    runs-on: 'ubuntu-latest'
-    steps:
-      - uses: 'actions/checkout@v6'
-        with:
-          # needed for proper diff
-          fetch-depth: 0
-
-      - name: 'Validate that changelog changes are under ## tip'
-        run: |
-          GITHUB_BASE_REF=${{ github.base_ref }} ./.github/scripts/lint-changelog-tip.sh

.github/workflows/check-commit-signed.yml

@@ -1,37 +0,0 @@
-name: check-commit-signed
-
-on:
-  pull_request:
-
-jobs:
-  check-commit-signed:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0  # we need full history for commit verification
-
-      - name: Check commit signatures
-        run: |
-          if [ "${{ github.event_name }}" != "pull_request" ]; then
-            echo "Not a PR event, skipping signature check"
-            exit 0
-          fi
-          
-          RANGE="${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
-          echo "Checking commits in PR range: $RANGE"
-          
-          if [ -z "$(git rev-list $RANGE)" ]; then
-            echo "No new commits in this PR, skipping signature check"
-            exit 0
-          fi
-          
-          unsigned=$(git log --pretty="%H %G?" $RANGE | grep -vE " (G|E)$" || true)
-          if [ -n "$unsigned" ]; then
-            echo "Found unsigned commits:"
-            echo "$unsigned"
-            exit 1
-          fi
-          
-          echo "All commits in PR are signed (G or E)"

.github/workflows/check-licenses.yml

@@ -19,7 +19,7 @@ jobs:
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v5
         with:
           go-version: stable
           cache: false

.github/workflows/codeql-analysis-go.yml

@@ -29,11 +29,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: Set up Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v5
         with:
           cache: false
           go-version: stable
@@ -49,14 +49,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4
+        uses: github/codeql-action/init@v3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4
+        uses: github/codeql-action/autobuild@v3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4
+        uses: github/codeql-action/analyze@v3
         with:
           category: 'language:go'

.github/workflows/codeql-analysis-js-ts.yml

@@ -0,0 +1,46 @@
+name: 'CodeQL JS/TS'
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.js'
+      - '**.ts'
+      - '**.tsx'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.js'
+      - '**.ts'
+      - '**.tsx'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: javascript-typescript
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: 'language:js/ts'

.github/workflows/docs.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/main.yml

@@ -0,0 +1,122 @@
+name: main
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+
+permissions:
+  contents: read
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: stable
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-check-all-
+
+      - name: Run check-all
+        run: |
+          make check-all
+          git diff --exit-code
+
+  test:
+    name: test
+    needs: lint
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        scenario:
+          - 'test-full'
+          - 'test-full-386'
+          - 'test-pure'
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: stable
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
+
+      - name: Run tests
+        run: GOGC=10 make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.txt
+
+  integration-test:
+    name: integration-test
+    needs: [lint, test]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: stable
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
+
+      - name: Run integration tests
+        run: make integration-test

.github/workflows/test.yml

@@ -1,113 +0,0 @@
-name: test
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-
-jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v6
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v6
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-
-      - name: Cache golangci-lint
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/golangci-lint
-            ~/go/bin
-          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
-
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
-
-  unit:
-    name: unit
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        scenario:
-          - 'test-full'
-          - 'test-full-386'
-          - 'test-pure'
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v6
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v6
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v5
-        with:
-          files: ./coverage.txt
-
-  integration:
-    name: integration
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v6
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v6
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Run integration tests
-        run: make integration-test

.github/workflows/vmui.yml

@@ -1,82 +0,0 @@
-name: vmui
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - 'app/vmui/packages/vmui/**'
-      - '.github/workflows/vmui.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - 'app/vmui/packages/vmui/**'
-      - '.github/workflows/vmui.yml'
-
-permissions:
-  contents: read
-  packages: read
-  pull-requests: read
-  checks: write
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  vmui-checks:
-    name: VMUI Checks (lint, test, typecheck)
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v6
-
-      - name: Setup Node
-        uses: actions/setup-node@v6
-        with:
-          node-version: '24.x'
-
-      - name: Cache node-modules
-        uses: actions/cache@v4
-        with:
-          path: |
-            app/vmui/packages/vmui/node_modules
-          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: vmui-artifacts-${{ runner.os }}-
-
-      - name: Run lint
-        id: lint
-        run: make vmui-lint
-        continue-on-error: true
-
-      - name: Run tests
-        id: test
-        run: make vmui-test
-        continue-on-error: true
-
-      - name: Run typecheck
-        id: typecheck
-        run: make vmui-typecheck
-        continue-on-error: true
-
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@v3
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
-      - name: Check overall status
-        run: |
-          echo "Lint status: ${{ steps.lint.outcome }}"
-          echo "Test status: ${{ steps.test.outcome }}"
-          echo "Typecheck status: ${{ steps.typecheck.outcome }}"
-
-          if [[ "${{ steps.lint.outcome }}" == "failure" || "${{ steps.test.outcome }}" == "failure" || "${{ steps.typecheck.outcome }}" == "failure" ]]; then
-            echo "One or more checks failed"
-            exit 1
-          else
-            echo "All checks passed"
-          fi

.golangci.yml

@@ -1,29 +1,22 @@
-version: "2"
+run:
+  timeout: 2m
+
 linters:
-  settings:
-    errcheck:
-      exclude-functions:
-        - fmt.Fprintf
-        - fmt.Fprint
-        - (net/http.ResponseWriter).Write
-  exclusions:
-    generated: lax
-    presets:
-      - common-false-positives
-      - legacy
-      - std-error-handling
-    rules:
-      - linters:
-          - staticcheck
-        text: 'SA(4003|1019|5011):'
-    paths:
-      - third_party$
-      - builtin$
-      - examples$
-formatters:
-  exclusions:
-    generated: lax
-    paths:
-      - third_party$
-      - builtin$
-      - examples$
+  enable:
+  - revive
+
+issues:
+  exclude-rules:
+  - linters:
+    - staticcheck
+    text: "SA(4003|1019|5011):"
+  include:
+  - EXC0012
+  - EXC0014
+
+linters-settings:
+  errcheck:
+    exclude-functions:
+      - "fmt.Fprintf"
+      - "fmt.Fprint"
+      - "(net/http.ResponseWriter).Write"

Makefile

@@ -11,14 +11,11 @@ ifeq ($(PKG_TAG),)
 PKG_TAG := $(BUILDINFO_TAG)
 endif
 
-EXTRA_DOCKER_TAG_SUFFIX ?=
-EXTRA_GO_BUILD_TAGS ?=
+EXTRA_DOCKER_TAG_SUFFIX ?= EXTRA_DOCKER_TAG_SUFFIX
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.7.2
-
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
@@ -27,10 +24,11 @@ include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
 include package/release/Makefile
-include benchmarks/Makefile
 
 all: \
 	victoria-metrics-prod \
+	victoria-logs-prod \
+	vlogscli-prod \
 	vmagent-prod \
 	vmalert-prod \
 	vmalert-tool-prod \
@@ -54,6 +52,8 @@ publish: \
 
 package: \
 	package-victoria-metrics \
+	package-victoria-logs \
+	package-vlogscli \
 	package-vmagent \
 	package-vmalert \
 	package-vmalert-tool \
@@ -125,15 +125,6 @@ vmutils-linux-ppc64le: \
 	vmrestore-linux-ppc64le \
 	vmctl-linux-ppc64le
 
-vmutils-linux-s390x: \
-	vmagent-linux-s390x \
-	vmalert-linux-s390x \
-	vmalert-tool-linux-s390x \
-	vmauth-linux-s390x \
-	vmbackup-linux-s390x \
-	vmrestore-linux-s390x \
-	vmctl-linux-s390x
-
 vmutils-darwin-amd64: \
 	vmagent-darwin-amd64 \
 	vmalert-darwin-amd64 \
@@ -179,11 +170,9 @@ vmutils-windows-amd64: \
 	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
-# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 crossbuild:
 	$(MAKE_PARALLEL) victoria-metrics-crossbuild vmutils-crossbuild
 
-# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 victoria-metrics-crossbuild: \
 	victoria-metrics-linux-386 \
 	victoria-metrics-linux-amd64 \
@@ -196,7 +185,6 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-openbsd-amd64 \
 	victoria-metrics-windows-amd64
 
-# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 vmutils-crossbuild: \
 	vmutils-linux-386 \
 	vmutils-linux-amd64 \
@@ -249,6 +237,10 @@ publish-latest:
 	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
 	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
 
+publish-victoria-logs-latest:
+	PKG_TAG=$(TAG) APP_NAME=victoria-logs $(MAKE) publish-via-docker-latest
+	PKG_TAG=$(TAG) APP_NAME=vlogscli $(MAKE) publish-via-docker-latest
+
 publish-release:
 	rm -rf bin/*
 	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
@@ -266,7 +258,6 @@ release-victoria-metrics: \
 	release-victoria-metrics-linux-amd64 \
 	release-victoria-metrics-linux-arm \
 	release-victoria-metrics-linux-arm64 \
-	release-victoria-metrics-linux-s390x \
 	release-victoria-metrics-darwin-amd64 \
 	release-victoria-metrics-darwin-arm64 \
 	release-victoria-metrics-freebsd-amd64 \
@@ -285,9 +276,6 @@ release-victoria-metrics-linux-arm:
 release-victoria-metrics-linux-arm64:
 	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-linux-s390x:
-	GOOS=linux GOARCH=s390x $(MAKE) release-victoria-metrics-goos-goarch
-
 release-victoria-metrics-darwin-amd64:
 	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
@@ -322,12 +310,133 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-metrics-windows-$(GOARCH)-prod.exe
 
+release-victoria-logs-bundle: \
+	release-victoria-logs \
+	release-vlogscli
+
+publish-victoria-logs-bundle: \
+	publish-victoria-logs \
+	publish-vlogscli
+
+release-victoria-logs:
+	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
+		release-victoria-logs-linux-amd64 \
+		release-victoria-logs-linux-arm \
+		release-victoria-logs-linux-arm64 \
+		release-victoria-logs-darwin-amd64 \
+		release-victoria-logs-darwin-arm64 \
+		release-victoria-logs-freebsd-amd64 \
+		release-victoria-logs-openbsd-amd64 \
+		release-victoria-logs-windows-amd64
+
+release-victoria-logs-linux-386:
+	GOOS=linux GOARCH=386 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-windows-amd64:
+	GOARCH=amd64 $(MAKE) release-victoria-logs-windows-goarch
+
+release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
+	cd bin && \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-logs-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-logs-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf victoria-logs-$(GOOS)-$(GOARCH)-prod
+
+release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
+	cd bin && \
+		zip victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-logs-windows-$(GOARCH)-prod.exe \
+		&& sha256sum victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-logs-windows-$(GOARCH)-prod.exe \
+			> victoria-logs-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		victoria-logs-windows-$(GOARCH)-prod.exe
+
+release-vlogscli:
+	$(MAKE_PARALLEL) release-vlogscli-linux-386 \
+		release-vlogscli-linux-amd64 \
+		release-vlogscli-linux-arm \
+		release-vlogscli-linux-arm64 \
+		release-vlogscli-darwin-amd64 \
+		release-vlogscli-darwin-arm64 \
+		release-vlogscli-freebsd-amd64 \
+		release-vlogscli-openbsd-amd64 \
+		release-vlogscli-windows-amd64
+
+release-vlogscli-linux-386:
+	GOOS=linux GOARCH=386 $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
+
+release-vlogscli-windows-amd64:
+	GOARCH=amd64 $(MAKE) release-vlogscli-windows-goarch
+
+release-vlogscli-goos-goarch: vlogscli-$(GOOS)-$(GOARCH)-prod
+	cd bin && \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vlogscli-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vlogscli-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf vlogscli-$(GOOS)-$(GOARCH)-prod
+
+release-vlogscli-windows-goarch: vlogscli-windows-$(GOARCH)-prod
+	cd bin && \
+		zip vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
+			vlogscli-windows-$(GOARCH)-prod.exe \
+		&& sha256sum vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
+			vlogscli-windows-$(GOARCH)-prod.exe \
+			> vlogscli-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vlogscli-windows-$(GOARCH)-prod.exe
+
 release-vmutils: \
 	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
 	release-vmutils-linux-arm64 \
 	release-vmutils-linux-arm \
-	release-vmutils-linux-s390x \
 	release-vmutils-darwin-amd64 \
 	release-vmutils-darwin-arm64 \
 	release-vmutils-freebsd-amd64 \
@@ -346,9 +455,6 @@ release-vmutils-linux-arm64:
 release-vmutils-linux-arm:
 	GOOS=linux GOARCH=arm $(MAKE) release-vmutils-goos-goarch
 
-release-vmutils-linux-s390x:
-	GOOS=linux GOARCH=s390x $(MAKE) release-vmutils-goos-goarch
-
 release-vmutils-darwin-amd64:
 	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
@@ -435,7 +541,7 @@ release-vmutils-windows-goarch: \
 		vmctl-windows-$(GOARCH)-prod.exe
 
 pprof-cpu:
-	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics $(PPROF_FILE)
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
 	gofmt -l -w -s ./lib
@@ -466,28 +572,8 @@ test-full:
 test-full-386:
 	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
-integration-test:
-	$(MAKE) apptest
-
-apptest:
-	$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
-	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
-
-integration-test-legacy: victoria-metrics vmbackup vmrestore
-	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
-	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
-	VERSION=v1.132.0; \
-	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
-	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
-	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
-	DIR=/tmp/$${VERSION}; \
-	test -d $${DIR} || (mkdir $${DIR} && \
-		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
-		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
-	); \
-	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
-	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
-	go test ./apptest/tests -run="^TestLegacySingle.*"
+integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
+	go test ./apptest/... -skip="^TestCluster.*"
 
 benchmark:
 	GOEXPERIMENT=synctest go test -bench=. ./lib/...
@@ -504,20 +590,19 @@ vendor-update:
 	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-goos-goarch:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-windows-goarch:
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
-	qtc -dir=lib
-	qtc -dir=app
+	qtc
 
 install-qtc:
 	which qtc || go install github.com/valyala/quicktemplate/qtc@latest
@@ -527,7 +612,7 @@ golangci-lint: install-golangci-lint
 	GOEXPERIMENT=synctest golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

README.md

@@ -3,7 +3,7 @@
 [![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 ![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
-[![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
+[![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml)
 [![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 ![Slack](https://img.shields.io/badge/Join-4A154B?logo=slack&link=https%3A%2F%2Fslack.victoriametrics.com)

SECURITY.md

@@ -4,11 +4,12 @@
 
 The following versions of VictoriaMetrics receive regular security fixes:
 
-| Version                                                                        | Supported          |
-|--------------------------------------------------------------------------------|--------------------|
-| [Latest release](https://docs.victoriametrics.com/victoriametrics/changelog/)  | :white_check_mark: |
-| [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| other releases                                                                 | :x:                |
+| Version | Supported          |
+|---------|--------------------|
+| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
+| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 

app/victoria-metrics/Makefile

@@ -27,9 +27,6 @@ victoria-metrics-linux-ppc64le-prod:
 victoria-metrics-linux-386-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-386
 
-victoria-metrics-linux-s390x-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-s390x
-
 victoria-metrics-darwin-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
 

app/victoria-metrics/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -124,6 +125,8 @@ func main() {
 	vmstorage.Stop()
 	vmselect.Stop()
 
+	fs.MustStopDirRemover()
+
 	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
 }
 
@@ -169,7 +172,7 @@ func usage() {
 	const s = `
 victoria-metrics is a time series database and monitoring solution.
 
-See the docs at https://docs.victoriametrics.com/victoriametrics/
+See the docs at https://docs.victoriametrics.com/
 `
 	flagutil.Usage(s)
 }

app/victoria-metrics/self_scraper.go

@@ -10,11 +10,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
@@ -50,9 +48,8 @@ func selfScraper(scrapeInterval time.Duration) {
 
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
-	var metadataRows prometheus.MetadataRows
 	var mrs []storage.MetricRow
-	var labels []prompb.Label
+	var labels []prompbmarshal.Label
 	t := time.NewTicker(scrapeInterval)
 	f := func(currentTime time.Time, sendStaleMarkers bool) {
 		currentTimestamp := currentTime.UnixNano() / 1e6
@@ -60,12 +57,7 @@ func selfScraper(scrapeInterval time.Duration) {
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
-		// Parse metrics and optionally metadata when enabled
-		if prommetadata.IsEnabled() {
-			rows, metadataRows = prometheus.UnmarshalWithMetadata(rows, metadataRows, s, nil)
-		} else {
-			rows.UnmarshalWithErrLogger(s, nil)
-		}
+		rows.Unmarshal(s)
 		mrs = mrs[:0]
 		for i := range rows.Rows {
 			r := &rows.Rows[i]
@@ -98,19 +90,6 @@ func selfScraper(scrapeInterval time.Duration) {
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
-		if len(metadataRows.Rows) > 0 {
-			mms := make([]metricsmetadata.Row, 0, len(metadataRows.Rows))
-			for _, mm := range metadataRows.Rows {
-				mms = append(mms, metricsmetadata.Row{
-					MetricFamilyName: bytesutil.ToUnsafeBytes(mm.Metric),
-					Help:             bytesutil.ToUnsafeBytes(mm.Help),
-					Type:             mm.Type,
-				})
-			}
-			if err := vmstorage.AddMetadataRows(mms); err != nil {
-				logger.Errorf("cannot store self-scraped metrics metadata: %s", err)
-			}
-		}
 	}
 	for {
 		select {
@@ -125,11 +104,11 @@ func selfScraper(scrapeInterval time.Duration) {
 	}
 }
 
-func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
+func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label {
 	if len(dst) < cap(dst) {
 		dst = dst[:len(dst)+1]
 	} else {
-		dst = append(dst, prompb.Label{})
+		dst = append(dst, prompbmarshal.Label{})
 	}
 	lb := &dst[len(dst)-1]
 	lb.Name = key

app/vmagent/Makefile

@@ -27,9 +27,6 @@ vmagent-linux-ppc64le-prod:
 vmagent-linux-386-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-linux-386
 
-vmagent-linux-s390x-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-linux-s390x
-
 vmagent-darwin-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
 

app/vmagent/common/push_ctx.go

@@ -3,7 +3,7 @@ package common
 import (
 	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
@@ -12,13 +12,13 @@ type PushCtx struct {
 	// WriteRequest contains the WriteRequest, which must be pushed later to remote storage.
 	//
 	// The actual labels and samples for the time series are stored in Labels and Samples fields.
-	WriteRequest prompb.WriteRequest
+	WriteRequest prompbmarshal.WriteRequest
 
 	// Labels contains flat list of all the labels used in WriteRequest.
-	Labels []prompb.Label
+	Labels []prompbmarshal.Label
 
 	// Samples contains flat list of all the samples used in WriteRequest.
-	Samples []prompb.Sample
+	Samples []prompbmarshal.Sample
 }
 
 // Reset resets ctx.

app/vmagent/csvimport/request_handler.go

@@ -6,7 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -31,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -41,23 +41,23 @@ func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompb.Label
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/datadogsketches/request_handler.go

@@ -6,7 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
@@ -33,7 +33,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -45,12 +45,12 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 		ms := sketch.ToSummary()
 		for _, m := range ms {
 			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: m.Name,
 			})
 			for _, label := range m.Labels {
-				labels = append(labels, prompb.Label{
+				labels = append(labels, prompbmarshal.Label{
 					Name:  label.Name,
 					Value: label.Value,
 				})
@@ -60,7 +60,7 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 				if name == "host" {
 					name = "exported_host"
 				}
-				labels = append(labels, prompb.Label{
+				labels = append(labels, prompbmarshal.Label{
 					Name:  name,
 					Value: value,
 				})
@@ -68,13 +68,13 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 			labels = append(labels, extraLabels...)
 			samplesLen := len(samples)
 			for _, p := range m.Points {
-				samples = append(samples, prompb.Sample{
+				samples = append(samples, prompbmarshal.Sample{
 					Timestamp: p.Timestamp,
 					Value:     p.Value,
 				})
 			}
 			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompb.TimeSeries{
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
 				Labels:  labels[labelsLen:],
 				Samples: samples[samplesLen:],
 			})

app/vmagent/datadogv1/request_handler.go

@@ -6,7 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
@@ -33,7 +33,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -45,18 +45,18 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.
 		ss := &series[i]
 		rowsTotal += len(ss.Points)
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: ss.Metric,
 		})
 		if ss.Host != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "host",
 				Value: ss.Host,
 			})
 		}
 		if ss.Device != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "device",
 				Value: ss.Device,
 			})
@@ -66,7 +66,7 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.
 			if name == "host" {
 				name = "exported_host"
 			}
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  name,
 				Value: value,
 			})
@@ -74,12 +74,12 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.
 		labels = append(labels, extraLabels...)
 		samplesLen := len(samples)
 		for _, pt := range ss.Points {
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Timestamp: pt.Timestamp(),
 				Value:     pt.Value(),
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})

app/vmagent/datadogv2/request_handler.go

@@ -6,7 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
@@ -36,7 +36,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -48,18 +48,18 @@ func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.
 		ss := &series[i]
 		rowsTotal += len(ss.Points)
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: ss.Metric,
 		})
 		for _, rs := range ss.Resources {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  rs.Type,
 				Value: rs.Name,
 			})
 		}
 		if ss.SourceTypeName != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "source_type_name",
 				Value: ss.SourceTypeName,
 			})
@@ -69,7 +69,7 @@ func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.
 			if name == "host" {
 				name = "exported_host"
 			}
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  name,
 				Value: value,
 			})
@@ -77,12 +77,12 @@ func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.
 		labels = append(labels, extraLabels...)
 		samplesLen := len(samples)
 		for _, pt := range ss.Points {
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Timestamp: pt.Timestamp * 1000,
 				Value:     pt.Value,
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})

app/vmagent/graphite/request_handler.go

@@ -6,7 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
 	"github.com/VictoriaMetrics/metrics"
@@ -36,22 +36,22 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/influx/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
@@ -60,7 +60,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 
@@ -80,13 +80,13 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 			if tag.Key == *dbLabel {
 				hasDBKey = true
 			}
-			commonLabels = append(commonLabels, prompb.Label{
+			commonLabels = append(commonLabels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		if len(db) > 0 && !hasDBKey {
-			commonLabels = append(commonLabels, prompb.Label{
+			commonLabels = append(commonLabels, prompbmarshal.Label{
 				Name:  *dbLabel,
 				Value: db,
 			})
@@ -110,16 +110,16 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 			}
 			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
 			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: metricGroup,
 			})
 			labels = append(labels, commonLabels...)
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Timestamp: r.Timestamp,
 				Value:     f.Value,
 			})
-			tssDst = append(tssDst, prompb.TimeSeries{
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
 				Labels:  labels[labelsLen:],
 				Samples: samples[len(samples)-1:],
 			})
@@ -144,7 +144,7 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 
 type pushCtx struct {
 	ctx            common.PushCtx
-	commonLabels   []prompb.Label
+	commonLabels   []prompbmarshal.Label
 	metricGroupBuf []byte
 	buf            []byte
 }

app/vmagent/main.go

@@ -27,7 +27,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -75,7 +74,7 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config and /remotewrite-.*-config pages. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
@@ -207,7 +206,7 @@ func main() {
 func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 	if !remotewrite.MultitenancyEnabled() {
 		return func(req *http.Request) error {
-			path := strings.ReplaceAll(req.URL.Path, "//", "/")
+			path := strings.Replace(req.URL.Path, "//", "/", -1)
 			if path != "/api/put" {
 				return fmt.Errorf("unsupported path requested: %q; expecting '/api/put'", path)
 			}
@@ -215,7 +214,7 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 		}
 	}
 	return func(req *http.Request) error {
-		path := strings.ReplaceAll(req.URL.Path, "//", "/")
+		path := strings.Replace(req.URL.Path, "//", "/", -1)
 		at, err := getAuthTokenFromPath(path)
 		if err != nil {
 			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
@@ -253,8 +252,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
-			{"remotewrite-relabel-config", "-remoteWrite.relabelConfig contents"},
-			{"remotewrite-url-relabel-config", "-remoteWrite.urlRelabelConfig contents"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"-/reload", "reload configuration"},
@@ -262,7 +259,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus") || strings.HasPrefix(path, "/api/v1/import/prometheus") {
 		prometheusimportRequests.Inc()
 		if err := prometheusimport.InsertHandler(nil, r); err != nil {
@@ -351,17 +348,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
-	case "/zabbixconnector/api/v1/history":
-		zabbixconnectorHistoryRequests.Inc()
-		if err := zabbixconnector.InsertHandlerForHTTP(nil, r); err != nil {
-			zabbixconnectorHistoryErrors.Inc()
-			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusBadRequest)
-			fmt.Fprintf(w, `{"error":%q}`, err.Error())
-			return true
-		}
-		w.WriteHeader(http.StatusOK)
-		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -491,42 +477,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promscrape.WriteConfigData(&bb)
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
 		return true
-	case "/remotewrite-relabel-config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
-			return true
-		}
-		remoteWriteRelabelConfigRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		remotewrite.WriteRelabelConfigData(w)
-		return true
-	case "/api/v1/status/remotewrite-relabel-config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
-			return true
-		}
-		remoteWriteStatusRelabelConfigRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		var bb bytesutil.ByteBuffer
-		remotewrite.WriteRelabelConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
-		return true
-	case "/remotewrite-url-relabel-config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
-			return true
-		}
-		remoteWriteURLRelabelConfigRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		remotewrite.WriteURLRelabelConfigData(w)
-		return true
-	case "/api/v1/status/remotewrite-url-relabel-config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
-			return true
-		}
-		remoteWriteStatusURLRelabelConfigRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		var bb bytesutil.ByteBuffer
-		remotewrite.WriteURLRelabelConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
-		return true
 	case "/prometheus/-/reload", "/-/reload":
 		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
@@ -656,17 +606,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
-	case "zabbixconnector/api/v1/history":
-		zabbixconnectorHistoryRequests.Inc()
-		if err := zabbixconnector.InsertHandlerForHTTP(at, r); err != nil {
-			zabbixconnectorHistoryErrors.Inc()
-			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusBadRequest)
-			fmt.Fprintf(w, `{"error":%q}`, err.Error())
-			return true
-		}
-		w.WriteHeader(http.StatusOK)
-		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -788,9 +727,6 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
-	zabbixconnectorHistoryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
-	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
-
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 
@@ -811,12 +747,6 @@ var (
 	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
 	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)
 
-	remoteWriteRelabelConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/remotewrite-relabel-config"}`)
-	remoteWriteStatusRelabelConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/remotewrite-relabel-config"}`)
-
-	remoteWriteURLRelabelConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/remotewrite-url-relabel-config"}`)
-	remoteWriteStatusURLRelabelConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/remotewrite-url-relabel-config"}`)
-
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
 

app/vmagent/native/request_handler.go

@@ -8,7 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
@@ -35,7 +35,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -53,13 +53,13 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompb.Label)
 	samples := ctx.Samples[:0]
 	mn := &block.MetricName
 	labelsLen := len(labels)
-	labels = append(labels, prompb.Label{
+	labels = append(labels, prompbmarshal.Label{
 		Name:  "__name__",
 		Value: bytesutil.ToUnsafeString(mn.MetricGroup),
 	})
 	for j := range mn.Tags {
 		tag := &mn.Tags[j]
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  bytesutil.ToUnsafeString(tag.Key),
 			Value: bytesutil.ToUnsafeString(tag.Value),
 		})
@@ -72,12 +72,12 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompb.Label)
 	}
 	samplesLen := len(samples)
 	for j, value := range values {
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     value,
 			Timestamp: timestamps[j],
 		})
 	}
-	tssDst = append(tssDst, prompb.TimeSeries{
+	tssDst = append(tssDst, prompbmarshal.TimeSeries{
 		Labels:  labels[labelsLen:],
 		Samples: samples[samplesLen:],
 	})

app/vmagent/newrelic/request_handler.go

@@ -9,7 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -34,7 +34,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -49,22 +49,22 @@ func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label)
 		for j := range srcSamples {
 			s := &srcSamples[j]
 			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: bytesutil.ToUnsafeString(s.Name),
 			})
 			for k := range tags {
 				t := &tags[k]
-				labels = append(labels, prompb.Label{
+				labels = append(labels, prompbmarshal.Label{
 					Name:  bytesutil.ToUnsafeString(t.Key),
 					Value: bytesutil.ToUnsafeString(t.Value),
 				})
 			}
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Value:     s.Value,
 				Timestamp: r.Timestamp,
 			})
-			tssDst = append(tssDst, prompb.TimeSeries{
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
 				Labels:  labels[labelsLen:],
 				Samples: samples[len(samples)-1:],
 			})
@@ -78,7 +78,7 @@ func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label)
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
-	rowsInserted.Add(samplesCount)
+	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(samplesCount)
 	}

app/vmagent/opentelemetry/request_handler.go

@@ -2,14 +2,12 @@ package opentelemetry
 
 import (
 	"fmt"
-	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -18,20 +16,11 @@ import (
 )
 
 var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="opentelemetry"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="opentelemetry"}`)
-	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
-// InsertHandlerForReader processes metrics from given reader.
-func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
-	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
-		return insertRows(at, tss, mms, nil)
-	})
-}
-
 // InsertHandler processes opentelemetry metrics.
 func InsertHandler(at *auth.Token, req *http.Request) error {
 	extraLabels, err := protoparserutil.GetExtraLabels(req)
@@ -47,12 +36,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
-		return insertRows(at, tss, mms, extraLabels)
+	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompbmarshal.TimeSeries) error {
+		return insertRows(at, tss, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -68,39 +57,20 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 		labels = append(labels, extraLabels...)
 		samplesLen := len(samples)
 		samples = append(samples, ts.Samples...)
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
 	}
 	ctx.WriteRequest.Timeseries = tssDst
-
-	var metadataTotal int
-	if prommetadata.IsEnabled() {
-		var accountID, projectID uint32
-		if at != nil {
-			accountID = at.AccountID
-			projectID = at.ProjectID
-			for i := range mms {
-				mm := &mms[i]
-				mm.AccountID = accountID
-				mm.ProjectID = projectID
-			}
-		}
-		ctx.WriteRequest.Metadata = mms
-		metadataTotal = len(mms)
-	}
-
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
 	rowsInserted.Add(rowsTotal)
-	metadataInserted.Add(metadataTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
-		metadataTenantInserted.Get(at).Add(metadataTotal)
 	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil

app/vmagent/opentsdb/request_handler.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
 	"github.com/VictoriaMetrics/metrics"
@@ -33,22 +33,22 @@ func insertRows(rows []parser.Row) error {
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/opentsdbhttp/request_handler.go

@@ -6,7 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -30,7 +30,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -40,23 +40,23 @@ func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompb.La
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/prometheusimport/request_handler.go

@@ -7,8 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -17,12 +16,9 @@ import (
 )
 
 var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="prometheus"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="prometheus"}`)
-
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
@@ -36,74 +32,53 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, prommetadata.IsEnabled(), func(rows []prometheus.Row, mms []prometheus.Metadata) error {
-		return insertRows(at, rows, mms, extraLabels)
+	return stream.Parse(req.Body, defaultTimestamp, encoding, true, func(rows []prometheus.Row) error {
+		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)
 	})
 }
 
-func insertRows(at *auth.Token, rows []prometheus.Row, mms []prometheus.Metadata, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []prometheus.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
 	tssDst := ctx.WriteRequest.Timeseries[:0]
-	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})
 	}
-	var accountID, projectID uint32
-	if at != nil {
-		accountID = at.AccountID
-		projectID = at.ProjectID
-	}
-	for i := range mms {
-		mm := &mms[i]
-		mmsDst = append(mmsDst, prompb.MetricMetadata{
-			MetricFamilyName: mm.Metric,
-			Help:             mm.Help,
-			Type:             mm.Type,
-			// there is no unit in Prometheus exposition formats
-
-			AccountID: accountID,
-			ProjectID: projectID,
-		})
-	}
 	ctx.WriteRequest.Timeseries = tssDst
-	ctx.WriteRequest.Metadata = mmsDst
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
 	rowsInserted.Add(len(rows))
-	metadataInserted.Add(len(mms))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))
-		metadataTenantInserted.Get(at).Add(len(mms))
 	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler_test.go

@@ -55,6 +55,6 @@ func tearDown() {
 	srv.Close()
 	logger.ResetOutputForTest()
 	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()
-	fs.MustRemoveDir(tmpDataDir)
+	fs.MustRemoveAll(tmpDataDir)
 
 }

app/vmagent/promremotewrite/request_handler.go

@@ -6,8 +6,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
@@ -15,11 +15,9 @@ import (
 )
 
 var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="promremotewrite"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="promremotewrite"}`)
-	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
@@ -29,18 +27,17 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
-		return insertRows(at, tss, mms, extraLabels)
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
+		return insertRows(at, tss, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
-	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range timeseries {
@@ -49,7 +46,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.Met
 		labelsLen := len(labels)
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  label.Name,
 				Value: label.Value,
 			})
@@ -58,41 +55,17 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.Met
 		samplesLen := len(samples)
 		for i := range ts.Samples {
 			sample := &ts.Samples[i]
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Value:     sample.Value,
 				Timestamp: sample.Timestamp,
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
 	}
 	ctx.WriteRequest.Timeseries = tssDst
-
-	var metadataTotal int
-	if prommetadata.IsEnabled() {
-		var accountID, projectID uint32
-		if at != nil {
-			accountID = at.AccountID
-			projectID = at.ProjectID
-		}
-		for i := range mms {
-			mm := &mms[i]
-			mmsDst = append(mmsDst, prompb.MetricMetadata{
-				MetricFamilyName: mm.MetricFamilyName,
-				Help:             mm.Help,
-				Type:             mm.Type,
-				Unit:             mm.Unit,
-
-				AccountID: accountID,
-				ProjectID: projectID,
-			})
-		}
-		ctx.WriteRequest.Metadata = mmsDst
-		metadataTotal = len(mms)
-	}
-
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
@@ -101,9 +74,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.Met
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
-		metadataTenantInserted.Get(at).Add(metadataTotal)
 	}
-	metadataInserted.Add(metadataTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmagent/remotewrite/client.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -447,8 +448,7 @@ again:
 	}
 
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
-	switch statusCode {
-	case 409:
+	if statusCode == 409 {
 		logBlockRejected(block, c.sanitizedURL, resp)
 
 		// Just drop block on 409 status code like Prometheus does.
@@ -461,28 +461,22 @@ again:
 		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
 		// - Real-world implementations of v1 use both 400 and 415 status codes.
 		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
-	case 415, 400:
+	} else if statusCode == 415 || statusCode == 400 {
+		if c.canDowngradeVMProto.Swap(false) {
+			logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
+				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
+			c.useVMProto.Store(false)
+		}
+
 		if encoding.IsZstd(block) {
 			logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
 				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
 
-			zstdBlockLen := len(block)
-			block, err = repackBlockFromZstdToSnappy(block)
-			if err == nil {
-				if c.canDowngradeVMProto.Swap(false) {
-					logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
-						"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-					c.useVMProto.Store(false)
-				}
+			block = mustRepackBlockFromZstdToSnappy(block)
 
-				c.retriesCount.Inc()
-				_ = resp.Body.Close()
-				goto again
-			}
-
-			logger.Warnf("failed to repack zstd block (%s bytes) to snappy: %s; The block will be rejected. "+
-				"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
-				zstdBlockLen, err)
+			c.retriesCount.Inc()
+			_ = resp.Body.Close()
+			goto again
 		}
 
 		// Just drop snappy blocks on 400 or 415 status codes like Prometheus does.
@@ -544,21 +538,14 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 	return retryDuration
 }
 
-// repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
-//
-// The input block may be corrupted, for example, if vmagent was shut down ungracefully and
-// failed to properly update the persisted queue files. In such cases, zstd decompression
-// will fail and an error will be returned.
-//
-// For more details, see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9417
-func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
+func mustRepackBlockFromZstdToSnappy(zstdBlock []byte) []byte {
 	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
+	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
 	if err != nil {
-		return nil, err
+		logger.Panicf("FATAL: cannot re-pack block with size %d bytes from Zstd to Snappy: %s", len(zstdBlock), err)
 	}
 
-	return snappy.Encode(nil, plainBlock), nil
+	return snappy.Encode(nil, plainBlock)
 }
 
 func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {

app/vmagent/remotewrite/client_test.go

@@ -25,7 +25,7 @@ func TestCalculateRetryDuration(t *testing.T) {
 		expectMaxDuration := helper(expectMinDuration)
 		expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
 
-		if retryDuration < expectMinDuration || retryDuration > expectMaxDuration {
+		if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
 			t.Fatalf(
 				"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
 				expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
@@ -93,7 +93,10 @@ func TestParseRetryAfterHeader(t *testing.T) {
 
 // helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
 func helper(d time.Duration) time.Duration {
-	dv := min(d/10, 10*time.Second)
+	dv := d / 10
+	if dv > 10*time.Second {
+		dv = 10 * time.Second
+	}
 
 	return d + dv
 }
@@ -102,10 +105,7 @@ func TestRepackBlockFromZstdToSnappy(t *testing.T) {
 	expectedPlainBlock := []byte(`foobar`)
 
 	zstdBlock := encoding.CompressZSTDLevel(nil, expectedPlainBlock, 1)
-	snappyBlock, err := repackBlockFromZstdToSnappy(zstdBlock)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
+	snappyBlock := mustRepackBlockFromZstdToSnappy(zstdBlock)
 
 	actualPlainBlock, err := snappy.Decode(nil, snappyBlock)
 	if err != nil {
@@ -116,14 +116,3 @@ func TestRepackBlockFromZstdToSnappy(t *testing.T) {
 		t.Fatalf("unexpected plain block; got %q; want %q", actualPlainBlock, expectedPlainBlock)
 	}
 }
-
-func TestRepackBlockFromZstdToSnappyInvalidBlock(t *testing.T) {
-	snappyBlock, err := repackBlockFromZstdToSnappy([]byte("invalid zstd block"))
-
-	if err == nil {
-		t.Fatalf("expected error for invalid zstd block; got nil")
-	}
-	if len(snappyBlock) != 0 {
-		t.Fatalf("expected empty snappy block; got %d bytes", len(snappyBlock))
-	}
-}

app/vmagent/remotewrite/pendingseries.go

@@ -14,7 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
@@ -24,10 +24,9 @@ import (
 
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
-		"This option takes effect only when less than -remoteWrite.maxRowsPerBlock data points per -remoteWrite.flushInterval are pushed to -remoteWrite.url")
+		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
 	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
-	maxMetadataPerBlock  = flag.Int("remoteWrite.maxMetadataPerBlock", 5000, "The maximum number of metadata to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
 		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
@@ -61,16 +60,9 @@ func (ps *pendingSeries) MustStop() {
 	ps.periodicFlusherWG.Wait()
 }
 
-func (ps *pendingSeries) TryPushTimeSeries(tss []prompb.TimeSeries) bool {
+func (ps *pendingSeries) TryPush(tss []prompbmarshal.TimeSeries) bool {
 	ps.mu.Lock()
-	ok := ps.wr.tryPushTimeSeries(tss)
-	ps.mu.Unlock()
-	return ok
-}
-
-func (ps *pendingSeries) TryPushMetadata(mms []prompb.MetricMetadata) bool {
-	ps.mu.Lock()
-	ok := ps.wr.tryPushMetadata(mms)
+	ok := ps.wr.tryPush(tss)
 	ps.mu.Unlock()
 	return ok
 }
@@ -116,37 +108,29 @@ type writeRequest struct {
 	// How many decimal digits after point must be left before sending the writeRequest to fq.
 	roundDigits int
 
-	wr prompb.WriteRequest
+	wr prompbmarshal.WriteRequest
 
-	tss     []prompb.TimeSeries
-	mms     []prompb.MetricMetadata
-	labels  []prompb.Label
-	samples []prompb.Sample
+	tss     []prompbmarshal.TimeSeries
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
 
 	// buf holds labels data
 	buf []byte
-	// metadatabuf holds metadata data
-	metadatabuf []byte
 }
 
 func (wr *writeRequest) reset() {
 	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are reused.
 
 	wr.wr.Timeseries = nil
-	wr.wr.Metadata = nil
 
 	clear(wr.tss)
 	wr.tss = wr.tss[:0]
 
-	clear(wr.mms)
-	wr.mms = wr.mms[:0]
-
 	promrelabel.CleanLabels(wr.labels)
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
 	wr.buf = wr.buf[:0]
-	wr.metadatabuf = wr.metadatabuf[:0]
 }
 
 // mustFlushOnStop force pushes wr data into wr.fq
@@ -154,7 +138,6 @@ func (wr *writeRequest) reset() {
 // This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
 func (wr *writeRequest) mustFlushOnStop() {
 	wr.wr.Timeseries = wr.tss
-	wr.wr.Metadata = wr.mms
 	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
 		logger.Panicf("BUG: final flush must always return true")
 	}
@@ -168,7 +151,6 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
-	wr.wr.Metadata = wr.mms
 	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
 	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
 		return false
@@ -177,7 +159,7 @@ func (wr *writeRequest) tryFlush() bool {
 	return true
 }
 
-func adjustSampleValues(samples []prompb.Sample, significantFigures, roundDigits int) {
+func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roundDigits int) {
 	if n := significantFigures; n > 0 {
 		for i := range samples {
 			s := &samples[i]
@@ -192,49 +174,7 @@ func adjustSampleValues(samples []prompb.Sample, significantFigures, roundDigits
 	}
 }
 
-func (wr *writeRequest) tryPushMetadata(mms []prompb.MetricMetadata) bool {
-	mmdDst := wr.mms
-	maxMetadataPerBlock := *maxMetadataPerBlock
-	for i := range mms {
-		if len(wr.mms) >= maxMetadataPerBlock {
-			if !wr.tryFlush() {
-				return false
-			}
-			mmdDst = wr.mms
-		}
-		mmSrc := &mms[i]
-		mmdDst = append(mmdDst, prompb.MetricMetadata{})
-		wr.copyMetadata(&mmdDst[len(mmdDst)-1], mmSrc)
-	}
-	wr.mms = mmdDst
-	return true
-}
-
-func (wr *writeRequest) copyMetadata(dst, src *prompb.MetricMetadata) {
-	// Direct copy for non-string fields, which are safe by value.
-	dst.Type = src.Type
-	dst.Unit = src.Unit
-
-	// Pre-allocate memory for all string fields.
-	neededBufLen := len(src.MetricFamilyName) + len(src.Help)
-	bufLen := len(wr.metadatabuf)
-	wr.metadatabuf = slicesutil.SetLength(wr.metadatabuf, bufLen+neededBufLen)
-	buf := wr.metadatabuf[:bufLen]
-
-	// Copy MetricFamilyName
-	bufLen = len(buf)
-	buf = append(buf, src.MetricFamilyName...)
-	dst.MetricFamilyName = bytesutil.ToUnsafeString(buf[bufLen:])
-
-	// Copy Help
-	bufLen = len(buf)
-	buf = append(buf, src.Help...)
-	dst.Help = bytesutil.ToUnsafeString(buf[bufLen:])
-
-	wr.metadatabuf = buf
-}
-
-func (wr *writeRequest) tryPushTimeSeries(src []prompb.TimeSeries) bool {
+func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
 	tssDst := wr.tss
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
@@ -249,7 +189,7 @@ func (wr *writeRequest) tryPushTimeSeries(src []prompb.TimeSeries) bool {
 		}
 		tsSrc := &src[i]
 		adjustSampleValues(tsSrc.Samples, wr.significantFigures, wr.roundDigits)
-		tssDst = append(tssDst, prompb.TimeSeries{})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
 		wr.copyTimeSeries(&tssDst[len(tssDst)-1], tsSrc)
 	}
 
@@ -257,7 +197,7 @@ func (wr *writeRequest) tryPushTimeSeries(src []prompb.TimeSeries) bool {
 	return true
 }
 
-func (wr *writeRequest) copyTimeSeries(dst, src *prompb.TimeSeries) {
+func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	labelsSrc := src.Labels
 
 	// Pre-allocate memory for labels.
@@ -300,8 +240,8 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompb.TimeSeries) {
 // marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
 var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
 
-func tryPushWriteRequest(wr *prompb.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
-	if wr.IsEmpty() {
+func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
+	if len(wr.Timeseries) == 0 {
 		// Nothing to push
 		return true
 	}
@@ -327,7 +267,6 @@ func tryPushWriteRequest(wr *prompb.WriteRequest, tryPushBlock func(block []byte
 			compressBufPool.Put(zb)
 			if ok {
 				blockSizeRows.Update(float64(len(wr.Timeseries)))
-				blockMetadataRows.Update(float64(len(wr.Metadata)))
 				blockSizeBytes.Update(float64(zbLen))
 			}
 			return ok
@@ -339,86 +278,47 @@ func tryPushWriteRequest(wr *prompb.WriteRequest, tryPushBlock func(block []byte
 		<-marshalConcurrencyCh
 	}
 
-	// Split timeseries or metadata into two smaller blocks
-	switch len(wr.Timeseries) {
-	case 0:
-		if len(wr.Metadata) == 1 {
-			logger.Warnf("dropping a metadata exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
-			return true
-		}
-		metadata := wr.Metadata
-		n := len(metadata) / 2
-		wr.Metadata = metadata[:n]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Metadata = metadata
-			return false
-		}
-		wr.Metadata = metadata[n:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Metadata = metadata
-			return false
-		}
-		wr.Metadata = metadata
-		return true
-
-	case 1:
-		// A single time series left. Recursively split its samples and metadata into smaller parts if possible.
+	// Too big block. Recursively split it into smaller parts if possible.
+	if len(wr.Timeseries) == 1 {
+		// A single time series left. Recursively split its samples into smaller parts if possible.
 		samples := wr.Timeseries[0].Samples
-		metaData := wr.Metadata
-		if len(samples) == 1 && len(metaData) <= 1 {
-			logger.Warnf("dropping a sample for metric and %d metadata which are exceeding -remoteWrite.maxBlockSize=%d bytes", len(metaData), maxUnpackedBlockSize.N)
+		if len(samples) == 1 {
+			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
 			return true
 		}
 		n := len(samples) / 2
-		m := len(metaData) / 2
 		wr.Timeseries[0].Samples = samples[:n]
-		wr.Metadata = metaData[:m]
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
-			wr.Metadata = metaData
 			return false
 		}
 		wr.Timeseries[0].Samples = samples[n:]
-		wr.Metadata = metaData[m:]
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
-			wr.Metadata = metaData
 			return false
 		}
 		wr.Timeseries[0].Samples = samples
-		wr.Metadata = metaData
-		return true
-
-	default:
-		// Split both timeseries and metadata.
-		timeseries := wr.Timeseries
-		metaData := wr.Metadata
-		n := len(timeseries) / 2
-		m := len(metaData) / 2
-		wr.Timeseries = timeseries[:n]
-		wr.Metadata = metaData[:m]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries = timeseries
-			wr.Metadata = metaData
-			return false
-		}
-		wr.Timeseries = timeseries[n:]
-		wr.Metadata = metaData[m:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries = timeseries
-			wr.Metadata = metaData
-			return false
-		}
-		wr.Timeseries = timeseries
-		wr.Metadata = metaData
 		return true
 	}
+	timeseries := wr.Timeseries
+	n := len(timeseries) / 2
+	wr.Timeseries = timeseries[:n]
+	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+		wr.Timeseries = timeseries
+		return false
+	}
+	wr.Timeseries = timeseries[n:]
+	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
+		wr.Timeseries = timeseries
+		return false
+	}
+	wr.Timeseries = timeseries
+	return true
 }
 
 var (
-	blockSizeBytes    = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
-	blockSizeRows     = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
-	blockMetadataRows = metrics.NewHistogram(`vmagent_remotewrite_block_metadata_rows`)
+	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
 var (

app/vmagent/remotewrite/pendingseries_test.go

@@ -5,7 +5,7 @@ import (
 	"math"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestPushWriteRequest(t *testing.T) {
@@ -49,19 +49,19 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 	f(true, expectedBlockLenVM, 15)
 }
 
-func newTestWriteRequest(seriesCount, labelsCount int) *prompb.WriteRequest {
-	var wr prompb.WriteRequest
+func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {
+	var wr prompbmarshal.WriteRequest
 	for i := 0; i < seriesCount; i++ {
-		var labels []prompb.Label
+		var labels []prompbmarshal.Label
 		for j := 0; j < labelsCount; j++ {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  fmt.Sprintf("label_%d_%d", i, j),
 				Value: fmt.Sprintf("value_%d_%d", i, j),
 			})
 		}
-		wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
+		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
 			Labels: labels,
-			Samples: []prompb.Sample{
+			Samples: []prompbmarshal.Sample{
 				{
 					Value:     float64(i),
 					Timestamp: 1000 * int64(i),

app/vmagent/remotewrite/relabel.go

@@ -3,18 +3,15 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"io"
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"go.yaml.in/yaml/v3"
 
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -35,12 +32,9 @@ var (
 		"See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels")
 )
 
+var labelsGlobal []prompbmarshal.Label
+
 var (
-	labelsGlobal []prompb.Label
-
-	remoteWriteRelabelConfigData    atomic.Pointer[[]byte]
-	remoteWriteURLRelabelConfigData atomic.Pointer[[]interface{}]
-
 	relabelConfigReloads      *metrics.Counter
 	relabelConfigReloadErrors *metrics.Counter
 	relabelConfigSuccess      *metrics.Gauge
@@ -73,42 +67,6 @@ func initRelabelConfigs() {
 	}
 }
 
-// WriteRelabelConfigData writes -remoteWrite.relabelConfig contents to w
-func WriteRelabelConfigData(w io.Writer) {
-	p := remoteWriteRelabelConfigData.Load()
-	if p == nil {
-		// Nothing to write to w
-		return
-	}
-	_, _ = w.Write(*p)
-}
-
-// WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
-func WriteURLRelabelConfigData(w io.Writer) {
-	p := remoteWriteURLRelabelConfigData.Load()
-	if p == nil {
-		// Nothing to write to w
-		return
-	}
-	type urlRelabelCfg struct {
-		Url           string      `yaml:"url"`
-		RelabelConfig interface{} `yaml:"relabel_config"`
-	}
-	var cs []urlRelabelCfg
-	for i, url := range *remoteWriteURLs {
-		cfgData := (*p)[i]
-		if !*showRemoteWriteURL {
-			url = fmt.Sprintf("%d:secret-url", i+1)
-		}
-		cs = append(cs, urlRelabelCfg{
-			Url:           url,
-			RelabelConfig: cfgData,
-		})
-	}
-	d, _ := yaml.Marshal(cs)
-	_, _ = w.Write(d)
-}
-
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {
@@ -132,42 +90,28 @@ func reloadRelabelConfigs() {
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
-		global, rawCfg, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
 		}
-		remoteWriteRelabelConfigData.Store(&rawCfg)
 		rcs.global = global
 	}
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
 	}
-
-	var urlRelabelCfgs []interface{}
 	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
-			urlRelabelCfgs = append(urlRelabelCfgs, nil)
+			// Skip empty relabel config.
 			continue
 		}
-		prc, rawCfg, err := promrelabel.LoadRelabelConfigs(path)
+		prc, err := promrelabel.LoadRelabelConfigs(path)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
 		}
 		rcs.perURL[i] = prc
-
-		var parsedCfg interface{}
-		_ = yaml.Unmarshal(rawCfg, &parsedCfg)
-		urlRelabelCfgs = append(urlRelabelCfgs, parsedCfg)
 	}
-	if len(*remoteWriteURLs) > len(*relabelConfigPaths) {
-		// fill the urlRelabelCfgs with empty relabel configs if not set
-		for i := len(*relabelConfigPaths); i < len(*remoteWriteURLs); i++ {
-			urlRelabelCfgs = append(urlRelabelCfgs, nil)
-		}
-	}
-	remoteWriteURLRelabelConfigData.Store(&urlRelabelCfgs)
 	return &rcs, nil
 }
 
@@ -202,14 +146,14 @@ func initLabelsGlobal() {
 		if n < 0 {
 			logger.Fatalf("missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
 		}
-		labelsGlobal = append(labelsGlobal, prompb.Label{
+		labelsGlobal = append(labelsGlobal, prompbmarshal.Label{
 			Name:  s[:n],
 			Value: s[n+1:],
 		})
 	}
 }
 
-func (rctx *relabelCtx) applyRelabeling(tss []prompb.TimeSeries, pcs *promrelabel.ParsedConfigs) []prompb.TimeSeries {
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
 	if pcs.Len() == 0 && !*usePromCompatibleNaming {
 		// Nothing to change.
 		return tss
@@ -230,7 +174,7 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompb.TimeSeries, pcs *promrelabe
 		if *usePromCompatibleNaming {
 			fixPromCompatibleNaming(labels[labelsLen:])
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: ts.Samples,
 		})
@@ -239,7 +183,7 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompb.TimeSeries, pcs *promrelabe
 	return tssDst
 }
 
-func (rctx *relabelCtx) appendExtraLabels(tss []prompb.TimeSeries, extraLabels []prompb.Label) {
+func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) {
 	if len(extraLabels) == 0 {
 		return
 	}
@@ -263,7 +207,7 @@ func (rctx *relabelCtx) appendExtraLabels(tss []prompb.TimeSeries, extraLabels [
 	rctx.labels = labels
 }
 
-func (rctx *relabelCtx) tenantToLabels(tss []prompb.TimeSeries, accountID, projectID uint32) {
+func (rctx *relabelCtx) tenantToLabels(tss []prompbmarshal.TimeSeries, accountID, projectID uint32) {
 	rctx.reset()
 	accountIDStr := strconv.FormatUint(uint64(accountID), 10)
 	projectIDStr := strconv.FormatUint(uint64(projectID), 10)
@@ -278,11 +222,11 @@ func (rctx *relabelCtx) tenantToLabels(tss []prompb.TimeSeries, accountID, proje
 			}
 			labels = append(labels, label)
 		}
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "vm_account_id",
 			Value: accountIDStr,
 		})
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "vm_project_id",
 			Value: projectIDStr,
 		})
@@ -293,7 +237,7 @@ func (rctx *relabelCtx) tenantToLabels(tss []prompb.TimeSeries, accountID, proje
 
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
-	labels []prompb.Label
+	labels []prompbmarshal.Label
 }
 
 func (rctx *relabelCtx) reset() {
@@ -316,7 +260,7 @@ func putRelabelCtx(rctx *relabelCtx) {
 	relabelCtxPool.Put(rctx)
 }
 
-func fixPromCompatibleNaming(labels []prompb.Label) {
+func fixPromCompatibleNaming(labels []prompbmarshal.Label) {
 	// Replace unsupported Prometheus chars in label names and metric names with underscores.
 	for i := range labels {
 		label := &labels[i]

app/vmagent/remotewrite/relabel_test.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 )
@@ -39,7 +39,7 @@ func TestApplyRelabeling(t *testing.T) {
 }
 
 func TestAppendExtraLabels(t *testing.T) {
-	f := func(extraLabels []prompb.Label, sTss, sExpTss string) {
+	f := func(extraLabels []prompbmarshal.Label, sTss, sExpTss string) {
 		t.Helper()
 		rctx := &relabelCtx{}
 		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
@@ -50,19 +50,19 @@ func TestAppendExtraLabels(t *testing.T) {
 	}
 
 	f(nil, "up", "up")
-	f([]prompb.Label{{Name: "foo", Value: "bar"}}, "up", `up{foo="bar"}`)
-	f([]prompb.Label{{Name: "foo", Value: "bar"}}, `up{foo="baz"}`, `up{foo="bar"}`)
-	f([]prompb.Label{{Name: "baz", Value: "qux"}}, `up{foo="baz"}`, `up{foo="baz",baz="qux"}`)
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, "up", `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, `up{foo="baz"}`, `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "baz", Value: "qux"}}, `up{foo="baz"}`, `up{foo="baz",baz="qux"}`)
 
 	oldVal := *usePromCompatibleNaming
 	*usePromCompatibleNaming = true
-	f([]prompb.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo.bar="baz"}`)
+	f([]prompbmarshal.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo.bar="baz"}`)
 	*usePromCompatibleNaming = oldVal
 }
 
-func parseSeries(data string) []prompb.TimeSeries {
-	var tss []prompb.TimeSeries
-	tss = append(tss, prompb.TimeSeries{
+func parseSeries(data string) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, prompbmarshal.TimeSeries{
 		Labels: promutil.MustNewLabelsFromString(data).GetLabels(),
 	})
 	return tss

app/vmagent/remotewrite/remotewrite.go

@@ -23,11 +23,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 	"github.com/VictoriaMetrics/metrics"
@@ -210,7 +209,7 @@ func Init() {
 	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
 	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
 	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
-	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(*remoteWriteURLs) > 1
+	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
 
 	dropDanglingQueues()
 
@@ -254,7 +253,7 @@ func dropDanglingQueues() {
 		if _, ok := existingQueues[dirname]; !ok {
 			logger.Infof("removing dangling queue %q", dirname)
 			fullPath := filepath.Join(queuesDir, dirname)
-			fs.MustRemoveDir(fullPath)
+			fs.MustRemoveAll(fullPath)
 			removed++
 		}
 	}
@@ -373,7 +372,7 @@ func Stop() {
 // PushDropSamplesOnFailure drops wr samples if they cannot be sent to -remoteWrite.url by any reason.
 //
 // PushDropSamplesOnFailure can modify wr contents.
-func PushDropSamplesOnFailure(at *auth.Token, wr *prompb.WriteRequest) {
+func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
 }
 
@@ -383,13 +382,19 @@ func PushDropSamplesOnFailure(at *auth.Token, wr *prompb.WriteRequest) {
 // TryPush may send partial data from wr on unsuccessful attempt, so repeated call for the same wr may send the data multiple times.
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
-func TryPush(at *auth.Token, wr *prompb.WriteRequest) bool {
+func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
 	return tryPush(at, wr, dropSamplesOnFailureGlobal)
 }
 
-func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure bool) bool {
+func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
 	tss := wr.Timeseries
-	mms := wr.Metadata
+
+	var tenantRctx *relabelCtx
+	if at != nil {
+		// Convert at to (vm_account_id, vm_project_id) labels.
+		tenantRctx = getRelabelCtx()
+		defer putRelabelCtx(tenantRctx)
+	}
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
@@ -406,23 +411,6 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 		return true
 	}
 
-	// Push metadata separately from time series, since it doesn't need sharding,
-	// relabeling, stream aggregation, deduplication, etc.
-	if !tryPushMetadataToRemoteStorages(rwctxs, mms, forceDropSamplesOnFailure) {
-		return false
-	}
-
-	if len(tss) == 0 {
-		return true
-	}
-
-	var tenantRctx *relabelCtx
-	if at != nil {
-		// Convert at to (vm_account_id, vm_project_id) labels.
-		tenantRctx = getRelabelCtx()
-		defer putRelabelCtx(tenantRctx)
-	}
-
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
@@ -486,9 +474,6 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 			matchIdxs.B = sas.Push(tssBlock, matchIdxs.B)
 			if !*streamAggrGlobalKeepInput {
 				tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
-			} else if *streamAggrGlobalDropInput {
-				// if both keep_input and drop_input are true, we keep only the aggregated series
-				tssBlock = dropUnaggregatedSeries(tssBlock, matchIdxs.B)
 			}
 			matchIdxsPool.Put(matchIdxs)
 		}
@@ -496,7 +481,7 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
-		if !tryPushTimeSeriesToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
@@ -507,7 +492,7 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 // returns only the unblocked rwctx.
 //
 // calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
-func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
+func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
 	if !disableOnDiskQueueAny {
 		return rwctxsGlobal, true
 	}
@@ -535,49 +520,18 @@ func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailu
 	return rwctxs, true
 }
 
-func pushTimeSeriesToRemoteStoragesTrackDropped(tss []prompb.TimeSeries) {
+func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
 	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
 	if len(rwctxs) == 0 {
 		return
 	}
 
-	if !tryPushTimeSeriesToRemoteStorages(rwctxs, tss, true) {
-		logger.Panicf("BUG: tryPushTimeSeriesToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
+	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
+		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
 	}
 }
 
-func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.MetricMetadata, forceDropSamplesOnFailure bool) bool {
-	if len(mms) == 0 {
-		// Nothing to push
-		return true
-	}
-	// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
-	// Since metadata is usually small and there is no guarantee that metadata can be sent to
-	// the same remote storage with the corresponding metrics.
-	//
-	// Push metadata to remote storage systems in parallel to reduce
-	// the time needed for sending the data to multiple remote storage systems.
-	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
-	var anyPushFailed atomic.Bool
-	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
-			if !rwctx.tryPushMetadataInternal(mms) {
-				rwctx.pushFailures.Inc()
-				if forceDropSamplesOnFailure {
-					rwctx.metadataDroppedOnPushFailure.Add(len(mms))
-					return
-				}
-				anyPushFailed.Store(true)
-			}
-		}(rwctx)
-	}
-	wg.Wait()
-	return !anyPushFailed.Load()
-}
-
-func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -585,7 +539,7 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 
 	if len(rwctxs) == 1 {
 		// Fast path - just push data to the configured single remote storage
-		return rwctxs[0].TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure)
+		return rwctxs[0].TryPush(tssBlock, forceDropSamplesOnFailure)
 	}
 
 	// We need to push tssBlock to multiple remote storages.
@@ -596,11 +550,11 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingTimeSeriesAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
-	// Push tssBlock to remote storage systems in parallel to reduce
+	// Push tssBlock to remote storage systems in parallel in order to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
 	wg.Add(len(rwctxs))
@@ -608,7 +562,7 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 	for _, rwctx := range rwctxs {
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
-			if !rwctx.TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure) {
+			if !rwctx.TryPush(tssBlock, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
 		}(rwctx)
@@ -617,7 +571,7 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 	return !anyPushFailed.Load()
 }
 
-func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompb.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -634,9 +588,9 @@ func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock
 			continue
 		}
 		wg.Add(1)
-		go func(rwctx *remoteWriteCtx, tss []prompb.TimeSeries) {
+		go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
 			defer wg.Done()
-			if !rwctx.TryPushTimeSeries(tss, forceDropSamplesOnFailure) {
+			if !rwctx.TryPush(tss, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
 		}(rwctx, shard)
@@ -670,7 +624,7 @@ func calculateHealthyRwctxIdx(healthyRwctxs []*remoteWriteCtx) ([]int, []int) {
 }
 
 // shardAmountRemoteWriteCtx distribute time series to shards by consistent hashing.
-func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.TimeSeries, rwctxs []*remoteWriteCtx, replicas int) {
+func shardAmountRemoteWriteCtx(tssBlock []prompbmarshal.TimeSeries, shards [][]prompbmarshal.TimeSeries, rwctxs []*remoteWriteCtx, replicas int) {
 	tmpLabels := promutil.GetLabels()
 	defer promutil.PutLabels(tmpLabels)
 
@@ -726,7 +680,7 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
 }
 
 type tssShards struct {
-	shards [][]prompb.TimeSeries
+	shards [][]prompbmarshal.TimeSeries
 }
 
 func getTSSShards(n int) *tssShards {
@@ -736,7 +690,7 @@ func getTSSShards(n int) *tssShards {
 	}
 	x := v.(*tssShards)
 	if cap(x.shards) < n {
-		x.shards = make([][]prompb.TimeSeries, n)
+		x.shards = make([][]prompbmarshal.TimeSeries, n)
 	}
 	x.shards = x.shards[:n]
 	return x
@@ -754,7 +708,7 @@ func putTSSShards(x *tssShards) {
 var tssShardsPool sync.Pool
 
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
-func sortLabelsIfNeeded(tss []prompb.TimeSeries) {
+func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
 	if !*sortLabels {
 		return
 	}
@@ -763,11 +717,11 @@ func sortLabelsIfNeeded(tss []prompb.TimeSeries) {
 	}
 }
 
-func limitSeriesCardinality(tss []prompb.TimeSeries) []prompb.TimeSeries {
+func limitSeriesCardinality(tss []prompbmarshal.TimeSeries) []prompbmarshal.TimeSeries {
 	if hourlySeriesLimiter == nil && dailySeriesLimiter == nil {
 		return tss
 	}
-	dst := make([]prompb.TimeSeries, 0, len(tss))
+	dst := make([]prompbmarshal.TimeSeries, 0, len(tss))
 	for i := range tss {
 		labels := tss[i].Labels
 		h := getLabelsHash(labels)
@@ -794,7 +748,7 @@ var (
 	dailySeriesLimitRowsDropped  = metrics.NewCounter(`vmagent_daily_series_limit_rows_dropped_total`)
 )
 
-func getLabelsHash(labels []prompb.Label) uint64 {
+func getLabelsHash(labels []prompbmarshal.Label) uint64 {
 	bb := labelsHashBufPool.Get()
 	b := bb.B[:0]
 	for _, label := range labels {
@@ -809,12 +763,12 @@ func getLabelsHash(labels []prompb.Label) uint64 {
 
 var labelsHashBufPool bytesutil.ByteBufferPool
 
-func logSkippedSeries(labels []prompb.Label, flagName string, flagValue int) {
+func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) {
 	select {
 	case <-logSkippedSeriesTicker.C:
 		// Do not use logger.WithThrottler() here, since this will increase CPU usage
-		// because every call to logSkippedSeries will result to a call to prompb.LabelsToString.
-		logger.Warnf("skip series %s because %s=%d reached", prompb.LabelsToString(labels), flagName, flagValue)
+		// because every call to logSkippedSeries will result to a call to prompbmarshal.LabelsToString.
+		logger.Warnf("skip series %s because %s=%d reached", prompbmarshal.LabelsToString(labels), flagName, flagValue)
 	default:
 	}
 }
@@ -843,9 +797,8 @@ type remoteWriteCtx struct {
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
 
-	pushFailures                 *metrics.Counter
-	metadataDroppedOnPushFailure *metrics.Counter
-	rowsDroppedOnPushFailure     *metrics.Counter
+	pushFailures             *metrics.Counter
+	rowsDroppedOnPushFailure *metrics.Counter
 }
 
 func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
@@ -909,9 +862,8 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 
-		pushFailures:                 metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		metadataDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_metadata_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure:     metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
 	rwctx.initStreamAggrConfig()
 
@@ -945,17 +897,17 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.rowsDroppedByRelabel = nil
 }
 
-// TryPushTimeSeries sends tss series to the configured remote write endpoint
+// TryPush sends tss series to the configured remote write endpoint
 //
-// TryPushTimeSeries doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
-func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
+// TryPush doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
+func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	var rctx *relabelCtx
-	var v *[]prompb.TimeSeries
+	var v *[]prompbmarshal.TimeSeries
 	defer func() {
 		if rctx == nil {
 			return
 		}
-		*v = prompb.ResetTimeSeries(tss)
+		*v = prompbmarshal.ResetTimeSeries(tss)
 		tssPool.Put(v)
 		putRelabelCtx(rctx)
 	}()
@@ -969,7 +921,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 		// from affecting time series for other remoteWrite.url configs.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
-		v = tssPool.Get().(*[]prompb.TimeSeries)
+		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
 		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, pcs)
@@ -988,21 +940,11 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 			if rctx == nil {
 				rctx = getRelabelCtx()
 				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
+				v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
 				tss = append(*v, tss...)
 			}
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
-		} else if rwctx.streamAggrDropInput {
-			// if both keep_input and drop_input are true, we keep only the aggregated series
-			if rctx == nil {
-				rctx = getRelabelCtx()
-				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompb.TimeSeries)
-				tss = append(*v, tss...)
-			}
-			tss = dropUnaggregatedSeries(tss, matchIdxs.B)
 		}
-
 		matchIdxsPool.Put(matchIdxs)
 	}
 	if rwctx.deduplicator != nil {
@@ -1011,7 +953,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 	}
 
 	// Try pushing tss to remote storage
-	if rwctx.tryPushTimeSeriesInternal(tss) {
+	if rwctx.tryPushInternal(tss) {
 		return true
 	}
 
@@ -1025,10 +967,9 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 	return false
 }
 
-var matchIdxsPool slicesutil.BufferPool[uint32]
+var matchIdxsPool bytesutil.ByteBufferPool
 
-// dropAggregatedSeries drops matched series, also the unmatched if dropInput is true.
-func dropAggregatedSeries(src []prompb.TimeSeries, matchIdxs []uint32, dropInput bool) []prompb.TimeSeries {
+func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, dropInput bool) []prompbmarshal.TimeSeries {
 	dst := src[:0]
 	if !dropInput {
 		for i, match := range matchIdxs {
@@ -1043,22 +984,8 @@ func dropAggregatedSeries(src []prompb.TimeSeries, matchIdxs []uint32, dropInput
 	return dst
 }
 
-// dropUnaggregatedSeries drops unmatched series.
-func dropUnaggregatedSeries(src []prompb.TimeSeries, matchIdxs []uint32) []prompb.TimeSeries {
-	dst := src[:0]
-	for i, match := range matchIdxs {
-		if match == 0 {
-			continue
-		}
-		dst = append(dst, src[i])
-	}
-	tail := src[len(dst):]
-	clear(tail)
-	return dst
-}
-
-func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompb.TimeSeries) {
-	if rwctx.tryPushTimeSeriesInternal(tss) {
+func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSeries) {
+	if rwctx.tryPushInternal(tss) {
 		return
 	}
 	if !rwctx.fq.IsPersistentQueueDisabled() {
@@ -1069,21 +996,14 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompb.TimeSeries) {
 	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 }
 
-func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata) bool {
-	pss := rwctx.pss
-	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
-
-	return pss[idx].TryPushMetadata(mms)
-}
-
-func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
+func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
 	var rctx *relabelCtx
-	var v *[]prompb.TimeSeries
+	var v *[]prompbmarshal.TimeSeries
 	defer func() {
 		if rctx == nil {
 			return
 		}
-		*v = prompb.ResetTimeSeries(tss)
+		*v = prompbmarshal.ResetTimeSeries(tss)
 		tssPool.Put(v)
 		putRelabelCtx(rctx)
 	}()
@@ -1092,7 +1012,7 @@ func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries)
 		// Make a copy of tss before adding extra labels in order to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
-		v = tssPool.Get().(*[]prompb.TimeSeries)
+		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
 		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
@@ -1100,17 +1020,17 @@ func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries)
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
-	return pss[idx].TryPushTimeSeries(tss)
+	return pss[idx].TryPush(tss)
 }
 
 var tssPool = &sync.Pool{
 	New: func() any {
-		a := []prompb.TimeSeries{}
+		a := []prompbmarshal.TimeSeries{}
 		return &a
 	},
 }
 
-func getRowsCount(tss []prompb.TimeSeries) int {
+func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 	rowsCount := 0
 	for _, ts := range tss {
 		rowsCount += len(ts.Samples)

app/vmagent/remotewrite/remotewrite_test.go

@@ -10,9 +10,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
@@ -27,14 +25,14 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		// Distribute itemsCount hashes returned by getLabelsHash() across bucketsCount buckets.
 		itemsCount := 1_000 * bucketsCount
 		m := make([]int, bucketsCount)
-		var labels []prompb.Label
+		var labels []prompbmarshal.Label
 		for i := 0; i < itemsCount; i++ {
-			labels = append(labels[:0], prompb.Label{
+			labels = append(labels[:0], prompbmarshal.Label{
 				Name:  "__name__",
 				Value: fmt.Sprintf("some_name_%d", i),
 			})
 			for j := 0; j < 10; j++ {
-				labels = append(labels, prompb.Label{
+				labels = append(labels, prompbmarshal.Label{
 					Name:  fmt.Sprintf("label_%d", j),
 					Value: fmt.Sprintf("value_%d_%d", i, j),
 				})
@@ -59,8 +57,8 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 	f(10)
 }
 
-func TestRemoteWriteContext_TryPushTimeSeries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string, expectedRowsPushedAfterRelabel, expectedPushedSample int) {
+func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
+	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
 		t.Helper()
 		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
 		if err != nil {
@@ -73,16 +71,10 @@ func TestRemoteWriteContext_TryPushTimeSeries(t *testing.T) {
 		}
 		allRelabelConfigs.Store(rcs)
 
-		path := "fast-queue-write-test"
-		fs.MustRemoveDir(path)
-		fq := persistentqueue.MustOpenFastQueue(path, "test", 100, 0, false)
-		defer fs.MustRemoveDir(path)
-		defer fq.MustClose()
-
 		pss := make([]*pendingSeries, 1)
 		isVMProto := &atomic.Bool{}
 		isVMProto.Store(true)
-		pss[0] = newPendingSeries(fq, isVMProto, 0, 100)
+		pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
 		rwctx := &remoteWriteCtx{
 			idx:                    0,
 			streamAggrKeepInput:    keepInput,
@@ -91,14 +83,12 @@ func TestRemoteWriteContext_TryPushTimeSeries(t *testing.T) {
 			rowsPushedAfterRelabel: metrics.GetOrCreateCounter(`foo`),
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
-		defer metrics.UnregisterAllMetrics()
-
 		if dedupInterval > 0 {
 			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, enableWindows, dedupInterval, nil, "dedup-global")
 		}
 
 		if streamAggrConfig != "" {
-			pushNoop := func(_ []prompb.TimeSeries) {}
+			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
 			opts := streamaggr.Options{
 				EnableWindows: enableWindows,
 			}
@@ -112,29 +102,25 @@ func TestRemoteWriteContext_TryPushTimeSeries(t *testing.T) {
 
 		offsetMsecs := time.Now().UnixMilli()
 		inputTss := prometheus.MustParsePromMetrics(input, offsetMsecs)
-		expectedTss := make([]prompb.TimeSeries, len(inputTss))
+		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
 
-		// check inputTss is not modified after TryPushTimeSeries
+		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		if !rwctx.TryPushTimeSeries(inputTss, false) {
+		if !rwctx.TryPush(inputTss, false) {
 			t.Fatalf("cannot push samples to rwctx")
 		}
 
-		if int(rwctx.rowsPushedAfterRelabel.Get()) != expectedRowsPushedAfterRelabel {
-			t.Fatalf("unexpected number of rows after relabel; got %d; want %d", rwctx.rowsPushedAfterRelabel.Get(), expectedRowsPushedAfterRelabel)
-		}
-
-		if len(pss[0].wr.tss) != expectedPushedSample {
-			t.Fatalf("unexpected number of pushed samples; got %d; want %d", len(pss[0].wr.tss), expectedPushedSample)
-		}
-
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
 		}
 	}
 
-	// relabeling
-	f(``, `
+	f(`
+- interval: 1m
+  outputs: [sum_samples]
+- interval: 2m
+  outputs: [count_series]
+`, `
 - action: keep
   source_labels: [env]
   regex: "dev"
@@ -143,66 +129,53 @@ metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
 metric{env="bar"} 25
-`, 2, 2)
-
-	// relabeling + aggregation
-	f(`
-- match: '{env="dev"}'
-  interval: 1m
-  outputs: [sum_samples]
-`, `
-- action: keep
-  source_labels: [env]
-  regex: ".*"
-`, false, 0, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`, 4, 2)
-
-	// aggregation + keepInput
-	f(`
-- match: '{env="dev"}'
-  interval: 1m
-  outputs: [sum_samples]
-`, ``, false, 0, true, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`, 4, 4)
-
-	// aggregation + dropInput
-	f(`
-- match: '{env="dev"}'
-  interval: 1m
-  outputs: [sum_samples]
-`, ``, false, 0, false, true, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`, 4, 0)
-
-	// aggregation + keepInput + dropInput
-	f(`
-- match: '{env="dev"}'
-  interval: 1m
-  outputs: [sum_samples]
-`, ``, false, 0, true, true, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="bar"} 25
-`, 3, 1)
-
-	// aggregation + deduplication
+`)
 	f(``, ``, true, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="foo"} 20
 metric{env="dev"} 15
 metric{env="foo"} 25
-`, 4, 0)
+`)
+	f(``, `
+- action: keep
+  source_labels: [env]
+  regex: "dev"
+`, true, time.Hour, false, false, `
+metric{env="dev"} 10
+metric{env="bar"} 20
+metric{env="dev"} 15
+metric{env="bar"} 25
+`)
+	f(``, `
+- action: keep
+  source_labels: [env]
+  regex: "dev"
+`, true, time.Hour, true, false, `
+metric{env="test"} 10
+metric{env="dev"} 20
+metric{env="foo"} 15
+metric{env="dev"} 25
+`)
+	f(``, `
+- action: keep
+  source_labels: [env]
+  regex: "dev"
+`, true, time.Hour, false, true, `
+metric{env="foo"} 10
+metric{env="dev"} 20
+metric{env="foo"} 15
+metric{env="dev"} 25
+`)
+	f(``, `
+- action: keep
+  source_labels: [env]
+  regex: "dev"
+`, true, time.Hour, true, true, `
+metric{env="dev"} 10
+metric{env="test"} 20
+metric{env="dev"} 15
+metric{env="bar"} 25
+`)
 }
 
 func TestShardAmountRemoteWriteCtx(t *testing.T) {
@@ -247,16 +220,16 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
 
 		seriesCount := 100000
 		// build 1000000 series
-		tssBlock := make([]prompb.TimeSeries, 0, seriesCount)
+		tssBlock := make([]prompbmarshal.TimeSeries, 0, seriesCount)
 		for i := 0; i < seriesCount; i++ {
-			tssBlock = append(tssBlock, prompb.TimeSeries{
-				Labels: []prompb.Label{
+			tssBlock = append(tssBlock, prompbmarshal.TimeSeries{
+				Labels: []prompbmarshal.Label{
 					{
 						Name:  "label",
 						Value: strconv.Itoa(i),
 					},
 				},
-				Samples: []prompb.Sample{
+				Samples: []prompbmarshal.Sample{
 					{
 						Timestamp: 0,
 						Value:     0,
@@ -285,7 +258,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
 		for i, nodeIdx := range healthyIdx {
 			for _, ts := range shards[i] {
 				// add it to node[nodeIdx]'s active time series
-				activeTimeSeriesByNodes[nodeIdx][prompb.LabelsToString(ts.Labels)] = struct{}{}
+				activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
 			}
 		}
 
@@ -308,7 +281,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
 		for i, nodeIdx := range healthyIdx {
 			for _, ts := range shards[i] {
 				// add it to node[nodeIdx]'s active time series
-				activeTimeSeriesByNodes[nodeIdx][prompb.LabelsToString(ts.Labels)] = struct{}{}
+				activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
 			}
 		}
 

app/vmagent/remotewrite/streamaggr.go

@@ -8,7 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -18,12 +18,12 @@ var (
 	streamAggrGlobalConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
 		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
-	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep input samples that match any rule in "+
-		"-streamAggr.config. By default, matched raw samples are aggregated and dropped, while unmatched samples "+
-		"are written to the remote storage. See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop input samples that not matching any rule in "+
-		"-streamAggr.config. By default, only matched raw samples are dropped, while unmatched samples "+
-		"are written to the remote storage. See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation "+
+		"with -streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
 		"aggregator before optional aggregation with -streamAggr.config . "+
 		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
@@ -43,11 +43,11 @@ var (
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
 		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
-	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop input samples that not matching any rule in "+
-		"the corresponding -remoteWrite.streamAggr.config. By default, only matched raw samples are dropped, while unmatched samples "+
+	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep input samples that match any rule in "+
-		"the corresponding -remoteWrite.streamAggr.config. By default, matched raw samples are aggregated and dropped, while unmatched samples "+
+	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
 		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
@@ -80,7 +80,7 @@ func CheckStreamAggrConfigs() error {
 		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
 	}
 
-	pushNoop := func(_ []prompb.TimeSeries) {}
+	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
 	for idx := range *streamAggrConfig {
 		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
 		if err != nil {
@@ -141,7 +141,7 @@ func initStreamAggrConfigGlobal() {
 	}
 	dedupInterval := *streamAggrGlobalDedupInterval
 	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushTimeSeriesToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 	}
 }
 
@@ -216,7 +216,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 		EnableWindows:        *streamAggrGlobalEnableWindows,
 	}
 
-	sas, err := streamaggr.LoadFromFile(path, pushTimeSeriesToRemoteStoragesTrackDropped, opts, "global")
+	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
 	if err != nil {
 		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
 	}

app/vmagent/vmimport/request_handler.go

@@ -8,7 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
@@ -36,7 +36,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -50,7 +50,7 @@ func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompb.Label)
 		labelsLen := len(labels)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  bytesutil.ToUnsafeString(tag.Key),
 				Value: bytesutil.ToUnsafeString(tag.Value),
 			})
@@ -63,12 +63,12 @@ func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompb.Label)
 		}
 		samplesLen := len(samples)
 		for j, value := range values {
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Value:     value,
 				Timestamp: timestamps[j],
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})

app/vmagent/zabbixconnector/request_handler.go

@@ -1,80 +0,0 @@
-package zabbixconnector
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="zabbixconnector"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="zabbixconnector"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="zabbixconnector"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
-		return insertRows(at, rows, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := len(rows)
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-
-		labelsLen := len(labels)
-		for j := range r.Tags {
-			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
-				Name:  bytesutil.ToUnsafeString(tag.Key),
-				Value: bytesutil.ToUnsafeString(tag.Value),
-			})
-		}
-		labels = append(labels, extraLabels...)
-
-		samplesLen := len(samples)
-		samples = append(samples, prompb.Sample{
-			Value:     r.Value,
-			Timestamp: r.Timestamp,
-		})
-
-		tssDst = append(tssDst, prompb.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmalert-tool/Makefile

@@ -27,9 +27,6 @@ vmalert-tool-linux-ppc64le-prod:
 vmalert-tool-linux-386-prod:
 	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-386
 
-vmalert-tool-linux-s390x-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-s390x
-
 vmalert-tool-darwin-amd64-prod:
 	APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-amd64
 

app/vmalert-tool/unittest/recording.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 	"github.com/VictoriaMetrics/metricsql"
 )
@@ -49,7 +49,7 @@ Outer:
 		}
 		var expSamples []parsedSample
 		for _, s := range mt.ExpSamples {
-			expLb := []prompb.Label{}
+			expLb := []prompbmarshal.Label{}
 			if s.Labels != "" {
 				metricsqlExpr, err := metricsql.Parse(s.Labels)
 				if err != nil {
@@ -65,7 +65,7 @@ Outer:
 				}
 				if len(metricsqlMetricExpr.LabelFilterss) > 0 {
 					for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
-						expLb = append(expLb, prompb.Label{
+						expLb = append(expLb, prompbmarshal.Label{
 							Name:  l.Label,
 							Value: l.Value,
 						})

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -15,10 +15,6 @@ groups:
       - alert: SameAlertNameWithDifferentGroup
         expr: absent(test)
         for: 1m
-      - alert: AlertWithTemplate
-        expr: test
-        annotations:
-          queryAnno: '{{ query "foo" | first | value }}'
 
   - name: group2
     rules:

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -10,9 +10,7 @@ tests:
     input_series:
       - series: "test"
         values: "_x5 1x5 _ stale"
-      - series: "foo"
-        values: "1x20"
-  
+
     alert_rule_test:
       - eval_time: 1m
         groupname: group1
@@ -34,14 +32,6 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts: []
-      - eval_time: 6m
-        groupname: group1
-        alertname: AlertWithTemplate
-        exp_alerts:
-          - exp_labels:
-              cluster: prod
-            exp_annotations:
-              queryAnno: '1'
 
     metricsql_expr_test:
       - expr: test
@@ -60,8 +50,6 @@ tests:
         values: "0+0x1440"
       - series: "test"
         values: "0+1x1440"
-      - series: "foo"
-        values: "1x20"
 
     metricsql_expr_test:
       - expr: count(ALERTS) by (alertgroup, alertname, alertstate)
@@ -71,8 +59,6 @@ tests:
             value: 1
           - labels: '{alertgroup="group1", alertname="InstanceDown", alertstate="pending"}'
             value: 1
-          - labels: '{alertgroup="group1", alertname="AlertWithTemplate", alertstate="firing"}'
-            value: 1
       - expr: t1
         eval_time: 4m
         exp_samples:

app/vmalert-tool/unittest/unittest.go

@@ -19,6 +19,7 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -34,7 +35,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -85,8 +85,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 		defer server.Close()
 	} else {
 		httpListenAddr = httpListenPort
-
-		ln, err := net.Listen(netutil.GetTCPNetwork(), fmt.Sprintf(":%s", httpListenPort))
+		ln, err := net.Listen("tcp", fmt.Sprintf(":%s", httpListenPort))
 		if err != nil {
 			logger.Fatalf("cannot listen on port %s: %v", httpListenPort, err)
 		}
@@ -108,12 +107,12 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	vminsert.Init()
 	vmselect.Init()
 	// storagePath will be created again when closing vmselect, so remove it again.
-	defer fs.MustRemoveDir(storagePath)
+	defer fs.MustRemoveAll(storagePath)
 	defer vminsert.Stop()
 	defer vmselect.Stop()
 	disableAlertgroupLabel = disableGroupLabel
 
-	testfiles, err := vmalertconfig.ReadFromFS(files)
+	testfiles, err := config.ReadFromFS(files)
 	if err != nil {
 		logger.Fatalf("failed to load test files %q: %v", files, err)
 	}
@@ -132,7 +131,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 		}
 		labels[s[:n]] = s[n+1:]
 	}
-	err = notifier.Init(labels, externalURL)
+	_, err = notifier.Init(nil, labels, externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init notifier: %v", err)
 	}
@@ -305,7 +304,7 @@ checkCheck:
 func tearDown() {
 	vmstorage.Stop()
 	metrics.UnregisterAllMetrics()
-	fs.MustRemoveDir(storagePath)
+	fs.MustRemoveAll(storagePath)
 }
 
 func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group, externalLabels map[string]string) (checkErrs []error) {
@@ -368,7 +367,6 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 			mergedExternalLabels[k] = v
 		}
 		ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
-		ng.Init()
 		groups = append(groups, ng)
 	}
 
@@ -379,7 +377,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 			if len(g.Rules) == 0 {
 				continue
 			}
-			errs := g.ExecOnce(context.Background(), rw, ts)
+			errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts)
 			for err := range errs {
 				if err != nil {
 					checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name,

app/vmalert/Makefile

@@ -27,9 +27,6 @@ vmalert-linux-ppc64le-prod:
 vmalert-linux-386-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-linux-386
 
-vmalert-linux-s390x-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-linux-s390x
-
 vmalert-darwin-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
 

app/vmalert/config/config.go

@@ -31,7 +31,7 @@ type Group struct {
 	// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
 	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155
 	EvalDelay   *promutil.Duration `yaml:"eval_delay,omitempty"`
-	Limit       *int               `yaml:"limit,omitempty"`
+	Limit       int                `yaml:"limit,omitempty"`
 	Rules       []Rule             `yaml:"rules"`
 	Concurrency int                `yaml:"concurrency"`
 	// Labels is a set of label value pairs, that will be added to every rule.
@@ -91,8 +91,8 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 	if g.EvalOffset != nil && g.EvalDelay != nil {
 		return fmt.Errorf("eval_offset cannot be used with eval_delay")
 	}
-	if g.Limit != nil && *g.Limit < 0 {
-		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", *g.Limit)
+	if g.Limit < 0 {
+		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", g.Limit)
 	}
 	if g.Concurrency < 0 {
 		return fmt.Errorf("invalid concurrency %d, shouldn't be less than 0", g.Concurrency)
@@ -295,7 +295,10 @@ func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpress
 }
 
 func parseConfig(data []byte) ([]Group, error) {
-	data = envtemplate.ReplaceBytes(data)
+	data, err := envtemplate.ReplaceBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
+	}
 
 	var result []Group
 	type cfgFile struct {
@@ -307,13 +310,13 @@ func parseConfig(data []byte) ([]Group, error) {
 	decoder := yaml.NewDecoder(bytes.NewReader(data))
 	for {
 		var cf cfgFile
-		if err := decoder.Decode(&cf); err != nil {
+		if err = decoder.Decode(&cf); err != nil {
 			if err == io.EOF { // EOF indicates no more documents to read
 				break
 			}
 			return nil, err
 		}
-		if err := checkOverflow(cf.XXX, "config"); err != nil {
+		if err = checkOverflow(cf.XXX, "config"); err != nil {
 			return nil, err
 		}
 		result = append(result, cf.Groups...)

app/vmalert/config/config_test.go

@@ -116,7 +116,7 @@ func TestParse_Failure(t *testing.T) {
 
 	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
 	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "invalid annotations")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
 	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
@@ -181,10 +181,9 @@ func TestGroupValidate_Failure(t *testing.T) {
 		EvalOffset: promutil.NewDuration(2 * time.Minute),
 	}, false, "eval_offset should be smaller than interval")
 
-	limit := -1
 	f(&Group{
 		Name:  "wrong limit",
-		Limit: &limit,
+		Limit: -1,
 	}, false, "invalid limit")
 
 	f(&Group{
@@ -343,6 +342,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 			},
 		},
 	}, true, "bad prometheus expr")
+
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/datasource/client.go

@@ -173,26 +173,22 @@ func (c *Client) Query(ctx context.Context, query string, ts time.Time) (Result,
 			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
 		}
 	}
-	defer func() { _ = resp.Body.Close() }()
 
 	// Process the received response.
-	var parseFn func(resp *http.Response) (Result, error)
+	var parseFn func(req *http.Request, resp *http.Response) (Result, error)
 	switch c.dataSourceType {
 	case datasourcePrometheus:
-		parseFn = parsePrometheusInstantResponse
+		parseFn = parsePrometheusResponse
 	case datasourceGraphite:
 		parseFn = parseGraphiteResponse
 	case datasourceVLogs:
-		parseFn = parseVLogsInstantResponse
+		parseFn = parseVLogsResponse
 	default:
 		logger.Panicf("BUG: unsupported datasource type %q to parse query response", c.dataSourceType)
 	}
-
-	result, err := parseFn(resp)
-	if err != nil {
-		return Result{}, nil, fmt.Errorf("error parsing response from %q: %w", req.URL.Redacted(), err)
-	}
-	return result, req, nil
+	result, err := parseFn(req, resp)
+	_ = resp.Body.Close()
+	return result, req, err
 }
 
 // QueryRange executes the given query on the given time range.
@@ -233,23 +229,19 @@ func (c *Client) QueryRange(ctx context.Context, query string, start, end time.T
 			return res, fmt.Errorf("second attempt: %w", err)
 		}
 	}
-	defer func() { _ = resp.Body.Close() }()
 
 	// Process the received response.
-	var parseFn func(resp *http.Response) (Result, error)
+	var parseFn func(req *http.Request, resp *http.Response) (Result, error)
 	switch c.dataSourceType {
 	case datasourcePrometheus:
-		parseFn = parsePrometheusRangeResponse
+		parseFn = parsePrometheusResponse
 	case datasourceVLogs:
-		parseFn = parseVLogsRangeResponse
+		parseFn = parseVLogsResponse
 	default:
 		logger.Panicf("BUG: unsupported datasource type %q to parse query range response", c.dataSourceType)
 	}
-
-	res, err = parseFn(resp)
-	if err != nil {
-		return Result{}, fmt.Errorf("error parsing response from %q: %w", req.URL.Redacted(), err)
-	}
+	res, err = parseFn(req, resp)
+	_ = resp.Body.Close()
 	return res, err
 }
 

app/vmalert/datasource/client_graphite.go

@@ -33,10 +33,10 @@ func (r graphiteResponse) metrics() []Metric {
 	return ms
 }
 
-func parseGraphiteResponse(resp *http.Response) (Result, error) {
+func parseGraphiteResponse(req *http.Request, resp *http.Response) (Result, error) {
 	r := &graphiteResponse{}
 	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return Result{}, fmt.Errorf("error parsing graphite metrics: %w", err)
+		return Result{}, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL.Redacted(), err)
 	}
 	return Result{Data: r.metrics()}, nil
 }

app/vmalert/datasource/client_prom.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 
 	"github.com/valyala/fastjson"
 )
@@ -85,14 +85,14 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 		labels := metric.GetObject()
 
 		r := &pi.ms[i]
-		r.Labels = make([]prompb.Label, 0, labels.Len())
+		r.Labels = make([]prompbmarshal.Label, 0, labels.Len())
 		labels.Visit(func(key []byte, v *fastjson.Value) {
 			lv, errLocal := v.StringBytes()
 			if errLocal != nil {
 				err = fmt.Errorf("error when parsing label value %q: %s", v, errLocal)
 				return
 			}
-			r.Labels = append(r.Labels, prompb.Label{
+			r.Labels = append(r.Labels, prompbmarshal.Label{
 				Name:  string(key),
 				Value: string(lv),
 			})
@@ -172,26 +172,17 @@ const (
 	rtVector, rtMatrix, rScalar = "vector", "matrix", "scalar"
 )
 
-func parsePromResponse(resp *http.Response) (*promResponse, error) {
+func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result, err error) {
 	r := &promResponse{}
-	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return nil, fmt.Errorf("failed to decode response: %w", err)
+	if err = json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return res, fmt.Errorf("error parsing response from %s: %w", req.URL.Redacted(), err)
 	}
 	if r.Status == statusError {
-		return nil, fmt.Errorf("response error %q: %s", r.ErrorType, r.Error)
+		return res, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error)
 	}
 	if r.Status != statusSuccess {
-		return nil, fmt.Errorf("unknown response status %q", r.Status)
+		return res, fmt.Errorf("unknown status: %s, Expected success or error", r.Status)
 	}
-	return r, nil
-}
-
-func parsePrometheusInstantResponse(resp *http.Response) (res Result, err error) {
-	r, err := parsePromResponse(resp)
-	if err != nil {
-		return res, fmt.Errorf("failed to parse response: %w", err)
-	}
-
 	var parseFn func() ([]Metric, error)
 	switch r.Data.ResultType {
 	case rtVector:
@@ -200,6 +191,12 @@ func parsePrometheusInstantResponse(resp *http.Response) (res Result, err error)
 			return res, fmt.Errorf("unmarshal err %w; \n %#v", err, string(r.Data.Result))
 		}
 		parseFn = pi.metrics
+	case rtMatrix:
+		var pr promRange
+		if err := json.Unmarshal(r.Data.Result, &pr.Result); err != nil {
+			return res, err
+		}
+		parseFn = pr.metrics
 	case rScalar:
 		var ps promScalar
 		if err := json.Unmarshal(r.Data.Result, &ps); err != nil {
@@ -209,6 +206,7 @@ func parsePrometheusInstantResponse(resp *http.Response) (res Result, err error)
 	default:
 		return res, fmt.Errorf("unknown result type %q", r.Data.ResultType)
 	}
+
 	ms, err := parseFn()
 	if err != nil {
 		return res, err
@@ -224,34 +222,6 @@ func parsePrometheusInstantResponse(resp *http.Response) (res Result, err error)
 	return res, nil
 }
 
-func parsePrometheusRangeResponse(resp *http.Response) (res Result, err error) {
-	r, err := parsePromResponse(resp)
-	if err != nil {
-		return res, fmt.Errorf("failed to parse response: %w", err)
-	}
-	if r.Data.ResultType != rtMatrix {
-		return res, fmt.Errorf("unexpected result type %q; expected result type %q", r.Data.ResultType, rtMatrix)
-	}
-
-	var pr promRange
-	if err := json.Unmarshal(r.Data.Result, &pr.Result); err != nil {
-		return res, err
-	}
-	ms, err := pr.metrics()
-	if err != nil {
-		return res, err
-	}
-	res = Result{Data: ms, IsPartial: r.IsPartial}
-	if r.Stats.SeriesFetched != nil {
-		intV, err := strconv.Atoi(*r.Stats.SeriesFetched)
-		if err != nil {
-			return res, fmt.Errorf("failed to convert stats.seriesFetched to int: %w", err)
-		}
-		res.SeriesFetched = &intV
-	}
-	return res, nil
-}
-
 func (c *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	if c.appendTypePrefix {
 		r.URL.Path += "/prometheus"

app/vmalert/datasource/client_test.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var (
@@ -65,23 +65,21 @@ func TestVMInstantQuery(t *testing.T) {
 		case 3:
 			w.Write([]byte(`{"status":"unknown"}`))
 		case 4:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"vector"}}`))
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
 		case 5:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"vm_rows"},"values":[[1583786142,"13763"]]}]}}`))
-		case 6:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
-		case 7:
+		case 6:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
-		case 8:
+		case 7:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]},"stats":{"seriesFetched": "42"}}`))
-		case 9:
+		case 8:
 			w.Write([]byte(`{"status":"success", "isPartial":true, "data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		}
 	})
 	mux.HandleFunc("/render", func(w http.ResponseWriter, _ *http.Request) {
 		c++
 		switch c {
-		case 10:
+		case 9:
 			w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
 		}
 	})
@@ -104,9 +102,9 @@ func TestVMInstantQuery(t *testing.T) {
 			t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
-		case 11:
+		case 10:
 			w.Write([]byte("[]"))
-		case 12:
+		case 11:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"total","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"total","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
 		}
 	})
@@ -125,7 +123,6 @@ func TestVMInstantQuery(t *testing.T) {
 	ts := time.Now()
 
 	expErr := func(query, err string) {
-		t.Helper()
 		_, _, gotErr := pq.Query(ctx, query, ts)
 		if gotErr == nil {
 			t.Fatalf("expected %q got nil", err)
@@ -138,11 +135,10 @@ func TestVMInstantQuery(t *testing.T) {
 	expErr(vmQuery, "500")                          // 0
 	expErr(vmQuery, "error parsing response")       // 1
 	expErr(vmQuery, "response error")               // 2
-	expErr(vmQuery, "unknown response status")      // 3
+	expErr(vmQuery, "unknown status")               // 3
 	expErr(vmQuery, "unexpected end of JSON input") // 4
-	expErr(vmQuery, "unknown result type")          // 5
 
-	res, _, err := pq.Query(ctx, vmQuery, ts) // 6 - vector
+	res, _, err := pq.Query(ctx, vmQuery, ts) // 5 - vector
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -151,19 +147,19 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected := []Metric{
 		{
-			Labels:     []prompb.Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []prompb.Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
 	}
 	metricsEqual(t, res.Data, expected)
 
-	res, req, err := pq.Query(ctx, vmQuery, ts) // 7 - scalar
+	res, req, err := pq.Query(ctx, vmQuery, ts) // 6 - scalar
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -188,7 +184,7 @@ func TestVMInstantQuery(t *testing.T) {
 			res.SeriesFetched)
 	}
 
-	res, _, err = pq.Query(ctx, vmQuery, ts) // 8 - scalar with stats
+	res, _, err = pq.Query(ctx, vmQuery, ts) // 7 - scalar with stats
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -209,7 +205,7 @@ func TestVMInstantQuery(t *testing.T) {
 			*res.SeriesFetched)
 	}
 
-	res, _, err = pq.Query(ctx, vmQuery, ts) // 9
+	res, _, err = pq.Query(ctx, vmQuery, ts) // 8
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -220,7 +216,7 @@ func TestVMInstantQuery(t *testing.T) {
 	// test graphite
 	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
 
-	res, _, err = gq.Query(ctx, queryRender, ts) // 10 - graphite
+	res, _, err = gq.Query(ctx, queryRender, ts) // 9 - graphite
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -229,7 +225,7 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	exp := []Metric{
 		{
-			Labels:     []prompb.Label{{Value: "constantLine(10)", Name: "name"}},
+			Labels:     []prompbmarshal.Label{{Value: "constantLine(10)", Name: "name"}},
 			Timestamps: []int64{1611758403},
 			Values:     []float64{10},
 		},
@@ -240,9 +236,9 @@ func TestVMInstantQuery(t *testing.T) {
 	vlogs := datasourceVLogs
 	pq = s.BuildWithParams(QuerierParams{DataSourceType: string(vlogs), EvaluationInterval: 15 * time.Second})
 
-	expErr(vlogsQuery, "error parsing response") // 11
+	expErr(vlogsQuery, "error parsing response") // 10
 
-	res, _, err = pq.Query(ctx, vlogsQuery, ts) // 12
+	res, _, err = pq.Query(ctx, vlogsQuery, ts) // 11
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -251,12 +247,12 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected = []Metric{
 		{
-			Labels:     []prompb.Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []prompb.Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
@@ -394,8 +390,6 @@ func TestVMRangeQuery(t *testing.T) {
 		switch c {
 		case 0:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"vm_rows"},"values":[[1583786142,"13763"]]}]}}`))
-		case 1:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[1583786142, "1"]}}`))
 		}
 	})
 	mux.HandleFunc("/select/logsql/stats_query_range", func(w http.ResponseWriter, r *http.Request) {
@@ -428,7 +422,7 @@ func TestVMRangeQuery(t *testing.T) {
 			t.Fatalf("expected 'step' query param to be 60s; got %q instead", step)
 		}
 		switch c {
-		case 2:
+		case 1:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"total"},"values":[[1583786142,"10"]]}]}}`))
 		}
 	})
@@ -452,16 +446,16 @@ func TestVMRangeQuery(t *testing.T) {
 
 	start, end := time.Now().Add(-time.Minute), time.Now()
 
-	res, err := pq.QueryRange(ctx, vmQuery, start, end) // case 0
+	res, err := pq.QueryRange(ctx, vmQuery, start, end)
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
 	m := res.Data
 	if len(m) != 1 {
-		t.Fatalf("expected 1 metric got %d in %+v", len(m), m)
+		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected := Metric{
-		Labels:     []prompb.Label{{Value: "vm_rows", Name: "__name__"}},
+		Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{13763},
 	}
@@ -469,9 +463,6 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
 
-	_, err = pq.QueryRange(ctx, vmQuery, start, end) // case 1
-	expectError(t, err, "unexpected result type")
-
 	// test unsupported graphite
 	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
 
@@ -495,7 +486,7 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected = Metric{
-		Labels:     []prompb.Label{{Value: "total", Name: "stats_result"}},
+		Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{10},
 	}

app/vmalert/datasource/client_vlogs.go

@@ -40,28 +40,8 @@ func (c *Client) setVLogsRangeReqParams(r *http.Request, query string, start, en
 	c.setReqParams(r, query)
 }
 
-func parseVLogsInstantResponse(resp *http.Response) (res Result, err error) {
-	res, err = parsePrometheusInstantResponse(resp)
-	if err != nil {
-		return Result{}, err
-	}
-	for i := range res.Data {
-		m := &res.Data[i]
-		for j := range m.Labels {
-			// reserve the stats func result name with a new label `stats_result` instead of dropping it,
-			// since there could be multiple stats results in a single query, for instance:
-			// 	_time:5m | stats quantile(0.5, request_duration_seconds) p50, quantile(0.9, request_duration_seconds) p90
-			if m.Labels[j].Name == "__name__" {
-				m.Labels[j].Name = "stats_result"
-				break
-			}
-		}
-	}
-	return
-}
-
-func parseVLogsRangeResponse(resp *http.Response) (res Result, err error) {
-	res, err = parsePrometheusRangeResponse(resp)
+func parseVLogsResponse(req *http.Request, resp *http.Response) (res Result, err error) {
+	res, err = parsePrometheusResponse(req, resp)
 	if err != nil {
 		return Result{}, err
 	}

app/vmalert/datasource/datasource.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Querier interface wraps Query and QueryRange methods
@@ -60,7 +60,7 @@ type QuerierParams struct {
 
 // Metric is the basic entity which should be return by datasource
 type Metric struct {
-	Labels     []prompb.Label
+	Labels     []prompbmarshal.Label
 	Timestamps []int64
 	Values     []float64
 }
@@ -79,7 +79,7 @@ func (m *Metric) SetLabel(key, value string) {
 
 // AddLabel appends the given label to the label set
 func (m *Metric) AddLabel(key, value string) {
-	m.Labels = append(m.Labels, prompb.Label{Name: key, Value: value})
+	m.Labels = append(m.Labels, prompbmarshal.Label{Name: key, Value: value})
 }
 
 // DelLabel deletes the given label from the label set
@@ -103,7 +103,7 @@ func (m *Metric) Label(key string) string {
 }
 
 // Labels is collection of Label
-type Labels []prompb.Label
+type Labels []prompbmarshal.Label
 
 func (ls Labels) Len() int           { return len(ls) }
 func (ls Labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
@@ -132,7 +132,10 @@ func (ls Labels) String() string {
 // a=[]Label{{Name: "a", Value: "2"}},b=[]Label{{Name: "a", Value: "1"}}, return 1
 // a=[]Label{{Name: "a", Value: "1"}},b=[]Label{{Name: "a", Value: "1"}}, return 0
 func LabelCompare(a, b Labels) int {
-	l := min(len(b), len(a))
+	l := len(a)
+	if len(b) < l {
+		l = len(b)
+	}
 
 	for i := 0; i < l; i++ {
 		if a[i].Name != b[i].Name {
@@ -155,7 +158,7 @@ func LabelCompare(a, b Labels) int {
 // ConvertToLabels convert map to Labels
 func ConvertToLabels(m map[string]string) (labelset Labels) {
 	for k, v := range m {
-		labelset = append(labelset, prompb.Label{
+		labelset = append(labelset, prompbmarshal.Label{
 			Name:  k,
 			Value: v,
 		})

app/vmalert/datasource/vm_prom_api_test.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestPromInstant_UnmarshalPositive(t *testing.T) {
@@ -23,7 +23,7 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 
 	f(`[{"metric":{"__name__":"up"},"value":[1583780000,"42"]}]`, []Metric{
 		{
-			Labels:     []prompb.Label{{Name: "__name__", Value: "up"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
@@ -33,17 +33,17 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 {"metric":{"__name__":"foo"},"value":[1583780001,"7"]},
 {"metric":{"__name__":"baz", "instance":"bar"},"value":[1583780002,"8"]}]`, []Metric{
 		{
-			Labels:     []prompb.Label{{Name: "__name__", Value: "up"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
 		{
-			Labels:     []prompb.Label{{Name: "__name__", Value: "foo"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "foo"}},
 			Timestamps: []int64{1583780001},
 			Values:     []float64{7},
 		},
 		{
-			Labels:     []prompb.Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
 			Timestamps: []int64{1583780002},
 			Values:     []float64{8},
 		},

app/vmalert/main.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -76,13 +77,14 @@ absolute path to all .tpl files in root.
 		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
-		"In case of conflicts, original labels are kept with prefix 'exported_'.")
+		"In case of conflicts, original labels are kept with prefix `exported_`.")
 
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
 var (
-	extURL *url.URL
+	alertURLGeneratorFn notifier.AlertURLGenerator
+	extURL              *url.URL
 )
 
 func main() {
@@ -119,7 +121,7 @@ func main() {
 		return
 	}
 
-	err = notifier.InitAlertURLGeneratorFn(extURL, *externalAlertSource, *validateTemplates)
+	alertURLGeneratorFn, err = getAlertURLGenerator(extURL, *externalAlertSource, *validateTemplates)
 	if err != nil {
 		logger.Fatalf("failed to init `external.alert.source`: %s", err)
 	}
@@ -226,13 +228,14 @@ func newManager(ctx context.Context) (*manager, error) {
 		labels[s[:n]] = s[n+1:]
 	}
 
-	err = notifier.Init(labels, *externalURL)
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
 	manager := &manager{
 		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: q,
+		notifiers:      nts,
 		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)
@@ -289,6 +292,35 @@ func getHostnameAsExternalURL(addr string, isSecure bool) (*url.URL, error) {
 	return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port))
 }
 
+func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) {
+	if externalAlertSource == "" {
+		return func(a notifier.Alert) string {
+			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
+			return fmt.Sprintf("%s/vmalert/alert?%s=%s&%s=%s", externalURL, paramGroupID, gID, paramAlertID, aID)
+		}, nil
+	}
+	if validateTemplate {
+		if err := notifier.ValidateTemplates(map[string]string{
+			"tpl": externalAlertSource,
+		}); err != nil {
+			return nil, fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
+		}
+	}
+	m := map[string]string{
+		"tpl": externalAlertSource,
+	}
+	return func(alert notifier.Alert) string {
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
+		}
+		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
+		if err != nil {
+			logger.Errorf("cannot template alert source: %s", err)
+		}
+		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
+	}, nil
+}
+
 func usage() {
 	const s = `
 vmalert processes alerts and recording rules.

app/vmalert/main_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
@@ -49,6 +48,30 @@ func TestGetExternalURL(t *testing.T) {
 	}
 }
 
+func TestGetAlertURLGenerator(t *testing.T) {
+	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	fn, err := getAlertURLGenerator(u, "", false)
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
+	if exp != fn(testAlert) {
+		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
+	}
+	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
+	if err == nil {
+		t.Fatalf("expected template validation error got nil")
+	}
+	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
+		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
+	}
+}
+
 func TestConfigReload(t *testing.T) {
 	originalRulePath := *rulePath
 	originalExternalURL := extURL
@@ -85,8 +108,8 @@ groups:
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer fs.MustRemovePath(f.Name())
-	writeToFile(f.Name(), rules1)
+	defer func() { _ = os.Remove(f.Name()) }()
+	writeToFile(t, f.Name(), rules1)
 
 	*configCheckInterval = 200 * time.Millisecond
 	*rulePath = []string{f.Name()}
@@ -96,10 +119,9 @@ groups:
 		querierBuilder: &datasource.FakeQuerier{},
 		groups:         make(map[uint64]*rule.Group),
 		labels:         map[string]string{},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 		rw:             &remotewrite.Client{},
 	}
-	_, cleanup := notifier.InitFakeNotifier()
-	defer cleanup()
 
 	syncCh := make(chan struct{})
 	sighupCh := procutil.NewSighupChan()
@@ -142,7 +164,7 @@ groups:
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
 	}
 
-	writeToFile(f.Name(), rules2)
+	writeToFile(t, f.Name(), rules2)
 	time.Sleep(*configCheckInterval * 2)
 	checkCfg(nil)
 	groupsLen = lenLocked(m)
@@ -150,7 +172,7 @@ groups:
 		t.Fatalf("expected to have exactly 2 groups loaded; got %d", groupsLen)
 	}
 
-	writeToFile(f.Name(), rules1)
+	writeToFile(t, f.Name(), rules1)
 	procutil.SelfSIGHUP()
 	time.Sleep(*configCheckInterval / 2)
 	checkCfg(nil)
@@ -159,7 +181,7 @@ groups:
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
 	}
 
-	writeToFile(f.Name(), `corrupted`)
+	writeToFile(t, f.Name(), `corrupted`)
 	procutil.SelfSIGHUP()
 	time.Sleep(*configCheckInterval / 2)
 	checkCfg(fmt.Errorf("config error"))
@@ -172,6 +194,10 @@ groups:
 	<-syncCh
 }
 
-func writeToFile(file, b string) {
-	fs.MustWriteSync(file, []byte(b))
+func writeToFile(t *testing.T, file, b string) {
+	t.Helper()
+	err := os.WriteFile(file, []byte(b), 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
 }

app/vmalert/manager.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"strconv"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -17,6 +16,7 @@ import (
 // manager controls group states
 type manager struct {
 	querierBuilder datasource.QuerierBuilder
+	notifiers      func() []notifier.Notifier
 
 	rw remotewrite.RWClient
 	// remote read builder.
@@ -29,8 +29,25 @@ type manager struct {
 	groups   map[uint64]*rule.Group
 }
 
-// groupAPI generates apiGroup object from group by its ID(hash)
-func (m *manager) groupAPI(gID uint64) (*rule.ApiGroup, error) {
+// ruleAPI generates apiRule object from alert by its ID(hash)
+func (m *manager) ruleAPI(gID, rID uint64) (apiRule, error) {
+	m.groupsMu.RLock()
+	defer m.groupsMu.RUnlock()
+
+	g, ok := m.groups[gID]
+	if !ok {
+		return apiRule{}, fmt.Errorf("can't find group with id %d", gID)
+	}
+	for _, rule := range g.Rules {
+		if rule.ID() == rID {
+			return ruleToAPI(rule), nil
+		}
+	}
+	return apiRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
+}
+
+// alertAPI generates apiAlert object from alert by its ID(hash)
+func (m *manager) alertAPI(gID, aID uint64) (*apiAlert, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
@@ -38,47 +55,13 @@ func (m *manager) groupAPI(gID uint64) (*rule.ApiGroup, error) {
 	if !ok {
 		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
-	return g.ToAPI(), nil
-}
-
-// ruleAPI generates apiRule object from alert by its ID(hash)
-func (m *manager) ruleAPI(gID, rID uint64) (rule.ApiRule, error) {
-	m.groupsMu.RLock()
-	defer m.groupsMu.RUnlock()
-
-	group, ok := m.groups[gID]
-	if !ok {
-		return rule.ApiRule{}, fmt.Errorf("can't find group with id %d", gID)
-	}
-	g := group.ToAPI()
-	ruleID := strconv.FormatUint(rID, 10)
 	for _, r := range g.Rules {
-		if r.ID == ruleID {
-			return r, nil
-		}
-	}
-	return rule.ApiRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
-}
-
-// alertAPI generates apiAlert object from alert by its ID(hash)
-func (m *manager) alertAPI(gID, aID uint64) (*rule.ApiAlert, error) {
-	m.groupsMu.RLock()
-	defer m.groupsMu.RUnlock()
-
-	group, ok := m.groups[gID]
-	if !ok {
-		return nil, fmt.Errorf("can't find group with id %d", gID)
-	}
-	g := group.ToAPI()
-	for _, r := range g.Rules {
-		if r.Type != rule.TypeAlerting {
+		ar, ok := r.(*rule.AlertingRule)
+		if !ok {
 			continue
 		}
-		alertID := strconv.FormatUint(aID, 10)
-		for _, a := range r.Alerts {
-			if a.ID == alertID {
-				return a, nil
-			}
+		if apiAlert := alertToAPI(ar, aID); apiAlert != nil {
+			return apiAlert, nil
 		}
 	}
 	return nil, fmt.Errorf("can't find alert with id %d in group %q", aID, g.Name)
@@ -99,16 +82,17 @@ func (m *manager) close() {
 }
 
 func (m *manager) startGroup(ctx context.Context, g *rule.Group, restore bool) error {
+	m.wg.Add(1)
 	id := g.GetID()
 	g.Init()
-	m.wg.Go(func() {
+	go func() {
+		defer m.wg.Done()
 		if restore {
-			g.Start(ctx, m.rw, m.rr)
+			g.Start(ctx, m.notifiers, m.rw, m.rr)
 		} else {
-			g.Start(ctx, m.rw, nil)
+			g.Start(ctx, m.notifiers, m.rw, nil)
 		}
-	})
-
+	}()
 	m.groups[id] = g
 	return nil
 }
@@ -135,7 +119,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	if rrPresent && m.rw == nil {
 		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
 	}
-	if arPresent && notifier.GetTargets() == nil {
+	if arPresent && m.notifiers == nil {
 		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` nor `-notifier.blackhole` aren't set")
 	}
 
@@ -172,15 +156,15 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	if len(toUpdate) > 0 {
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
-			oldG := item.old
-			newG := item.new
-			wg.Go(func() {
-				// cancel evaluation so the Update will be applied as fast as possible.
-				// it is important to call InterruptEval before the update, because cancel fn
-				// can be re-assigned during the update.
-				oldG.InterruptEval()
-				oldG.UpdateWith(newG)
-			})
+			wg.Add(1)
+			// cancel evaluation so the Update will be applied as fast as possible.
+			// it is important to call InterruptEval before the update, because cancel fn
+			// can be re-assigned during the update.
+			item.old.InterruptEval()
+			go func(oldGroup *rule.Group, newGroup *rule.Group) {
+				oldGroup.UpdateWith(newGroup)
+				wg.Done()
+			}(item.old, item.new)
 		}
 		wg.Wait()
 	}

app/vmalert/manager_test.go

@@ -40,11 +40,10 @@ func TestManagerEmptyRulesDir(t *testing.T) {
 // execution of configuration update.
 // Should be executed with -race flag
 func TestManagerUpdateConcurrent(t *testing.T) {
-	_, cleanup := notifier.InitFakeNotifier()
-	defer cleanup()
 	m := &manager{
 		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: &datasource.FakeQuerier{},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 	}
 	paths := []string{
 		"config/testdata/dir/rules0-good.rules",
@@ -128,9 +127,8 @@ func TestManagerUpdate_Success(t *testing.T) {
 		m := &manager{
 			groups:         make(map[uint64]*rule.Group),
 			querierBuilder: &datasource.FakeQuerier{},
+			notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 		}
-		_, cleanup := notifier.InitFakeNotifier()
-		defer cleanup()
 
 		cfgInit := loadCfg(t, []string{initPath}, true, true)
 		if err := m.update(ctx, cfgInit, false); err != nil {
@@ -279,8 +277,7 @@ func TestManagerUpdate_Failure(t *testing.T) {
 			rw:             rw,
 		}
 		if notifiers != nil {
-			_, cleanup := notifier.InitFakeNotifier()
-			defer cleanup()
+			m.notifiers = func() []notifier.Notifier { return notifiers }
 		}
 		err := m.update(context.Background(), []config.Group{cfg}, false)
 		if err == nil {

app/vmalert/notifier/alert.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
@@ -166,8 +166,8 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl
 		ctmpl, _ := tmpl.Clone()
 		ctmpl = ctmpl.Option("missingkey=zero")
 		if err := templateAnnotation(&buf, builder.String(), tData, ctmpl, execute); err != nil {
-			r[key] = err.Error()
-			eg.Add(fmt.Errorf("(key: %q, value: %q): %w", key, text, err))
+			r[key] = text
+			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
 			continue
 		}
 		r[key] = buf.String()
@@ -184,21 +184,21 @@ type tplData struct {
 func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
 	tpl, err := tpl.Parse(text)
 	if err != nil {
-		return fmt.Errorf("error parsing template: %w", err)
+		return fmt.Errorf("error parsing annotation template: %w", err)
 	}
 	if !execute {
 		return nil
 	}
 	if err = tpl.Execute(dst, data); err != nil {
-		return fmt.Errorf("error evaluating template: %w", err)
+		return fmt.Errorf("error evaluating annotation template: %w", err)
 	}
 	return nil
 }
 
-func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompb.Label {
-	var labels []prompb.Label
+func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
+	var labels []prompbmarshal.Label
 	for k, v := range a.Labels {
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  promrelabel.SanitizeMetricName(k),
 			Value: v,
 		})

app/vmalert/notifier/alert_test.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
@@ -20,7 +20,7 @@ func TestAlertExecTemplate(t *testing.T) {
 	)
 	extLabels["cluster"] = extCluster
 	extLabels["dc"] = extDC
-	err := Init(extLabels, extURL)
+	_, err := Init(nil, extLabels, extURL)
 	checkErr(t, err)
 
 	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
@@ -33,7 +33,7 @@ func TestAlertExecTemplate(t *testing.T) {
 		qFn := func(_ string) ([]datasource.Metric, error) {
 			return []datasource.Metric{
 				{
-					Labels: []prompb.Label{
+					Labels: []prompbmarshal.Label{
 						{Name: "foo", Value: "bar"},
 						{Name: "baz", Value: "qux"},
 					},
@@ -41,7 +41,7 @@ func TestAlertExecTemplate(t *testing.T) {
 					Timestamps: []int64{1},
 				},
 				{
-					Labels: []prompb.Label{
+					Labels: []prompbmarshal.Label{
 						{Name: "foo", Value: "garply"},
 						{Name: "baz", Value: "fred"},
 					},
@@ -213,7 +213,7 @@ func TestAlertExecTemplate(t *testing.T) {
 }
 
 func TestAlert_toPromLabels(t *testing.T) {
-	fn := func(labels map[string]string, exp []prompb.Label, relabel *promrelabel.ParsedConfigs) {
+	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
 		t.Helper()
 		a := Alert{Labels: labels}
 		got := a.applyRelabelingIfNeeded(relabel)
@@ -226,12 +226,12 @@ func TestAlert_toPromLabels(t *testing.T) {
 	fn(nil, nil, nil)
 	fn(
 		map[string]string{"foo": "bar", "a": "baz"}, // unsorted
-		[]prompb.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "bar"}},
 		nil,
 	)
 	fn(
 		map[string]string{"foo.bar": "baz", "service!name": "qux"},
-		[]prompb.Label{{Name: "foo_bar", Value: "baz"}, {Name: "service_name", Value: "qux"}},
+		[]prompbmarshal.Label{{Name: "foo_bar", Value: "baz"}, {Name: "service_name", Value: "qux"}},
 		nil,
 	)
 
@@ -247,17 +247,17 @@ func TestAlert_toPromLabels(t *testing.T) {
 
 	fn(
 		map[string]string{"a": "baz"},
-		[]prompb.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
 		pcs,
 	)
 	fn(
 		map[string]string{"foo": "bar", "a": "baz"},
-		[]prompb.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
+		[]prompbmarshal.Label{{Name: "a", Value: "baz"}, {Name: "foo", Value: "aaa"}},
 		pcs,
 	)
 	fn(
 		map[string]string{"qux": "bar", "env": "prod", "environment": "production"},
-		[]prompb.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
+		[]prompbmarshal.Label{{Name: "foo", Value: "aaa"}, {Name: "qux", Value: "bar"}},
 		pcs,
 	)
 }

app/vmalert/notifier/alertmanager.go

@@ -3,7 +3,6 @@ package notifier
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -16,18 +15,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertManager represents integration provider with Prometheus alert manager
 // https://github.com/prometheus/alertmanager
 type AlertManager struct {
-	addr      *url.URL
-	argFunc   AlertURLGenerator
-	client    *http.Client
-	timeout   time.Duration
-	lastError string
+	addr    *url.URL
+	argFunc AlertURLGenerator
+	client  *http.Client
+	timeout time.Duration
 
 	authCfg *promauth.Config
 	// stores already parsed RelabelConfigs object
@@ -73,42 +71,24 @@ func (am AlertManager) Addr() string {
 	return am.addr.Redacted()
 }
 
-func (am *AlertManager) LastError() string {
-	return am.lastError
-}
-
 // Send an alert or resolve message
-func (am *AlertManager) Send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, headers map[string]string) error {
-	if len(alerts) != len(alertLabels) {
-		return fmt.Errorf("mismatched number of alerts and label sets after global alert relabeling")
-	}
+func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	am.metrics.alertsSent.Add(len(alerts))
 	startTime := time.Now()
-	err := am.send(ctx, alerts, alertLabels, headers)
+	err := am.send(ctx, alerts, headers)
 	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
-		// the context can be cancelled on graceful shutdown
-		// or on group update. So no need to handle the error as usual.
-		if errors.Is(err, context.Canceled) {
-			return nil
-		}
 		am.metrics.alertsSendErrors.Add(len(alerts))
-		am.lastError = err.Error()
-	} else {
-		am.lastError = ""
 	}
 	return err
 }
 
-func (am *AlertManager) send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, headers map[string]string) error {
+func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	b := &bytes.Buffer{}
 	alertsToSend := make([]Alert, 0, len(alerts))
-	lblss := make([][]prompb.Label, 0, len(alerts))
-	for i, a := range alerts {
-		lbls := alertLabels[i]
-		if am.relabelConfigs != nil {
-			lbls = am.relabelConfigs.Apply(lbls, 0)
-		}
+	lblss := make([][]prompbmarshal.Label, 0, len(alerts))
+	for _, a := range alerts {
+		lbls := a.applyRelabelingIfNeeded(am.relabelConfigs)
 		if len(lbls) == 0 {
 			continue
 		}

app/vmalert/notifier/alertmanager_request.qtpl

@@ -1,11 +1,11 @@
 {% import (
     "time"
 
-    "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 ) %}
 {% stripspace %}
 
-{% func amRequest(alerts []Alert, generatorURL func(Alert) string, lblss [][]prompb.Label) %}
+{% func amRequest(alerts []Alert, generatorURL func(Alert) string, lblss [][]prompbmarshal.Label) %}
 [
 {% for i, alert := range alerts %}
 {% code lbls := lblss[i] %}

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -8,7 +8,7 @@ package notifier
 import (
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 //line app/vmalert/notifier/alertmanager_request.qtpl:8
@@ -25,7 +25,7 @@ var (
 )
 
 //line app/vmalert/notifier/alertmanager_request.qtpl:8
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, lblss [][]prompb.Label) {
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, lblss [][]prompbmarshal.Label) {
 //line app/vmalert/notifier/alertmanager_request.qtpl:8
 	qw422016.N().S(`[`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:10
@@ -114,7 +114,7 @@ func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL fun
 }
 
 //line app/vmalert/notifier/alertmanager_request.qtpl:35
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, lblss [][]prompb.Label) {
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, lblss [][]prompbmarshal.Label) {
 //line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line app/vmalert/notifier/alertmanager_request.qtpl:35
@@ -125,7 +125,7 @@ func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL fun
 }
 
 //line app/vmalert/notifier/alertmanager_request.qtpl:35
-func amRequest(alerts []Alert, generatorURL func(Alert) string, lblss [][]prompb.Label) string {
+func amRequest(alerts []Alert, generatorURL func(Alert) string, lblss [][]prompbmarshal.Label) string {
 //line app/vmalert/notifier/alertmanager_request.qtpl:35
 	qb422016 := qt422016.AcquireByteBuffer()
 //line app/vmalert/notifier/alertmanager_request.qtpl:35

app/vmalert/notifier/alertmanager_test.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
@@ -146,11 +145,11 @@ func TestAlertManager_Send(t *testing.T) {
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, [][]prompb.Label{{{Name: "a", Value: "b"}}}, nil); err == nil {
+	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
 
-	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, [][]prompb.Label{{{Name: "a", Value: "b"}}}, nil); err == nil {
+	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
 		t.Fatalf("expected wrong http code error got nil")
 	}
 
@@ -161,7 +160,7 @@ func TestAlertManager_Send(t *testing.T) {
 		End:         time.Now().UTC(),
 		Labels:      map[string]string{"alertname": "alert0"},
 		Annotations: map[string]string{"a": "b", "c": "d"},
-	}}, [][]prompb.Label{{{Name: "alertname", Value: "alert0"}}}, map[string]string{headerKey: "bar"}); err != nil {
+	}}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
@@ -175,7 +174,7 @@ func TestAlertManager_Send(t *testing.T) {
 			Name:   "alert2",
 			Labels: map[string]string{"rule": "test", "tenant": "1"},
 		},
-	}, [][]prompb.Label{{{Name: "rule", Value: "test"}, {Name: "tenant", Value: "0"}}, {{Name: "rule", Value: "test"}, {Name: "tenant", Value: "1"}}}, map[string]string{headerKey: "bar"}); err != nil {
+	}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
@@ -188,7 +187,7 @@ func TestAlertManager_Send(t *testing.T) {
 			Name:   "alert2",
 			Labels: map[string]string{},
 		},
-	}, [][]prompb.Label{{{Name: "rule", Value: "test"}}, {{}}}, map[string]string{}); err != nil {
+	}, map[string]string{}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 

app/vmalert/notifier/config.go

@@ -27,9 +27,15 @@ type Config struct {
 	// PathPrefix is added to URL path before adding alertManagerPath value
 	PathPrefix string `yaml:"path_prefix,omitempty"`
 
-	ConsulSDConfigs []ConsulSDConfigs `yaml:"consul_sd_configs,omitempty"`
-	DNSSDConfigs    []DNSSDConfigs    `yaml:"dns_sd_configs,omitempty"`
-	StaticConfigs   []StaticConfig    `yaml:"static_configs,omitempty"`
+	// ConsulSDConfigs contains list of settings for service discovery via Consul
+	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
+	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// DNSSDConfigs contains list of settings for service discovery via DNS.
+	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+	DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
+
+	// StaticConfigs contains list of static targets
+	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
 
 	// HTTPClientConfig contains HTTP configuration for Notifier clients
 	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
@@ -56,29 +62,14 @@ type Config struct {
 	parsedAlertRelabelConfigs *promrelabel.ParsedConfigs
 }
 
-// staticConfig contains list of static targets in the following form:
+// StaticConfig contains list of static targets in the following form:
 //
 //	targets:
 //	[ - '<host>' ]
 type StaticConfig struct {
 	Targets []string `yaml:"targets"`
 	// HTTPClientConfig contains HTTP configuration for the Targets
-	HTTPClientConfig    promauth.HTTPClientConfig   `yaml:",inline"`
-	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
-}
-
-// ConsulSDConfigs contains list of settings for service discovery via Consul,
-// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
-type ConsulSDConfigs struct {
-	consul.SDConfig     `yaml:",inline"`
-	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
-}
-
-// DNSSDConfigs contains list of settings for service discovery via DNS,
-// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
-type DNSSDConfigs struct {
-	dns.SDConfig        `yaml:",inline"`
-	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -104,31 +95,6 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(any) error) error {
 	}
 	cfg.parsedAlertRelabelConfigs = arCfg
 
-	for _, s := range cfg.StaticConfigs {
-		if len(s.AlertRelabelConfigs) > 0 {
-			_, err := promrelabel.ParseRelabelConfigs(s.AlertRelabelConfigs)
-			if err != nil {
-				return fmt.Errorf("failed to parse alert_relabel_configs in static_config: %w", err)
-			}
-		}
-	}
-	for _, s := range cfg.ConsulSDConfigs {
-		if len(s.AlertRelabelConfigs) > 0 {
-			_, err := promrelabel.ParseRelabelConfigs(s.AlertRelabelConfigs)
-			if err != nil {
-				return fmt.Errorf("failed to parse alert_relabel_configs in consul_sd_config: %w", err)
-			}
-		}
-	}
-	for _, s := range cfg.DNSSDConfigs {
-		if len(s.AlertRelabelConfigs) > 0 {
-			_, err := promrelabel.ParseRelabelConfigs(s.AlertRelabelConfigs)
-			if err != nil {
-				return fmt.Errorf("failed to parse alert_relabel_configs in dns_sd_config: %w", err)
-			}
-		}
-	}
-
 	b, err := yaml.Marshal(cfg)
 	if err != nil {
 		return fmt.Errorf("failed to marshal configuration for checksum: %w", err)

app/vmalert/notifier/config_test.go

@@ -35,6 +35,4 @@ func TestParseConfig_Failure(t *testing.T) {
 
 	f("testdata/unknownFields.bad.yaml", "unknown field")
 	f("non-existing-file", "error reading")
-	f("testdata/consul.bad.yaml", "failed to parse alert_relabel_configs in consul_sd_config")
-	f("testdata/dns.bad.yaml", "failed to parse alert relabeling config")
 }

app/vmalert/notifier/config_watcher.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
@@ -29,7 +28,11 @@ type configWatcher struct {
 	targets   map[TargetType][]Target
 }
 
-func newWatcher(cfg *Config, gen AlertURLGenerator) (*configWatcher, error) {
+func newWatcher(path string, gen AlertURLGenerator) (*configWatcher, error) {
+	cfg, err := parseConfig(path)
+	if err != nil {
+		return nil, err
+	}
 	cw := &configWatcher{
 		cfg:       cfg,
 		wg:        sync.WaitGroup{},
@@ -85,15 +88,18 @@ func (cw *configWatcher) reload(path string) error {
 	return cw.start()
 }
 
-func (cw *configWatcher) add(typeK TargetType, interval time.Duration, targetsFn getTargets) error {
-	targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
+func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
+	targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
 	for _, err := range errors {
 		return fmt.Errorf("failed to init notifier for %q: %w", typeK, err)
 	}
 
 	cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 
-	cw.wg.Go(func() {
+	cw.wg.Add(1)
+	go func() {
+		defer cw.wg.Done()
+
 		ticker := time.NewTicker(interval)
 		defer ticker.Stop()
 
@@ -103,77 +109,62 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, targetsFn
 				return
 			case <-ticker.C:
 			}
-			targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
+			targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
 			for _, err := range errors {
 				logger.Errorf("failed to init notifier for %q: %w", typeK, err)
 			}
 			cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 		}
-	})
+	}()
 	return nil
 }
 
-type targetMetadata struct {
-	*promutil.Labels
-	alertRelabelConfigs *promrelabel.ParsedConfigs
-}
-
-func getTargetMetadata(targetsFn getTargets, cfg *Config) (map[string]targetMetadata, []error) {
-	metaLabelsList, alertRelabelCfgs, err := targetsFn()
+func getTargetMetadata(labelsFn getLabels, cfg *Config) (map[string]*promutil.Labels, []error) {
+	metaLabels, err := labelsFn()
 	if err != nil {
 		return nil, []error{fmt.Errorf("failed to get labels: %w", err)}
 	}
-	targetMts := make(map[string]targetMetadata, len(metaLabelsList))
+	targetMetadata := make(map[string]*promutil.Labels, len(metaLabels))
 	var errors []error
 	duplicates := make(map[string]struct{})
-	for i := range metaLabelsList {
-		metaLabels := metaLabelsList[i]
-		alertRelabelCfg := alertRelabelCfgs[i]
-		for _, labels := range metaLabels {
-			target := labels.Get("__address__")
-			u, processedLabels, err := parseLabels(target, labels, cfg)
-			if err != nil {
-				errors = append(errors, err)
-				continue
-			}
-			if len(u) == 0 {
-				continue
-			}
-			// check for duplicated targets
-			// targets with same address but different alert_relabel_configs are still considered duplicates since it's mostly due to misconfiguration and could cause duplicated notifications.
-			if _, ok := duplicates[u]; ok {
-				if !*suppressDuplicateTargetErrors {
-					logger.Errorf("skipping duplicate target with identical address %q; "+
-						"make sure service discovery and relabeling is set up properly; "+
-						"original labels: %s; resulting labels: %s",
-						u, labels, processedLabels)
-				}
-				continue
-			}
-			duplicates[u] = struct{}{}
-			targetMts[u] = targetMetadata{
-				Labels:              processedLabels,
-				alertRelabelConfigs: alertRelabelCfg,
-			}
+	for _, labels := range metaLabels {
+		target := labels.Get("__address__")
+		u, processedLabels, err := parseLabels(target, labels, cfg)
+		if err != nil {
+			errors = append(errors, err)
+			continue
 		}
+		if len(u) == 0 {
+			continue
+		}
+		if _, ok := duplicates[u]; ok { // check for duplicates
+			if !*suppressDuplicateTargetErrors {
+				logger.Errorf("skipping duplicate target with identical address %q; "+
+					"make sure service discovery and relabeling is set up properly; "+
+					"original labels: %s; resulting labels: %s",
+					u, labels, processedLabels)
+			}
+			continue
+		}
+		duplicates[u] = struct{}{}
+		targetMetadata[u] = processedLabels
 	}
-	return targetMts, errors
+	return targetMetadata, errors
 }
 
-type getTargets func() ([][]*promutil.Labels, []*promrelabel.ParsedConfigs, error)
+type getLabels func() ([]*promutil.Labels, error)
 
 func (cw *configWatcher) start() error {
 	if len(cw.cfg.StaticConfigs) > 0 {
 		var targets []Target
-		for i, cfg := range cw.cfg.StaticConfigs {
-			alertRelabelConfig, _ := promrelabel.ParseRelabelConfigs(cw.cfg.StaticConfigs[i].AlertRelabelConfigs)
+		for _, cfg := range cw.cfg.StaticConfigs {
 			httpCfg := mergeHTTPClientConfigs(cw.cfg.HTTPClientConfig, cfg.HTTPClientConfig)
 			for _, target := range cfg.Targets {
 				address, labels, err := parseLabels(target, nil, cw.cfg)
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %w", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, alertRelabelConfig, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %w", address, err)
 				}
@@ -187,20 +178,17 @@ func (cw *configWatcher) start() error {
 	}
 
 	if len(cw.cfg.ConsulSDConfigs) > 0 {
-		err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([][]*promutil.Labels, []*promrelabel.ParsedConfigs, error) {
-			var labels [][]*promutil.Labels
-			var alertRelabelConfigs []*promrelabel.ParsedConfigs
+		err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([]*promutil.Labels, error) {
+			var labels []*promutil.Labels
 			for i := range cw.cfg.ConsulSDConfigs {
-				alertRelabelConfig, _ := promrelabel.ParseRelabelConfigs(cw.cfg.ConsulSDConfigs[i].AlertRelabelConfigs)
 				sdc := &cw.cfg.ConsulSDConfigs[i]
 				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
 				if err != nil {
-					return nil, nil, fmt.Errorf("got labels err: %w", err)
+					return nil, fmt.Errorf("got labels err: %w", err)
 				}
-				labels = append(labels, targetLabels)
-				alertRelabelConfigs = append(alertRelabelConfigs, alertRelabelConfig)
+				labels = append(labels, targetLabels...)
 			}
-			return labels, alertRelabelConfigs, nil
+			return labels, nil
 		})
 		if err != nil {
 			return fmt.Errorf("failed to start consulSD discovery: %w", err)
@@ -208,21 +196,17 @@ func (cw *configWatcher) start() error {
 	}
 
 	if len(cw.cfg.DNSSDConfigs) > 0 {
-		err := cw.add(TargetDNS, *dns.SDCheckInterval, func() ([][]*promutil.Labels, []*promrelabel.ParsedConfigs, error) {
-			var labels [][]*promutil.Labels
-			var alertRelabelConfigs []*promrelabel.ParsedConfigs
+		err := cw.add(TargetDNS, *dns.SDCheckInterval, func() ([]*promutil.Labels, error) {
+			var labels []*promutil.Labels
 			for i := range cw.cfg.DNSSDConfigs {
-				alertRelabelConfig, _ := promrelabel.ParseRelabelConfigs(cw.cfg.DNSSDConfigs[i].AlertRelabelConfigs)
 				sdc := &cw.cfg.DNSSDConfigs[i]
 				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
 				if err != nil {
-					return nil, nil, fmt.Errorf("got labels err: %w", err)
+					return nil, fmt.Errorf("got labels err: %w", err)
 				}
-				labels = append(labels, targetLabels)
-				alertRelabelConfigs = append(alertRelabelConfigs, alertRelabelConfig)
-
+				labels = append(labels, targetLabels...)
 			}
-			return labels, alertRelabelConfigs, nil
+			return labels, nil
 		})
 		if err != nil {
 			return fmt.Errorf("failed to start DNSSD discovery: %w", err)
@@ -256,30 +240,30 @@ func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
 	cw.targetsMu.Unlock()
 }
 
-func (cw *configWatcher) updateTargets(key TargetType, targetMts map[string]targetMetadata, cfg *Config, genFn AlertURLGenerator) {
+func (cw *configWatcher) updateTargets(key TargetType, targetMetadata map[string]*promutil.Labels, cfg *Config, genFn AlertURLGenerator) {
 	cw.targetsMu.Lock()
 	defer cw.targetsMu.Unlock()
 	oldTargets := cw.targets[key]
 	var updatedTargets []Target
 	for _, ot := range oldTargets {
-		if _, ok := targetMts[ot.Addr()]; !ok {
+		if _, ok := targetMetadata[ot.Addr()]; !ok {
 			// if target not exists in currentTargets, close it
-			ot.Close()
+			ot.Notifier.Close()
 		} else {
 			updatedTargets = append(updatedTargets, ot)
-			delete(targetMts, ot.Addr())
+			delete(targetMetadata, ot.Addr())
 		}
 	}
 	// create new resources for the new targets
-	for addr, metadata := range targetMts {
-		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, metadata.alertRelabelConfigs, cfg.Timeout.Duration())
+	for addr, labels := range targetMetadata {
+		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
 			logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
 			continue
 		}
 		updatedTargets = append(updatedTargets, Target{
 			Notifier: am,
-			Labels:   metadata.Labels,
+			Labels:   labels,
 		})
 	}
 

app/vmalert/notifier/config_watcher_test.go

@@ -7,11 +7,9 @@ import (
 	"net/http/httptest"
 	"os"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
 )
@@ -21,19 +19,15 @@ func TestConfigWatcherReload(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer fs.MustRemovePath(f.Name())
+	defer func() { _ = os.Remove(f.Name()) }()
 
-	writeToFile(f.Name(), `
+	writeToFile(t, f.Name(), `
 static_configs:
   - targets:
       - localhost:9093
       - localhost:9094
 `)
-	cfg, err := parseConfig(f.Name())
-	if err != nil {
-		t.Fatalf("failed to parse config: %s", err)
-	}
-	cw, err := newWatcher(cfg, nil)
+	cw, err := newWatcher(f.Name(), nil)
 	if err != nil {
 		t.Fatalf("failed to start config watcher: %s", err)
 	}
@@ -47,9 +41,9 @@ static_configs:
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer fs.MustRemovePath(f2.Name())
+	defer func() { _ = os.Remove(f2.Name()) }()
 
-	writeToFile(f2.Name(), `
+	writeToFile(t, f2.Name(), `
 static_configs:
   - targets:
       - 127.0.0.1:9093
@@ -79,73 +73,42 @@ func TestConfigWatcherStart(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer fs.MustRemovePath(consulSDFile.Name())
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
 
-	writeToFile(consulSDFile.Name(), fmt.Sprintf(`
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
 scheme: https
 path_prefix: proxy
 consul_sd_configs:
   - server: %s
     services:
       - alertmanager
-  - server: %s
-    services:
-      - alertmanager
-    alert_relabel_configs:
-    - target_label: "foo"
-      replacement: "tar"
-`, consulSDServer.URL, consulSDServer.URL))
+`, consulSDServer.URL))
 
-	cfg, err := parseConfig(consulSDFile.Name())
-	if err != nil {
-		t.Fatalf("failed to parse config: %s", err)
-	}
-	cw, err := newWatcher(cfg, nil)
+	cw, err := newWatcher(consulSDFile.Name(), nil)
 	if err != nil {
 		t.Fatalf("failed to start config watcher: %s", err)
 	}
 	defer cw.mustStop()
 
-	if len(cw.notifiers()) != 3 {
-		t.Fatalf("expected to get 3 notifiers; got %d", len(cw.notifiers()))
+	if len(cw.notifiers()) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(cw.notifiers()))
 	}
 
 	expAddr1 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService1)
 	expAddr2 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService2)
-	expAddr3 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService3)
 
-	n1, n2, n3 := cw.notifiers()[0], cw.notifiers()[1], cw.notifiers()[2]
+	n1, n2 := cw.notifiers()[0], cw.notifiers()[1]
 	if n1.Addr() != expAddr1 {
 		t.Fatalf("exp address %q; got %q", expAddr1, n1.Addr())
 	}
 	if n2.Addr() != expAddr2 {
 		t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr())
 	}
-	if n3.Addr() != expAddr3 {
-		t.Fatalf("exp address %q; got %q", expAddr3, n3.Addr())
-	}
-
-	if n1.(*AlertManager).relabelConfigs.String() != "" {
-		t.Fatalf("unexpected relabel configs: %q", n1.(*AlertManager).relabelConfigs.String())
-	}
-	if n2.(*AlertManager).relabelConfigs.String() != "" {
-		t.Fatalf("unexpected relabel configs: %q", n2.(*AlertManager).relabelConfigs.String())
-	}
-	if n3.(*AlertManager).relabelConfigs.String() != "- target_label: foo\n  replacement: tar\n" {
-		t.Fatalf("unexpected relabel configs: %q", n3.(*AlertManager).relabelConfigs.String())
-	}
 
 	f := func() bool { return len(cw.notifiers()) == 1 }
 	if !waitFor(f, time.Second) {
 		t.Fatalf("expected to get 1 notifiers; got %d", len(cw.notifiers()))
 	}
-	n3 = cw.notifiers()[0]
-	if n3.Addr() != expAddr3 {
-		t.Fatalf("exp address %q; got %q", expAddr3, n3.Addr())
-	}
-	if n3.(*AlertManager).relabelConfigs.String() != "- target_label: foo\n  replacement: tar\n" {
-		t.Fatalf("unexpected relabel configs: %q", n3.(*AlertManager).relabelConfigs.String())
-	}
 }
 
 // TestConfigWatcherReloadConcurrent supposed to test concurrent
@@ -161,9 +124,9 @@ func TestConfigWatcherReloadConcurrent(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer fs.MustRemovePath(consulSDFile.Name())
+	defer func() { _ = os.Remove(consulSDFile.Name()) }()
 
-	writeToFile(consulSDFile.Name(), fmt.Sprintf(`
+	writeToFile(t, consulSDFile.Name(), fmt.Sprintf(`
 consul_sd_configs:
   - server: %s
     services:
@@ -177,9 +140,9 @@ consul_sd_configs:
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer fs.MustRemovePath(staticAndConsulSDFile.Name())
+	defer func() { _ = os.Remove(staticAndConsulSDFile.Name()) }()
 
-	writeToFile(staticAndConsulSDFile.Name(), fmt.Sprintf(`
+	writeToFile(t, staticAndConsulSDFile.Name(), fmt.Sprintf(`
 static_configs:
   - targets:
       - localhost:9093
@@ -200,11 +163,7 @@ consul_sd_configs:
 		"unknownFields.bad.yaml",
 	}
 
-	cfg, err := parseConfig(paths[0])
-	if err != nil {
-		t.Fatalf("failed to parse config: %s", err)
-	}
-	cw, err := newWatcher(cfg, nil)
+	cw, err := newWatcher(paths[0], nil)
 	if err != nil {
 		t.Fatalf("failed to start config watcher: %s", err)
 	}
@@ -228,8 +187,9 @@ consul_sd_configs:
 	wg.Wait()
 }
 
-func writeToFile(file, b string) {
-	fs.MustWriteSync(file, []byte(b))
+func writeToFile(t *testing.T, file, b string) {
+	t.Helper()
+	checkErr(t, os.WriteFile(file, []byte(b), 0644))
 }
 
 func checkErr(t *testing.T, err error) {
@@ -242,11 +202,10 @@ func checkErr(t *testing.T, err error) {
 const (
 	fakeConsulService1 = "127.0.0.1:9093"
 	fakeConsulService2 = "127.0.0.1:9095"
-	fakeConsulService3 = "127.0.0.1:9097"
 )
 
 func newFakeConsulServer() *httptest.Server {
-	var requestCount atomic.Int32
+	requestCount := 0
 	mux := http.NewServeMux()
 	mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) {
 		rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`))
@@ -261,7 +220,7 @@ func newFakeConsulServer() *httptest.Server {
 }`))
 	})
 	mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) {
-		if requestCount.Load() == 0 {
+		if requestCount == 0 {
 			rw.Header().Set("X-Consul-Index", "1")
 			rw.Write([]byte(`
 [
@@ -401,7 +360,7 @@ func newFakeConsulServer() *httptest.Server {
     }
 ]`))
 		}
-		requestCount.Add(1)
+		requestCount++
 	})
 
 	return httptest.NewServer(mux)

app/vmalert/notifier/faker.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"sync"
 	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
 // FakeNotifier is a mock notifier
@@ -17,32 +15,14 @@ type FakeNotifier struct {
 	counter int
 }
 
-// InitFakeNotifier initializes global notifier to FakeNotifier,
-// and returns a cleanup function to restore the original getActiveNotifiers.
-func InitFakeNotifier() (*FakeNotifier, func()) {
-	originalGetActiveNotifiers := getActiveNotifiers
-	fn := &FakeNotifier{}
-	getActiveNotifiers = func() []Notifier {
-		return []Notifier{fn}
-	}
-	return fn, func() {
-		getActiveNotifiers = originalGetActiveNotifiers
-	}
-}
-
 // Close does nothing
 func (*FakeNotifier) Close() {}
 
-// LastError returns last error message
-func (*FakeNotifier) LastError() string {
-	return ""
-}
-
 // Addr returns ""
 func (*FakeNotifier) Addr() string { return "" }
 
 // Send sets alerts and increases counter
-func (fn *FakeNotifier) Send(_ context.Context, alerts []Alert, _ [][]prompb.Label, _ map[string]string) error {
+func (fn *FakeNotifier) Send(_ context.Context, alerts []Alert, _ map[string]string) error {
 	fn.Lock()
 	defer fn.Unlock()
 	fn.counter += len(alerts)

app/vmalert/notifier/init.go

@@ -1,22 +1,14 @@
 package notifier
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"net/url"
-	"strconv"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 )
 
@@ -65,61 +57,11 @@ var (
 	sendTimeout = flagutil.NewArrayDuration("notifier.sendTimeout", 10*time.Second, "Timeout when sending alerts to the corresponding -notifier.url")
 )
 
-// AlertURLGeneratorFn returns a URL to the passed alert object.
-// Call InitAlertURLGeneratorFn before using this function.
-var AlertURLGeneratorFn AlertURLGenerator
-
-// InitAlertURLGeneratorFn populates AlertURLGeneratorFn
-func InitAlertURLGeneratorFn(externalURL *url.URL, externalAlertSource string, validateTemplate bool) error {
-	if externalAlertSource == "" {
-		AlertURLGeneratorFn = func(a Alert) string {
-			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
-			return fmt.Sprintf("%s/vmalert/alert?%s=%s&%s=%s", externalURL, "group_id", gID, "alert_id", aID)
-		}
-		return nil
-	}
-	if validateTemplate {
-		if err := ValidateTemplates(map[string]string{
-			"tpl": externalAlertSource,
-		}); err != nil {
-			return fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
-		}
-	}
-	m := map[string]string{
-		"tpl": externalAlertSource,
-	}
-	AlertURLGeneratorFn = func(alert Alert) string {
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
-		}
-		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
-		if err != nil {
-			logger.Errorf("cannot template alert source: %s", err)
-		}
-		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
-	}
-	return nil
-}
-
-var (
-	// getActiveNotifiers returns the current list of Notifier objects.
-	getActiveNotifiers func() []Notifier
-	// globalRelabelCfg stores the parsed alert relabeling config from the config file if there is
-	globalRelabelCfg *promrelabel.ParsedConfigs
-
-	// cw holds a configWatcher for configPath configuration file
-	// configWatcher provides a list of Notifier objects discovered
-	// from static config or via service discovery.
-	// cw is not nil only if configPath is provided.
-	cw *configWatcher
-
-	// externalLabels is a global variable for holding external labels configured via flags
-	// It is supposed to be inited via Init function only.
-	externalLabels map[string]string
-	// externalURL is a global variable for holding external URL value configured via flag
-	// It is supposed to be inited via Init function only.
-	externalURL string
-)
+// cw holds a configWatcher for configPath configuration file
+// configWatcher provides a list of Notifier objects discovered
+// from static config or via service discovery.
+// cw is not nil only if configPath is provided.
+var cw *configWatcher
 
 // Reload checks the changes in configPath configuration file
 // and applies changes if any.
@@ -130,62 +72,66 @@ func Reload() error {
 	return cw.reload(*configPath)
 }
 
+var staticNotifiersFn func() []Notifier
+
+var (
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
+
+// Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
 //   - configuration via flags (for backward compatibility). Is always static
 //     and don't support live reloads.
 //   - configuration via file. Supports live reloads and service discovery.
 //
 // Init returns an error if both mods are used.
-func Init(extLabels map[string]string, extURL string) error {
+func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
 	externalURL = extURL
 	externalLabels = extLabels
 	_, err := url.Parse(externalURL)
 	if err != nil {
-		return fmt.Errorf("failed to parse external URL: %w", err)
+		return nil, fmt.Errorf("failed to parse external URL: %w", err)
 	}
 
 	if *blackHole {
 		if len(*addrs) > 0 || *configPath != "" {
-			return fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
+			return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
 		}
 		notifier := newBlackHoleNotifier()
-		getActiveNotifiers = func() []Notifier {
+		staticNotifiersFn = func() []Notifier {
 			return []Notifier{notifier}
 		}
-		return nil
+		return staticNotifiersFn, nil
 	}
 
 	if *configPath == "" && len(*addrs) == 0 {
-		return nil
+		return nil, nil
 	}
 	if *configPath != "" && len(*addrs) > 0 {
-		return fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
+		return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
 	}
 
 	if len(*addrs) > 0 {
-		notifiers, err := notifiersFromFlags(AlertURLGeneratorFn)
+		notifiers, err := notifiersFromFlags(gen)
 		if err != nil {
-			return fmt.Errorf("failed to create notifier from flag values: %w", err)
+			return nil, fmt.Errorf("failed to create notifier from flag values: %w", err)
 		}
-		getActiveNotifiers = func() []Notifier {
+		staticNotifiersFn = func() []Notifier {
 			return notifiers
 		}
-		return nil
+		return staticNotifiersFn, nil
 	}
 
-	cfg, err := parseConfig(*configPath)
+	cw, err = newWatcher(*configPath, gen)
 	if err != nil {
-		return err
+		return nil, fmt.Errorf("failed to init config watcher: %w", err)
 	}
-	if cfg.AlertRelabelConfigs != nil {
-		globalRelabelCfg = cfg.parsedAlertRelabelConfigs
-	}
-	cw, err = newWatcher(cfg, AlertURLGeneratorFn)
-	if err != nil {
-		return fmt.Errorf("failed to init config watcher: %w", err)
-	}
-	getActiveNotifiers = cw.notifiers
-	return nil
+	return cw.notifiers, nil
 }
 
 // InitSecretFlags must be called after flag.Parse and before any logging
@@ -260,57 +206,23 @@ const (
 
 // GetTargets returns list of static or discovered targets
 // via notifier configuration.
-//
-// Must be called after Init.
 func GetTargets() map[TargetType][]Target {
-	if getActiveNotifiers == nil {
-		return nil
-	}
 	var targets = make(map[TargetType][]Target)
-	// use cached targets from configWatcher instead of getActiveNotifiers for the extra target labels
+
+	if staticNotifiersFn != nil {
+		for _, ns := range staticNotifiersFn() {
+			targets[TargetStatic] = append(targets[TargetStatic], Target{
+				Notifier: ns,
+			})
+		}
+	}
+
 	if cw != nil {
 		cw.targetsMu.RLock()
 		for key, ns := range cw.targets {
 			targets[key] = append(targets[key], ns...)
 		}
 		cw.targetsMu.RUnlock()
-		return targets
-	}
-
-	// static notifiers don't have labels
-	for _, ns := range getActiveNotifiers() {
-		targets[TargetStatic] = append(targets[TargetStatic], Target{
-			Notifier: ns,
-		})
 	}
 	return targets
 }
-
-// Send sends alerts to all active notifiers
-func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) *vmalertutil.ErrGroup {
-	alertsToSend := make([]Alert, 0, len(alerts))
-	lblss := make([][]prompb.Label, 0, len(alerts))
-	// apply global relabel config first without modifying original alerts in alerts
-	for _, a := range alerts {
-		lbls := a.applyRelabelingIfNeeded(globalRelabelCfg)
-		if len(lbls) == 0 {
-			continue
-		}
-		alertsToSend = append(alertsToSend, a)
-		lblss = append(lblss, lbls)
-	}
-
-	errGr := new(vmalertutil.ErrGroup)
-	wg := sync.WaitGroup{}
-	activeNotifiers := getActiveNotifiers()
-	for i := range activeNotifiers {
-		nt := activeNotifiers[i]
-		wg.Go(func() {
-			if err := nt.Send(ctx, alertsToSend, lblss, notifierHeaders); err != nil {
-				errGr.Add(fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err))
-			}
-		})
-	}
-	wg.Wait()
-	return errGr
-}

app/vmalert/notifier/init_test.go

@@ -1,17 +1,9 @@
 package notifier
 
 import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"os"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 )
 
 func TestInit(t *testing.T) {
@@ -20,13 +12,14 @@ func TestInit(t *testing.T) {
 
 	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
 
-	err := Init(nil, "")
+	fn, err := Init(nil, nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
 
-	if len(getActiveNotifiers()) != 2 {
-		t.Fatalf("expected to get 2 notifiers; got %d", len(getActiveNotifiers()))
+	nfs := fn()
+	if len(nfs) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(nfs))
 	}
 
 	targets := GetTargets()
@@ -59,7 +52,7 @@ func TestInitNegative(t *testing.T) {
 		*configPath = path
 		*addrs = flagutil.ArrayString{addr}
 		*blackHole = bh
-		if err := Init(nil, ""); err == nil {
+		if _, err := Init(nil, nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
 		}
 	}
@@ -76,13 +69,14 @@ func TestBlackHole(t *testing.T) {
 
 	*blackHole = true
 
-	err := Init(nil, "")
+	fn, err := Init(nil, nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
 
-	if len(getActiveNotifiers()) != 1 {
-		t.Fatalf("expected to get 1 notifier; got %d", len(getActiveNotifiers()))
+	nfs := fn()
+	if len(nfs) != 1 {
+		t.Fatalf("expected to get 1 notifier; got %d", len(nfs))
 	}
 
 	targets := GetTargets()
@@ -97,112 +91,3 @@ func TestBlackHole(t *testing.T) {
 		t.Fatalf("expected to get \"blackhole\"; got %q instead", nf1.Addr())
 	}
 }
-
-func TestGetAlertURLGenerator(t *testing.T) {
-	oldAlertURLGeneratorFn := AlertURLGeneratorFn
-	defer func() { AlertURLGeneratorFn = oldAlertURLGeneratorFn }()
-
-	testAlert := Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
-	u, _ := url.Parse("https://victoriametrics.com/path")
-	err := InitAlertURLGeneratorFn(u, "", false)
-	if err != nil {
-		t.Fatalf("unexpected error %s", err)
-	}
-	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", "group_id", "alert_id")
-	if exp != AlertURLGeneratorFn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, AlertURLGeneratorFn(testAlert))
-	}
-	err = InitAlertURLGeneratorFn(nil, "foo?{{invalid}}", true)
-	if err == nil {
-		t.Fatalf("expected template validation error got nil")
-	}
-	err = InitAlertURLGeneratorFn(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
-	if err != nil {
-		t.Fatalf("unexpected error %s", err)
-	}
-	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != AlertURLGeneratorFn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, AlertURLGeneratorFn(testAlert))
-	}
-}
-
-func TestSendAlerts(t *testing.T) {
-	oldAlertURLGeneratorFn := AlertURLGeneratorFn
-	defer func() { AlertURLGeneratorFn = oldAlertURLGeneratorFn }()
-	AlertURLGeneratorFn = func(alert Alert) string {
-		return ""
-	}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
-	})
-	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
-		var a []struct {
-			Labels map[string]string `json:"labels"`
-		}
-		if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
-			t.Fatalf("can not unmarshal data into alert %s", err)
-		}
-		if len(a) != 2 {
-			t.Fatalf("expected 2 alert in array got %d", len(a))
-		}
-		if len(a[0].Labels) != 4 {
-			t.Fatalf("expected 4 labels got %d", len(a[0].Labels))
-		}
-		if a[0].Labels["env"] != "prod" {
-			t.Fatalf("expected env label to be prod during relabeling, got %s", a[0].Labels["env"])
-		}
-		if a[0].Labels["c"] != "baz" {
-			t.Fatalf("expected c label to be baz during relabeling, got %s", a[0].Labels["c"])
-		}
-		if len(a[1].Labels) != 1 {
-			t.Fatalf("expected 1 labels got %d", len(a[1].Labels))
-		}
-	})
-	srv := httptest.NewServer(mux)
-	defer srv.Close()
-
-	f, err := os.CreateTemp("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer fs.MustRemovePath(f.Name())
-
-	rawConfig := `
-static_configs:
-  - targets:
-      - %s
-    alert_relabel_configs:
-    - source_labels: [b]
-      target_label: "c"
-alert_relabel_configs:
-  - source_labels: [a]
-    target_label: "b"
-  - target_label: "env"
-    replacement: "prod"
-`
-	config := fmt.Sprintf(rawConfig, srv.URL+alertManagerPath)
-	writeToFile(f.Name(), config)
-
-	oldConfigPath := configPath
-	defer func() { configPath = oldConfigPath }()
-	*configPath = f.Name()
-	err = Init(nil, "")
-	if err != nil {
-		t.Fatalf("unexpected error when parse notifier config: %s", err)
-	}
-
-	firingAlerts := []Alert{
-		{
-			Name:   "alert1",
-			Labels: map[string]string{"a": "baz"},
-		},
-		{
-			Name:   "alert2",
-			Labels: map[string]string{},
-		},
-	}
-	errG := Send(context.Background(), firingAlerts, nil)
-	if errG.Err() != nil {
-		t.Fatalf("unexpected error when sending alerts: %s", err)
-	}
-}

app/vmalert/notifier/notifier.go

@@ -1,21 +1,15 @@
 package notifier
 
-import (
-	"context"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-)
+import "context"
 
 // Notifier is a common interface for alert manager provider
 type Notifier interface {
 	// Send sends the given list of alerts.
 	// Returns an error if fails to send the alerts.
 	// Must unblock if the given ctx is cancelled.
-	Send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, notifierHeaders map[string]string) error
+	Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error
 	// Addr returns address where alerts are sent.
 	Addr() string
-	// LastError returns error, that occured during last attempt to send data
-	LastError() string
 	// Close is a destructor for the Notifier
 	Close()
 }

app/vmalert/notifier/notifier_blackhole.go

@@ -1,10 +1,6 @@
 package notifier
 
-import (
-	"context"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-)
+import "context"
 
 // blackHoleNotifier is a Notifier stub, used when no notifications need
 // to be sent.
@@ -14,7 +10,7 @@ type blackHoleNotifier struct {
 }
 
 // Send will send no notifications, but increase the metric.
-func (bh *blackHoleNotifier) Send(_ context.Context, alerts []Alert, _ [][]prompb.Label, _ map[string]string) error { //nolint:revive
+func (bh *blackHoleNotifier) Send(_ context.Context, alerts []Alert, _ map[string]string) error { //nolint:revive
 	bh.metrics.alertsSent.Add(len(alerts))
 	return nil
 }
@@ -29,11 +25,6 @@ func (bh *blackHoleNotifier) Close() {
 	bh.metrics.close()
 }
 
-// LastError return last notifier's error
-func (bh *blackHoleNotifier) LastError() string {
-	return ""
-}
-
 // newBlackHoleNotifier creates a new blackHoleNotifier
 func newBlackHoleNotifier() *blackHoleNotifier {
 	address := "blackhole"

app/vmalert/notifier/notifier_blackhole_test.go

@@ -5,7 +5,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	metricset "github.com/VictoriaMetrics/metrics"
 )
 
@@ -17,7 +16,7 @@ func TestBlackHoleNotifier_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, [][]prompb.Label{{}}, nil); err != nil {
+	}}, nil); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
@@ -35,7 +34,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, [][]prompb.Label{{}}, nil); err != nil {
+	}}, nil); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 

app/vmalert/notifier/testdata/consul.bad.yaml

@@ -1,19 +0,0 @@
-consul_sd_configs:
-  - server: localhost:8500
-    scheme: http
-    services:
-      - alertmanager
-    alert_relabel_configs:
-    - action: keep
-      source_labels: [env]
-      regex: "prod"
-  - server: localhost:8500
-    services:
-      - consul
-    alert_relabel_configs:
-    - action: keep
-      source_labels: [env]
-      regex: "(abc"
-alert_relabel_configs:
-  - target_label: "foo"
-    replacement: "aaa"

app/vmalert/notifier/testdata/dns.bad.yaml

@@ -1,13 +0,0 @@
-dns_sd_configs:
-  - names:
-      - cloudflare.com
-    type: 'A'
-    port: 9093
-relabel_configs:
-  - source_labels: [__meta_dns_name]
-    replacement: '${1}'
-    target_label: dns_name
-alert_relabel_configs:
-  - action: keep
-    source_labels: [env]
-    regex: "(abc"

app/vmalert/notifier/testdata/mixed.good.yaml

@@ -2,19 +2,12 @@ static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
-    alert_relabel_configs:
-      - action: keep
-        source_labels: [env]
-        regex: "static"
+
 consul_sd_configs:
   - server: localhost:8500
     scheme: http
     services:
       - alertmanager
-    alert_relabel_configs:
-      - action: keep
-        source_labels: [env]
-        regex: "consul"
   - server: localhost:8500
     services:
       - consul
@@ -24,10 +17,6 @@ dns_sd_configs:
       - cloudflare.com
     type: 'A'
     port: 9093
-    alert_relabel_configs:
-      - action: keep
-        source_labels: [env]
-        regex: "dns"
 
 relabel_configs:
   - source_labels: [__meta_consul_tags]
@@ -36,4 +25,4 @@ relabel_configs:
     target_label: __scheme__
   - source_labels: [__meta_dns_name]
     replacement: '${1}'
-    target_label: dns_name
+    target_label: dns_name

app/vmalert/notifier/testdata/static.good.yaml

@@ -1,14 +1,22 @@
+headers:
+  - 'CustomHeader: foo'
+
 static_configs:
   - targets:
-      - http://192.168.0.101:9093
-    alert_relabel_configs:
-    - target_label: "foo"
-      replacement: "aaa"
+      - localhost:9093
+      - localhost:9095
+      - https://localhost:9093/test/api/v2/alerts
+    basic_auth:
+      username: foo
+      password: bar
 
   - targets:
-      - http://192.168.0.101:9093
-    alert_relabel_configs:
-    - target_label: "foo"
-      replacement: "ccc"
-
+      - localhost:9096
+      - localhost:9097
+    basic_auth:
+      username: foo
+      password: baz
 
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/web.go

@@ -1,19 +0,0 @@
-package notifier
-
-// ApiNotifier represents a Notifier configuration for WEB view
-type ApiNotifier struct {
-	// Kind is a Notifier type
-	Kind TargetType `json:"kind"`
-	// Targets is a list of Notifier targets
-	Targets []*ApiTarget `json:"targets"`
-}
-
-// ApiTarget represents a specific Notifier target for WEB view
-type ApiTarget struct {
-	// Address is a URL for sending notifications
-	Address string `json:"address"`
-	// Labels is a list of labels to add to each sent notification
-	Labels map[string]string `json:"labels"`
-	// LastError contains the error faced while sending to notifier.
-	LastError string `json:"lastError"`
-}

app/vmalert/remoteread/init.go

@@ -14,9 +14,9 @@ import (
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect. "+
-		"Remote read is used to restore alerts state. "+
-		"This configuration makes sense only if vmalert was configured with '-remoteWrite.url' before and has been successfully persisted its state. "+
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect."+
+		"Remote read is used to restore alerts state."+
+		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
 		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
 

app/vmalert/remotewrite/client.go

@@ -20,7 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -46,7 +46,7 @@ type Client struct {
 	addr          string
 	c             *http.Client
 	authCfg       *promauth.Config
-	input         chan prompb.TimeSeries
+	input         chan prompbmarshal.TimeSeries
 	flushInterval time.Duration
 	maxBatchSize  int
 	maxQueueSize  int
@@ -110,7 +110,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		maxBatchSize:  cfg.MaxBatchSize,
 		maxQueueSize:  cfg.MaxQueueSize,
 		doneCh:        make(chan struct{}),
-		input:         make(chan prompb.TimeSeries, cfg.MaxQueueSize),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
 	}
 
 	for i := 0; i < cc; i++ {
@@ -121,7 +121,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 
 // Push adds timeseries into queue for writing into remote storage.
 // Push returns and error if client is stopped or if queue is full.
-func (c *Client) Push(s prompb.TimeSeries) error {
+func (c *Client) Push(s prompbmarshal.TimeSeries) error {
 	rwTotal.Inc()
 	select {
 	case <-c.doneCh:
@@ -158,7 +158,7 @@ func (c *Client) Close() error {
 
 func (c *Client) run(ctx context.Context) {
 	ticker := time.NewTicker(c.flushInterval)
-	wr := &prompb.WriteRequest{}
+	wr := &prompbmarshal.WriteRequest{}
 	shutdown := func() {
 		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
 
@@ -173,8 +173,9 @@ func (c *Client) run(ctx context.Context) {
 
 		cancel()
 	}
-
-	c.wg.Go(func() {
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
 		defer ticker.Stop()
 		for {
 			select {
@@ -196,7 +197,7 @@ func (c *Client) run(ctx context.Context) {
 				}
 			}
 		}
-	})
+	}()
 }
 
 var (
@@ -220,7 +221,7 @@ func GetDroppedRows() int { return int(droppedRows.Get()) }
 // flush is a blocking function that marshals WriteRequest and sends
 // it to remote-write endpoint. Flush performs limited amount of retries
 // if request fails.
-func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
+func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 	if len(wr.Timeseries) < 1 {
 		return
 	}

app/vmalert/remotewrite/client_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestClient_Push(t *testing.T) {
@@ -45,8 +46,8 @@ func TestClient_Push(t *testing.T) {
 	r := rand.New(rand.NewSource(1))
 	const rowsN = int(1e4)
 	for i := 0; i < rowsN; i++ {
-		s := prompb.TimeSeries{
-			Samples: []prompb.Sample{{
+		s := prompbmarshal.TimeSeries{
+			Samples: []prompbmarshal.Sample{{
 				Value:     r.Float64(),
 				Timestamp: time.Now().Unix(),
 			}},
@@ -103,7 +104,7 @@ func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
 
 		// push time series to the client.
 		for i := 0; i < pushCnt; i++ {
-			if err = rwClient.Push(prompb.TimeSeries{}); err != nil {
+			if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
 				t.Fatalf("cannot time series to the client: %s", err)
 			}
 		}
@@ -182,9 +183,8 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("decode err: %w", err))
 		return
 	}
-	wru := &prompb.WriteRequestUnmarshaler{}
-	wr, err := wru.UnmarshalProtobuf(b)
-	if err != nil {
+	wr := &prompb.WriteRequest{}
+	if err := wr.UnmarshalProtobuf(b); err != nil {
 		rw.err(w, fmt.Errorf("unmarhsal err: %w", err))
 		return
 	}

app/vmalert/remotewrite/debug_client.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // DebugClient won't push series periodically, but will write data to remote endpoint
@@ -48,10 +48,10 @@ func NewDebugClient() (*DebugClient, error) {
 }
 
 // Push sends the given timeseries to the remote storage.
-func (c *DebugClient) Push(s prompb.TimeSeries) error {
+func (c *DebugClient) Push(s prompbmarshal.TimeSeries) error {
 	c.wg.Add(1)
 	defer c.wg.Done()
-	wr := &prompb.WriteRequest{Timeseries: []prompb.TimeSeries{s}}
+	wr := &prompbmarshal.WriteRequest{Timeseries: []prompbmarshal.TimeSeries{s}}
 	data := wr.MarshalProtobuf(nil)
 
 	return c.send(data)

app/vmalert/remotewrite/debug_client_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestDebugClient_Push(t *testing.T) {
@@ -23,8 +23,8 @@ func TestDebugClient_Push(t *testing.T) {
 	const rowsN = 100
 	var sent int
 	for i := 0; i < rowsN; i++ {
-		s := prompb.TimeSeries{
-			Samples: []prompb.Sample{{
+		s := prompbmarshal.TimeSeries{
+			Samples: []prompbmarshal.Sample{{
 				Value:     float64(i),
 				Timestamp: time.Now().Unix(),
 			}},

app/vmalert/remotewrite/remotewrite.go

@@ -1,13 +1,13 @@
 package remotewrite
 
 import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // RWClient represents an HTTP client for pushing data via remote write protocol
 type RWClient interface {
 	// Push pushes the give time series to remote storage
-	Push(s prompb.TimeSeries) error
+	Push(s prompbmarshal.TimeSeries) error
 	// Close stops the client. Client can't be reused after Close call.
 	Close() error
 }

app/vmalert/replay.go

@@ -28,8 +28,8 @@ var (
 		"Defines how many retries to make before giving up on rule if request for it returns an error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
-	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
-		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
+	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent `/query_range` requests for a single rule. "+
+		"Increasing this value when replaying for a long time and a single request range is limited by `-replay.maxDatapointsPerQuery`.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {

app/vmalert/replay_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 )
 
@@ -23,7 +23,7 @@ func (fr *fakeReplayQuerier) BuildWithParams(_ datasource.QuerierParams) datasou
 
 type fakeRWClient struct{}
 
-func (fc *fakeRWClient) Push(_ prompb.TimeSeries) error {
+func (fc *fakeRWClient) Push(_ prompbmarshal.TimeSeries) error {
 	return nil
 }
 
@@ -246,33 +246,24 @@ func TestReplay(t *testing.T) {
 
 	// multiple rules + rule concurrency + group concurrency
 	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 3, 0, []config.Group{
-		{Rules: []config.Rule{{Alert: "foo-group-single-concurrent", For: promutil.NewDuration(30 * time.Second), Expr: "sum(up) > 1"}, {Alert: "bar-group-single-concurrent", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
+		{Rules: []config.Rule{{Alert: "foo-group-single-concurrent", Expr: "sum(up) > 1"}, {Alert: "bar-group-single-concurrent", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
 		registry: map[string]map[string][]datasource.Metric{
 			"sum(up) > 1": {
-				"12:00:00+12:01:00": {{
-					Timestamps: []int64{1609502460},
-					Values:     []float64{1},
-				}},
+				"12:00:00+12:01:00": {},
 				"12:01:00+12:02:00": {{
-					Timestamps: []int64{1609502520},
-					Values:     []float64{1},
-				}},
-				"12:02:00+12:02:30": {{
-					Timestamps: []int64{1609502580},
+					Timestamps: []int64{1},
 					Values:     []float64{1},
 				}},
+				"12:02:00+12:02:30": {},
 			},
 			"max(up) < 1": {
-				"12:00:00+12:01:00": {{
-					Timestamps: []int64{1609502460},
-					Values:     []float64{1},
-				}},
+				"12:00:00+12:01:00": {},
 				"12:01:00+12:02:00": {{
-					Timestamps: []int64{1609502520},
+					Timestamps: []int64{1},
 					Values:     []float64{1},
 				}},
 				"12:02:00+12:02:30": {},
 			},
 		},
-	}, 10)
+	}, 4)
 }

app/vmalert/rule/alerting.go

@@ -2,7 +2,6 @@ package rule
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"hash/fnv"
 	"math"
@@ -20,7 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
@@ -188,54 +187,6 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
-// ToAPI returns ApiRule representation of ar
-func (ar *AlertingRule) ToAPI() ApiRule {
-	state := ar.state
-	lastState := state.getLast()
-	r := ApiRule{
-		Type:              TypeAlerting,
-		DatasourceType:    ar.Type.String(),
-		Name:              ar.Name,
-		Query:             ar.Expr,
-		Duration:          ar.For.Seconds(),
-		KeepFiringFor:     ar.KeepFiringFor.Seconds(),
-		Labels:            ar.Labels,
-		Annotations:       ar.Annotations,
-		LastEvaluation:    lastState.Time,
-		EvaluationTime:    lastState.Duration.Seconds(),
-		Health:            "ok",
-		State:             "inactive",
-		Alerts:            ar.AlertsToAPI(),
-		LastSamples:       lastState.Samples,
-		LastSeriesFetched: lastState.SeriesFetched,
-		MaxUpdates:        state.size(),
-		Updates:           state.getAll(),
-		Debug:             ar.Debug,
-
-		// encode as strings to avoid rounding in JSON
-		ID:        fmt.Sprintf("%d", ar.ID()),
-		GroupID:   fmt.Sprintf("%d", ar.GroupID),
-		GroupName: ar.GroupName,
-		File:      ar.File,
-	}
-	if lastState.Err != nil {
-		r.LastError = lastState.Err.Error()
-		r.Health = "err"
-	}
-	// satisfy apiRule.State logic
-	if len(r.Alerts) > 0 {
-		r.State = notifier.StatePending.String()
-		stateFiring := notifier.StateFiring.String()
-		for _, a := range r.Alerts {
-			if a.State == stateFiring {
-				r.State = stateFiring
-				break
-			}
-		}
-	}
-	return r
-}
-
 // GetAlerts returns active alerts of rule
 func (ar *AlertingRule) GetAlerts() []*notifier.Alert {
 	ar.alertsMu.RLock()
@@ -247,6 +198,16 @@ func (ar *AlertingRule) GetAlerts() []*notifier.Alert {
 	return alerts
 }
 
+// GetAlert returns alert if id exists
+func (ar *AlertingRule) GetAlert(id uint64) *notifier.Alert {
+	ar.alertsMu.RLock()
+	defer ar.alertsMu.RUnlock()
+	if ar.alerts == nil {
+		return nil
+	}
+	return ar.alerts[id]
+}
+
 func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...any) {
 	if !ar.Debug {
 		return
@@ -312,11 +273,6 @@ type labelSet struct {
 // On k conflicts in origin set, the original value is preferred and copied
 // to processed with `exported_%k` key. The copy happens only if passed v isn't equal to origin[k] value.
 func (ls *labelSet) add(k, v string) {
-	// do not add label with empty value, since it has no meaning.
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9984
-	if v == "" {
-		return
-	}
 	ls.processed[k] = v
 	ov, ok := ls.origin[k]
 	if !ok {
@@ -351,6 +307,9 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		Value:  m.Values[0],
 		Expr:   ar.Expr,
 	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %w", err)
+	}
 	for k, v := range extraLabels {
 		ls.add(k, v)
 	}
@@ -361,7 +320,7 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
 		ls.add(alertGroupNameLabel, ar.GroupName)
 	}
-	return ls, err
+	return ls, nil
 }
 
 // execRange executes alerting rule on the given time range similarly to exec.
@@ -369,12 +328,12 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 // as this function modifies AlertingRule alerts state.
 // It is not thread safe.
 // It returns ALERT and ALERT_FOR_STATE time series as a result.
-func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]prompb.TimeSeries, error) {
+func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error) {
 	res, err := ar.q.QueryRange(ctx, ar.Expr, start, end)
 	if err != nil {
 		return nil, err
 	}
-	var result []prompb.TimeSeries
+	var result []prompbmarshal.TimeSeries
 	holdAlertState := make(map[uint64]*notifier.Alert)
 	qFn := func(_ string) ([]datasource.Metric, error) {
 		logger.Warnf("`query` template isn't supported in replay mode, mocked data is used")
@@ -382,15 +341,11 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 		return []datasource.Metric{{Timestamps: []int64{0}, Values: []float64{math.NaN()}}}, nil
 	}
 	for _, s := range res.Data {
-		ls, err := ar.expandLabelTemplates(s, qFn)
+		ls, as, err := ar.expandTemplates(s, qFn, time.Time{})
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to expand templates: %s", err)
 		}
 		alertID := hash(ls.processed)
-		as, err := ar.expandAnnotationTemplates(s, qFn, time.Time{}, ls)
-		if err != nil {
-			return nil, err
-		}
 		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
 
 		prevT := time.Time{}
@@ -408,7 +363,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				a.State = notifier.StatePending
 				a.ActiveAt = at
 				// re-template the annotations as active timestamp is changed
-				a.Annotations, _ = ar.expandAnnotationTemplates(s, qFn, at, ls)
+				_, a.Annotations, _ = ar.expandTemplates(s, qFn, at)
 				a.Start = time.Time{}
 			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
 				a.State = notifier.StateFiring
@@ -421,15 +376,13 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
 
-			// if for>0, save alert's state on last iteration, so it can be used on the next execRange call
-			if ar.For > 0 && at.Equal(end) {
+			// save alert's state on last iteration, so it can be used on the next execRange call
+			if at.Equal(end) {
 				holdAlertState[alertID] = a
 			}
 		}
 	}
-	if len(holdAlertState) > 0 {
-		ar.alerts = holdAlertState
-	}
+	ar.alerts = holdAlertState
 	return result, nil
 }
 
@@ -439,7 +392,7 @@ const resolvedRetention = 15 * time.Minute
 
 // exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]prompb.TimeSeries, error) {
+func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
 	start := time.Now()
 	res, req, err := ar.q.Query(ctx, ar.Expr, ts)
 	curState := StateEntry{
@@ -454,7 +407,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 
 	defer func() {
 		ar.state.add(curState)
-		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
+		if curState.Err != nil {
 			ar.metrics.errors.Inc()
 		}
 	}()
@@ -475,25 +428,10 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	expandedLabels := make([]*labelSet, len(res.Data))
 	expandedAnnotations := make([]map[string]string, len(res.Data))
 	for i, m := range res.Data {
-		ls, err := ar.expandLabelTemplates(m, qFn)
+		ls, as, err := ar.expandTemplates(m, qFn, ts)
 		if err != nil {
-			// only set error in current state, but do not break alert processing
-			curState.Err = err
-			logger.Errorf("got templating error in rule %s: %q", ar.Name, err)
-		}
-		at := ts
-		alertID := hash(ls.processed)
-		if a, ok := ar.alerts[alertID]; ok {
-			// modify activeAt for annotation templating if the alert has already triggered(in state Pending or Firing)
-			if a.State != notifier.StateInactive {
-				at = a.ActiveAt
-			}
-		}
-		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
-		if err != nil {
-			// only set error in current state, but do not break alert processing
-			curState.Err = err
-			logger.Errorf("got templating error in rule %s: %q", ar.Name, err)
+			curState.Err = fmt.Errorf("failed to expand templates: %w", err)
+			return nil, curState.Err
 		}
 		expandedLabels[i] = ls
 		expandedAnnotations[i] = as
@@ -535,7 +473,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
-
 		a := ar.newAlert(m, ts, labels.processed, annotations)
 		a.ID = alertID
 		a.State = notifier.StatePending
@@ -543,7 +480,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
 	var numActivePending int
-	var tss []prompb.TimeSeries
+	var tss []prompbmarshal.TimeSeries
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
@@ -599,15 +536,12 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	return append(tss, ar.toTimeSeries(ts.Unix())...), nil
 }
 
-func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
+func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
 	ls, err := ar.toLabels(m, qFn)
 	if err != nil {
-		return ls, fmt.Errorf("failed to expand label templates: %s", err)
+		return nil, nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
-	return ls, nil
-}
 
-func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet) (map[string]string, error) {
 	tplData := notifier.AlertTplData{
 		Value:    m.Values[0],
 		Type:     ar.Type.String(),
@@ -615,19 +549,19 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
 		Expr:     ar.Expr,
 		AlertID:  hash(ls.processed),
 		GroupID:  ar.GroupID,
-		ActiveAt: activeAt,
+		ActiveAt: ts,
 		For:      ar.For,
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {
-		return as, fmt.Errorf("failed to expand annotation templates: %s", err)
+		return nil, nil, fmt.Errorf("failed to template annotations: %w", err)
 	}
-	return as, nil
+	return ls, as, nil
 }
 
 // toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts
-func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompb.TimeSeries {
-	var tss []prompb.TimeSeries
+func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
 	for _, a := range ar.alerts {
 		if a.State == notifier.StateInactive {
 			continue
@@ -698,67 +632,67 @@ const (
 )
 
 // alertToTimeSeries converts the given alert with the given timestamp to time series
-func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompb.TimeSeries {
-	return []prompb.TimeSeries{
+func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
+	return []prompbmarshal.TimeSeries{
 		alertToTimeSeries(a, timestamp),
 		alertForToTimeSeries(a, timestamp),
 	}
 }
 
-func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompb.TimeSeries {
-	labels := make([]prompb.Label, 0, len(a.Labels)+2)
+func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+	labels := make([]prompbmarshal.Label, 0, len(a.Labels)+2)
 	for k, v := range a.Labels {
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  k,
 			Value: v,
 		})
 	}
 	// __name__ already been dropped, no need to check duplication
-	labels = append(labels, prompb.Label{Name: "__name__", Value: alertMetricName})
+	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
 	if ol := promrelabel.GetLabelByName(labels, alertStateLabel); ol != nil {
 		ol.Value = a.State.String()
 	} else {
-		labels = append(labels, prompb.Label{Name: alertStateLabel, Value: a.State.String()})
+		labels = append(labels, prompbmarshal.Label{Name: alertStateLabel, Value: a.State.String()})
 	}
 	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
 // alertForToTimeSeries returns a time series that represents
 // state of active alerts, where value is time when alert become active
-func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompb.TimeSeries {
-	labels := make([]prompb.Label, 0, len(a.Labels)+1)
+func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+	labels := make([]prompbmarshal.Label, 0, len(a.Labels)+1)
 	for k, v := range a.Labels {
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  k,
 			Value: v,
 		})
 	}
 	// __name__ already been dropped, no need to check duplication
-	labels = append(labels, prompb.Label{Name: "__name__", Value: alertForStateMetricName})
+	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
 	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
 // pendingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
 // for alerts which changed their state from Pending to Inactive or Firing.
-func pendingAlertStaleTimeSeries(ls map[string]string, timestamp int64, includeAlertForState bool) []prompb.TimeSeries {
-	var result []prompb.TimeSeries
-	baseLabels := make([]prompb.Label, 0, len(ls)+1)
+func pendingAlertStaleTimeSeries(ls map[string]string, timestamp int64, includeAlertForState bool) []prompbmarshal.TimeSeries {
+	var result []prompbmarshal.TimeSeries
+	baseLabels := make([]prompbmarshal.Label, 0, len(ls)+1)
 	for k, v := range ls {
-		baseLabels = append(baseLabels, prompb.Label{
+		baseLabels = append(baseLabels, prompbmarshal.Label{
 			Name:  k,
 			Value: v,
 		})
 	}
 
-	alertsLabels := make([]prompb.Label, 0, len(ls)+2)
+	alertsLabels := make([]prompbmarshal.Label, 0, len(ls)+2)
 	alertsLabels = append(alertsLabels, baseLabels...)
 	// __name__ already been dropped, no need to check duplication
-	alertsLabels = append(alertsLabels, prompb.Label{Name: "__name__", Value: alertMetricName})
-	alertsLabels = append(alertsLabels, prompb.Label{Name: alertStateLabel, Value: notifier.StatePending.String()})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StatePending.String()})
 	result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels))
 
 	if includeAlertForState {
-		baseLabels = append(baseLabels, prompb.Label{Name: "__name__", Value: alertForStateMetricName})
+		baseLabels = append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
 		result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, baseLabels))
 	}
 	return result
@@ -766,24 +700,24 @@ func pendingAlertStaleTimeSeries(ls map[string]string, timestamp int64, includeA
 
 // firingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
 // for alerts which changed their state from Firing to Inactive.
-func firingAlertStaleTimeSeries(ls map[string]string, timestamp int64) []prompb.TimeSeries {
-	baseLabels := make([]prompb.Label, 0, len(ls)+1)
+func firingAlertStaleTimeSeries(ls map[string]string, timestamp int64) []prompbmarshal.TimeSeries {
+	baseLabels := make([]prompbmarshal.Label, 0, len(ls)+1)
 	for k, v := range ls {
-		baseLabels = append(baseLabels, prompb.Label{
+		baseLabels = append(baseLabels, prompbmarshal.Label{
 			Name:  k,
 			Value: v,
 		})
 	}
 
-	alertsLabels := make([]prompb.Label, 0, len(ls)+2)
+	alertsLabels := make([]prompbmarshal.Label, 0, len(ls)+2)
 	alertsLabels = append(alertsLabels, baseLabels...)
 	// __name__ already been dropped, no need to check duplication
-	alertsLabels = append(alertsLabels, prompb.Label{Name: "__name__", Value: alertMetricName})
-	alertsLabels = append(alertsLabels, prompb.Label{Name: alertStateLabel, Value: notifier.StateFiring.String()})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StateFiring.String()})
 
-	baseLabels = append(baseLabels, prompb.Label{Name: "__name__", Value: alertForStateMetricName})
+	baseLabels = append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
 
-	return []prompb.TimeSeries{
+	return []prompbmarshal.TimeSeries{
 		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels),
 		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, baseLabels),
 	}

app/vmalert/rule/alerting_test.go

@@ -6,11 +6,9 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"testing"
-	"testing/synctest"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -20,7 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 )
 
@@ -53,7 +51,7 @@ func TestNewAlertingRule(t *testing.T) {
 func TestAlertingRuleToTimeSeries(t *testing.T) {
 	timestamp := time.Now()
 
-	f := func(rule *AlertingRule, alert *notifier.Alert, tssExpected []prompb.TimeSeries) {
+	f := func(rule *AlertingRule, alert *notifier.Alert, tssExpected []prompbmarshal.TimeSeries) {
 		t.Helper()
 
 		rule.alerts[alert.ID] = alert
@@ -66,8 +64,8 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 	f(newTestAlertingRule("instant", 0), &notifier.Alert{
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
-	}, []prompb.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompb.Label{
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
 			{
 				Name:  "__name__",
 				Value: alertMetricName,
@@ -79,7 +77,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompb.Label{
+			[]prompbmarshal.Label{
 				{
 					Name:  "__name__",
 					Value: alertForStateMetricName,
@@ -93,9 +91,9 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 			"job":      "foo",
 			"instance": "bar",
 		},
-	}, []prompb.TimeSeries{
+	}, []prompbmarshal.TimeSeries{
 		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()},
-			[]prompb.Label{
+			[]prompbmarshal.Label{
 				{
 					Name:  "__name__",
 					Value: alertMetricName,
@@ -115,7 +113,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 			}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompb.Label{
+			[]prompbmarshal.Label{
 				{
 					Name:  "__name__",
 					Value: alertForStateMetricName,
@@ -136,8 +134,8 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		Labels: map[string]string{
 			alertStateLabel: "foo",
 		},
-	}, []prompb.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompb.Label{
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
 			{
 				Name:  "__name__",
 				Value: alertMetricName,
@@ -149,7 +147,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompb.Label{
+			[]prompbmarshal.Label{
 				{
 					Name:  "__name__",
 					Value: alertForStateMetricName,
@@ -164,8 +162,8 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 	f(newTestAlertingRule("for", time.Second), &notifier.Alert{
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
-	}, []prompb.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompb.Label{
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
 			{
 				Name:  "__name__",
 				Value: alertMetricName,
@@ -177,7 +175,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompb.Label{
+			[]prompbmarshal.Label{
 				{
 					Name:  "__name__",
 					Value: alertForStateMetricName,
@@ -188,8 +186,8 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 	f(newTestAlertingRule("for pending", 10*time.Second), &notifier.Alert{
 		State:    notifier.StatePending,
 		ActiveAt: timestamp.Add(time.Second),
-	}, []prompb.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompb.Label{
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
 			{
 				Name:  "__name__",
 				Value: alertMetricName,
@@ -199,7 +197,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 				Value: notifier.StatePending.String(),
 			},
 		}),
-		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, []prompb.Label{
+		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
 			{
 				Name:  "__name__",
 				Value: alertForStateMetricName,
@@ -217,7 +215,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 
 	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
 
-	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert, tssExpected map[int][]prompb.TimeSeries) {
+	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert, tssExpected map[int][]prompbmarshal.TimeSeries) {
 		t.Helper()
 
 		fq := &datasource.FakeQuerier{}
@@ -269,15 +267,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 				if got.State != exp.State {
 					t.Fatalf("evalIndex %d: expected state %d; got %d", i, exp.State, got.State)
 				}
-				if rule.Annotations != nil && exp.Annotations != nil {
-					if !reflect.DeepEqual(got.Annotations, exp.Annotations) {
-						t.Fatalf("evalIndex %d: expected annotations %v; got %v", i, exp.Annotations, got.Annotations)
-					}
-				}
 			}
 		}
-		// reset ts for next test
-		ts, _ = time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
 	}
 
 	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil, nil)
@@ -287,15 +278,15 @@ func TestAlertingRule_Exec(t *testing.T) {
 	}, map[int][]testAlert{
 		0: {{alert: &notifier.Alert{State: notifier.StateFiring}}},
 	},
-		map[int][]prompb.TimeSeries{
+		map[int][]prompbmarshal.TimeSeries{
 			0: {
 				{
-					Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "empty_labels"}, {Name: "alertstate", Value: "firing"}},
-					Samples: []prompb.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
+					Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "empty_labels"}, {Name: "alertstate", Value: "firing"}},
+					Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
 				},
 				{
-					Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "empty_labels"}},
-					Samples: []prompb.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
+					Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "empty_labels"}},
+					Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
 				},
 			},
 		})
@@ -312,35 +303,35 @@ func TestAlertingRule_Exec(t *testing.T) {
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
-	}, map[int][]prompb.TimeSeries{
+	}, map[int][]prompbmarshal.TimeSeries{
 		0: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
 			},
 		},
 		1: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 		},
 		2: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 		},
 	})
@@ -395,55 +386,55 @@ func TestAlertingRule_Exec(t *testing.T) {
 			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 		},
-	}, map[int][]prompb.TimeSeries{
+	}, map[int][]prompbmarshal.TimeSeries{
 		0: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
 			},
 		},
 		1: {
 			// stale time series for foo, `firing -> inactive`
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 			// new time series for foo1
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 		},
 		2: {
 			// stale time series for foo1
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 			// new time series for foo2
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo2"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo2"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo2"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo2"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 		},
 	})
@@ -460,30 +451,30 @@ func TestAlertingRule_Exec(t *testing.T) {
 	}, map[int][]testAlert{
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	}, map[int][]prompb.TimeSeries{
+	}, map[int][]prompbmarshal.TimeSeries{
 		0: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
 			},
 		},
 		1: {
 			// stale time series for `pending -> firing`
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 		},
 	})
@@ -497,41 +488,41 @@ func TestAlertingRule_Exec(t *testing.T) {
 		0: {{labels: []string{"name", "foo", "a1", "b1", "a2", "b2", "a3", "b3"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo", "a1", "b1", "a2", "b2", "a3", "b3"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		2: {},
-	}, map[int][]prompb.TimeSeries{
+	}, map[int][]prompbmarshal.TimeSeries{
 		0: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}},
 			},
 		},
 		1: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: float64(ts.Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}},
 			},
 		},
 		// stale time series for `pending -> inactive`
 		2: {
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 			{
-				Labels:  []prompb.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
-				Samples: []prompb.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
+				Labels:  []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "a1", Value: "b1"}, {Name: "a2", Value: "b2"}, {Name: "a3", Value: "b3"}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}},
 			},
 		},
 	})
 
-	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>inactive=>pending=>firing", defaultStep, 0, 0, map[string]string{"activeAt": "{{ $activeAt.UnixMilli }}"}), [][]datasource.Metric{
+	f(newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
 		{metricWithLabels(t, "name", "foo")},
 		// empty step to set alert inactive
@@ -539,11 +530,11 @@ func TestAlertingRule_Exec(t *testing.T) {
 		{metricWithLabels(t, "name", "foo")},
 		{metricWithLabels(t, "name", "foo")},
 	}, map[int][]testAlert{
-		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.UnixMilli(), 10)}}}},
-		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.UnixMilli(), 10)}}}},
-		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.UnixMilli(), 10)}}}},
-		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.Add(defaultStep*3).UnixMilli(), 10)}}}},
-		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring, Annotations: map[string]string{"activeAt": strconv.FormatInt(ts.Add(defaultStep*3).UnixMilli(), 10)}}}},
+		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 	}, nil)
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>firing", defaultStep, 0, defaultStep, nil), [][]datasource.Metric{
@@ -597,7 +588,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		var expTS []prompb.TimeSeries
+		var expTS []prompbmarshal.TimeSeries
 		var j int
 		for _, series := range data {
 			for _, timestamp := range series.Timestamps {
@@ -737,7 +728,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []prompb.Label{{Name: "foo", Value: "bar"}},
+			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
@@ -785,7 +776,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []prompb.Label{{Name: "foo", Value: "bar"}},
+			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{
@@ -827,9 +818,12 @@ func TestGroup_Restore(t *testing.T) {
 		fg := NewGroup(config.Group{Name: "TestRestore", Rules: rules}, fqr, time.Second, nil)
 		fg.Init()
 		wg := sync.WaitGroup{}
-		wg.Go(func() {
-			fg.Start(context.Background(), nil, fqr)
-		})
+		wg.Add(1)
+		go func() {
+			nts := func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} }
+			fg.Start(context.Background(), nts, nil, fqr)
+			wg.Done()
+		}()
 		fg.Close()
 		wg.Wait()
 
@@ -1359,7 +1353,7 @@ func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, kee
 
 func TestAlertingRule_ToLabels(t *testing.T) {
 	metric := datasource.Metric{
-		Labels: []prompb.Label{
+		Labels: []prompbmarshal.Label{
 			{Name: "instance", Value: "0.0.0.0:8800"},
 			{Name: "group", Value: "vmalert"},
 			{Name: "alertname", Value: "ConfigurationReloadFailure"},
@@ -1370,10 +1364,8 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 
 	ar := &AlertingRule{
 		Labels: map[string]string{
-			"instance":      "override", // this should override instance with new value
-			"group":         "vmalert",  // this shouldn't have effect since value in metric is equal
-			"invalid_label": "{{ .Values.mustRuntimeFail }}",
-			"empty_label":   "", // this should be dropped
+			"instance": "override", // this should override instance with new value
+			"group":    "vmalert",  // this shouldn't have effect since value in metric is equal
 		},
 		Expr:      "sum(vmalert_alerting_rules_error) by(instance, group, alertname) > 0",
 		Name:      "AlertingRulesError",
@@ -1381,11 +1373,10 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 	}
 
 	expectedOriginLabels := map[string]string{
-		"instance":      "0.0.0.0:8800",
-		"group":         "vmalert",
-		"alertname":     "ConfigurationReloadFailure",
-		"alertgroup":    "vmalert",
-		"invalid_label": `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"instance":   "0.0.0.0:8800",
+		"group":      "vmalert",
+		"alertname":  "ConfigurationReloadFailure",
+		"alertgroup": "vmalert",
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1395,12 +1386,11 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)
-	if err == nil || !strings.Contains(err.Error(), "error evaluating template") {
-		t.Fatalf("unexpected error %q", err.Error())
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
 	}
 
 	if !reflect.DeepEqual(ls.origin, expectedOriginLabels) {
@@ -1431,142 +1421,3 @@ func TestAlertingRuleExec_Partial(t *testing.T) {
 		t.Fatalf("unexpected error: %s", err)
 	}
 }
-
-func TestAlertingRule_QueryTemplateInLabels(t *testing.T) {
-	fq := &datasource.FakeQuerier{}
-	fakeGroup := Group{
-		Name: "TestQueryTemplateInLabels",
-	}
-
-	ar := &AlertingRule{
-		Name: "test_alert",
-		Labels: map[string]string{
-			"suppress_for_mass_alert": `{{ if (printf "ALERTS{alertname='SomeAlert', alertstate='firing', device='%s'} == 1" $labels.device | query) }}true{{ else }}false{{ end }}`,
-		},
-		Annotations: map[string]string{
-			"summary": "Test alert with query template in labels",
-		},
-		alerts: make(map[uint64]*notifier.Alert),
-	}
-	ar.GroupID = fakeGroup.GetID()
-	ar.q = fq
-	ar.state = &ruleState{
-		entries: make([]StateEntry, 10),
-	}
-
-	// Add a metric that should trigger the alert
-	fq.Add(metricWithValueAndLabels(t, 1, "device", "sda1"))
-
-	ts := time.Now()
-	_, err := ar.exec(context.TODO(), ts, 0)
-	if err != nil {
-		t.Fatalf("unexpected error with query template in labels: %s", err)
-	}
-
-	// Verify that the alert was created and the query template was executed
-	if len(ar.alerts) != 1 {
-		t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
-	}
-
-	alert := ar.GetAlerts()[0]
-	suppressLabel, exists := alert.Labels["suppress_for_mass_alert"]
-	if !exists {
-		t.Fatalf("expected 'suppress_for_mass_alert' label to exist")
-	}
-	// The query template should have been executed (even if it returns false due to mock data)
-	if suppressLabel != "true" && suppressLabel != "false" {
-		t.Fatalf("expected 'suppress_for_mass_alert' label to be 'true' or 'false', got '%s'", suppressLabel)
-	}
-}
-
-// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
-// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
-// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
-func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
-	// wrap into synctest because of time manipulations
-	synctest.Test(t, func(t *testing.T) {
-		fq := &datasource.FakeQuerier{}
-
-		ar := &AlertingRule{
-			Name: "TestActiveAtPreservation",
-			Labels: map[string]string{
-				"test_query_in_label": `{{ "static_value" }}`,
-			},
-			Annotations: map[string]string{
-				"description": "Alert active since {{ $activeAt }}",
-			},
-			alerts: make(map[uint64]*notifier.Alert),
-			q:      fq,
-			state: &ruleState{
-				entries: make([]StateEntry, 10),
-			},
-		}
-
-		// Mock query result - return empty result to make suppress_for_mass_alert = false
-		// (no need to add anything to fq for empty result)
-
-		// Add a metric that should trigger the alert
-		fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
-
-		// First execution - creates new alert
-		ts1 := time.Now()
-		_, err := ar.exec(context.TODO(), ts1, 0)
-		if err != nil {
-			t.Fatalf("unexpected error on first exec: %s", err)
-		}
-
-		if len(ar.alerts) != 1 {
-			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
-		}
-
-		firstAlert := ar.GetAlerts()[0]
-		// Verify first execution: activeAt should be ts1 and annotation should reflect it
-		if !firstAlert.ActiveAt.Equal(ts1) {
-			t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
-		}
-
-		// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
-		expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
-		if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
-			t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
-		}
-
-		// Second execution - should preserve activeAt in annotation
-
-		// Ensure different timestamp with different seconds
-		// sleep is non-blocking thanks to synctest
-		time.Sleep(2 * time.Second)
-		ts2 := time.Now()
-		_, err = ar.exec(context.TODO(), ts2, 0)
-		if err != nil {
-			t.Fatalf("unexpected error on second exec: %s", err)
-		}
-
-		// Get the alert again (should be the same alert)
-		if len(ar.alerts) != 1 {
-			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
-		}
-		secondAlert := ar.GetAlerts()[0]
-
-		// Critical test: activeAt should still be ts1, not ts2
-		if !secondAlert.ActiveAt.Equal(ts1) {
-			t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
-		}
-
-		// Critical test: annotation should still contain ts1 time, not ts2
-		if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
-			t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
-		}
-
-		// Additional verification: annotation should NOT contain ts2 time
-		ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
-		if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
-			t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
-		}
-
-		// Verify query template in labels still works (this would fail if query templates were broken)
-		if firstAlert.Labels["test_query_in_label"] != "static_value" {
-			t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
-		}
-	})
-}

app/vmalert/rule/group.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -18,15 +19,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var (
-	ruleResultsLimit = flag.Int("rule.resultsLimit", 0, "Limits the number of alerts or recording results a single rule can produce. "+
-		"Can be overridden by the limit option under group if specified. "+
-		"If exceeded, the rule will be marked with an error and all its results will be discarded. "+
-		"0 means no limit.")
 	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
 		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
 	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
@@ -38,8 +36,6 @@ var (
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
 	remoteReadLookBack     = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries. "+
 		"For example, if lookback=1h then range from now() to now()-1h will be scanned.")
-	maxStartDelay = flag.Duration("group.maxStartDelay", 5*time.Minute, "Defines the max delay before starting the group evaluation. Group's start is artificially delayed for random duration on interval"+
-		" [0..min(--group.maxStartDelay, group.interval)]. This helps smoothing out the load on the configured datasource, so evaluations aren't executed too close to each other.")
 )
 
 // Group is an entity for grouping rules
@@ -116,6 +112,7 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Name:            cfg.Name,
 		File:            cfg.File,
 		Interval:        cfg.Interval.Duration(),
+		Limit:           cfg.Limit,
 		Concurrency:     cfg.Concurrency,
 		checksum:        cfg.Checksum,
 		Params:          cfg.Params,
@@ -132,11 +129,6 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 	if g.Interval == 0 {
 		g.Interval = defaultInterval
 	}
-	if cfg.Limit != nil {
-		g.Limit = *cfg.Limit
-	} else {
-		g.Limit = *ruleResultsLimit
-	}
 	if g.Concurrency < 1 {
 		g.Concurrency = 1
 	}
@@ -297,7 +289,7 @@ func (g *Group) InterruptEval() {
 	}
 }
 
-// Close stops the group and its rules, unregisters group metrics
+// Close stops the group and it's rules, unregisters group metrics
 func (g *Group) Close() {
 	if g.doneCh == nil {
 		return
@@ -306,6 +298,10 @@ func (g *Group) Close() {
 	g.InterruptEval()
 	<-g.finishedCh
 
+	g.closeGroupMetrics()
+}
+
+func (g *Group) closeGroupMetrics() {
 	metrics.UnregisterSet(g.metrics.set, true)
 }
 
@@ -331,13 +327,13 @@ func (g *Group) Init() {
 }
 
 // Start starts group's evaluation
-func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasource.QuerierBuilder) {
+func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 	evalTS := time.Now()
 	// sleep random duration to spread group rules evaluation
-	// over maxStartDelay to reduce the load on datasource.
+	// over time in order to reduce load on datasource.
 	if !SkipRandSleepOnGroupStart {
-		sleepBeforeStart := g.delayBeforeStart(evalTS, *maxStartDelay)
+		sleepBeforeStart := delayBeforeStart(evalTS, g.GetID(), g.Interval, g.EvalOffset)
 		g.infof("will start in %v", sleepBeforeStart)
 
 		sleepTimer := time.NewTimer(sleepBeforeStart)
@@ -369,6 +365,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 
 	e := &executor{
 		Rw:              rw,
+		Notifiers:       nts,
 		notifierHeaders: g.NotifierHeaders,
 	}
 
@@ -475,31 +472,32 @@ func (g *Group) UpdateWith(newGroup *Group) {
 	g.updateCh <- newGroup
 }
 
-// delayBeforeStart returns duration for delaying the evaluation start
-// based on given ts and Group settings. The delay can't exceed maxDelay.
-// maxDelay is ignored if g.EvalOffset != nil.
-//
-// Delaying is important to smooth out the load on the datasource when all groups start at the same time.
-// delayBeforeStart calculates delay based on Group ID, so all groups will start at different moments of time.
-func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Duration {
-	if g.EvalOffset != nil {
-		// if offset is specified, ignore the maxDelay and return a duration aligned with offset
-		currentOffsetPoint := ts.Truncate(g.Interval).Add(*g.EvalOffset)
+// DeepCopy returns a deep copy of group
+func (g *Group) DeepCopy() *Group {
+	g.mu.RLock()
+	data, _ := json.Marshal(g)
+	g.mu.RUnlock()
+	newG := Group{}
+	_ = json.Unmarshal(data, &newG)
+	newG.Rules = g.Rules
+	newG.id = g.id
+	return &newG
+}
+
+// if offset is specified, delayBeforeStart returns a duration to help aligning timestamp with offset;
+// otherwise, it returns a random duration between [0..interval] based on group key.
+func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *time.Duration) time.Duration {
+	if offset != nil {
+		currentOffsetPoint := ts.Truncate(interval).Add(*offset)
 		if currentOffsetPoint.Before(ts) {
 			// wait until the next offset point
-			return currentOffsetPoint.Add(g.Interval).Sub(ts)
+			return currentOffsetPoint.Add(interval).Sub(ts)
 		}
 		return currentOffsetPoint.Sub(ts)
 	}
 
-	// otherwise, return a random duration between [0..min(interval, maxDelay)] based on group ID
-	interval := g.Interval
-	if interval > maxDelay {
-		// artificially limit interval, so groups with big intervals could start sooner.
-		interval = maxDelay
-	}
 	var randSleep time.Duration
-	randSleep = time.Duration(float64(interval) * (float64(g.GetID()) / (1 << 64)))
+	randSleep = time.Duration(float64(interval) * (float64(key) / (1 << 64)))
 	sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())
 	if randSleep < sleepOffset {
 		randSleep += interval
@@ -561,13 +559,15 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 	if !disableProgressBar {
 		bar = pb.StartNew(iterations * len(g.Rules))
 	}
-	for i := range g.Rules {
-		rule := g.Rules[i]
+	for _, r := range g.Rules {
 		sem <- struct{}{}
-		wg.Go(func() {
-			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+		wg.Add(1)
+		go func(r Rule, ri rangeIterator) {
+			// pass ri as a copy, so it can be modified within the replayRuleRange
+			res <- replayRuleRange(r, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
 			<-sem
-		})
+			wg.Done()
+		}(r, ri)
 	}
 
 	wg.Wait()
@@ -587,20 +587,15 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 
 func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
 	fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
-	// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
-	if r, ok := r.(*AlertingRule); ok && r.For > 0 {
-		ruleEvaluationConcurrency = 1
-	}
 	sem := make(chan struct{}, ruleEvaluationConcurrency)
 	wg := sync.WaitGroup{}
 	res := make(chan int, int(ri.end.Sub(ri.start)/ri.step)+1)
 	for ri.next() {
 		sem <- struct{}{}
-		start := ri.s
-		end := ri.e
-		wg.Go(func() {
-			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
+		wg.Add(1)
+
+		go func(s, e time.Time) {
+			n, err := replayRule(r, s, e, rw, replayRuleRetryAttempts)
 			if err != nil {
 				logger.Fatalf("rule %q: %s", r, err)
 			}
@@ -609,7 +604,8 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 			}
 			res <- n
 			<-sem
-		})
+			wg.Done()
+		}(ri.s, ri.e)
 	}
 	wg.Wait()
 	close(res)
@@ -623,9 +619,10 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 }
 
 // ExecOnce evaluates all the rules under group for once with given timestamp.
-func (g *Group) ExecOnce(ctx context.Context, rw remotewrite.RWClient, evalTS time.Time) chan error {
+func (g *Group) ExecOnce(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, evalTS time.Time) chan error {
 	e := &executor{
 		Rw:              rw,
+		Notifiers:       nts,
 		notifierHeaders: g.NotifierHeaders,
 	}
 	if len(g.Rules) < 1 {
@@ -700,6 +697,7 @@ func (g *Group) getEvalDelay() time.Duration {
 
 // executor contains group's notify and rw configs
 type executor struct {
+	Notifiers       func() []notifier.Notifier
 	notifierHeaders map[string]string
 
 	Rw remotewrite.RWClient
@@ -720,13 +718,14 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.T
 	sem := make(chan struct{}, concurrency)
 	go func() {
 		wg := sync.WaitGroup{}
-		for i := range rules {
-			rule := rules[i]
+		for _, r := range rules {
 			sem <- struct{}{}
-			wg.Go(func() {
-				res <- e.exec(ctx, rule, ts, resolveDuration, limit)
+			wg.Add(1)
+			go func(r Rule) {
+				res <- e.exec(ctx, r, ts, resolveDuration, limit)
 				<-sem
-			})
+				wg.Done()
+			}(r)
 		}
 		wg.Wait()
 		close(res)
@@ -756,7 +755,7 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 	}
 
 	if e.Rw != nil {
-		pushToRW := func(tss []prompb.TimeSeries) error {
+		pushToRW := func(tss []prompbmarshal.TimeSeries) error {
 			var lastErr error
 			for _, ts := range tss {
 				if err := e.Rw.Push(ts); err != nil {
@@ -780,6 +779,17 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		return nil
 	}
 
-	errGr := notifier.Send(ctx, alerts, e.notifierHeaders)
+	wg := sync.WaitGroup{}
+	errGr := new(vmalertutil.ErrGroup)
+	for _, nt := range e.Notifiers() {
+		wg.Add(1)
+		go func(nt notifier.Notifier) {
+			if err := nt.Send(ctx, alerts, e.notifierHeaders); err != nil {
+				errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", r, nt.Addr(), err))
+			}
+			wg.Done()
+		}(nt)
+	}
+	wg.Wait()
 	return errGr.Err()
 }

app/vmalert/rule/group_test.go

@@ -262,7 +262,7 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 		updateCh: make(chan *Group),
 	}
 	g.Init()
-	go g.Start(context.Background(), nil, nil)
+	go g.Start(context.Background(), nil, nil, nil)
 
 	rule1 := AlertingRule{
 		Name: "jobDown",
@@ -346,8 +346,7 @@ func TestGroupStart(t *testing.T) {
 	}
 
 	fs := &datasource.FakeQuerier{}
-	fn, cleanup := notifier.InitFakeNotifier()
-	defer cleanup()
+	fn := &notifier.FakeNotifier{}
 
 	const evalInterval = time.Millisecond
 	g := NewGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
@@ -396,7 +395,7 @@ func TestGroupStart(t *testing.T) {
 	fs.Add(m2)
 	g.Init()
 	go func() {
-		g.Start(context.Background(), nil, fs)
+		g.Start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil, fs)
 		close(finished)
 	}()
 
@@ -473,10 +472,15 @@ func TestFaultyNotifier(t *testing.T) {
 	r := newTestAlertingRule("instant", 0)
 	r.q = fq
 
-	fn, cleanup := notifier.InitFakeNotifier()
-	defer cleanup()
-
-	e := &executor{}
+	fn := &notifier.FakeNotifier{}
+	e := &executor{
+		Notifiers: func() []notifier.Notifier {
+			return []notifier.Notifier{
+				&notifier.FaultyNotifier{},
+				fn,
+			}
+		},
+	}
 	delay := 5 * time.Second
 	ctx, cancel := context.WithTimeout(context.Background(), delay)
 	defer cancel()
@@ -549,7 +553,7 @@ func TestCloseWithEvalInterruption(t *testing.T) {
 	g := NewGroup(groups[0], fq, evalInterval, nil)
 	g.Init()
 
-	go g.Start(context.Background(), nil, nil)
+	go g.Start(context.Background(), nil, nil, nil)
 
 	time.Sleep(evalInterval * 20)
 
@@ -567,10 +571,9 @@ func TestCloseWithEvalInterruption(t *testing.T) {
 
 func TestGroupStartDelay(t *testing.T) {
 	g := &Group{}
-	g.id = uint64(math.MaxUint64 / 10)
 	// interval of 5min and key generate a static delay of 30s
 	g.Interval = time.Minute * 5
-	maxDelay := time.Minute * 5
+	key := uint64(math.MaxUint64 / 10)
 
 	f := func(atS, expS string) {
 		t.Helper()
@@ -582,7 +585,7 @@ func TestGroupStartDelay(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		delay := g.delayBeforeStart(at, maxDelay)
+		delay := delayBeforeStart(at, key, g.Interval, g.EvalOffset)
 		gotStart := at.Add(delay)
 		if expTS != gotStart {
 			t.Fatalf("expected to get %v; got %v instead", expTS, gotStart)
@@ -603,15 +606,6 @@ func TestGroupStartDelay(t *testing.T) {
 	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
 	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
 	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
-
-	maxDelay = time.Minute * 1
-	g.EvalOffset = nil
-
-	// test group with maxDelay, and offset disabled
-	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:00:06.000+00:00")
-	f("2023-01-01T00:00:01.000+00:00", "2023-01-01T00:00:06.000+00:00")
-	f("2023-01-01T00:00:06.100+00:00", "2023-01-01T00:01:06.000+00:00")
-	f("2023-01-01T00:00:11.000+00:00", "2023-01-01T00:01:06.000+00:00")
 }
 
 func TestGetPrometheusReqTimestamp(t *testing.T) {