cut v0.4.1-victorialogs release

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -6,7 +6,7 @@ body:
     attributes:
       value: |
         Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
-        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
         and verify whether the bug is reproducible there.
         It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
   - type: textarea
@@ -60,8 +60,8 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -1,35 +0,0 @@
-### Describe Your Changes
-
-Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications.
-
-### Checklist
-
-The following checks are mandatory:
-
-- [ ] I have read the [Contributing Guidelines](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/CONTRIBUTING.md)
-- [ ] All commits are signed and include `Signed-off-by` line. Use `git commit -s` to include `Signed-off-by` your commits. See this [doc](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) about how to sign your commits.
-- [ ] Tests are passing locally. Use `make test` to run all tests locally.
-- [ ] Linting is passing locally. Use `make check-all` to run all linters locally.
-
-Further checks are optional for External Contributions:
-
-- [ ] Include a link to the GitHub issue in the commit message, if issue exists.
-- [ ] Mention the change in the [Changelog](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md). Explain what has changed and why. If there is a related issue or documentation change - link them as well.
-
-  Tips for writing a good changelog message::
-
-    * Write a human-readable changelog message that describes the problem and solution.
-    * Include a link to the issue or pull request in your changelog message.
-    * Use specific language identifying the fix, such as an error message, metric name, or flag name.
-    * Provide a link to the relevant documentation for any new features you add or modify.
-
-- [ ] After your pull request is merged, please add a message to the issue with instructions for how to test the fix or try the feature you added. Here is an [example](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048#issuecomment-1546453726)
-- [ ] Do not close the original issue before the change is released. Please note, in some cases Github can automatically close the issue once PR is merged. Re-open the issue in such case.
-- [ ] If the change somehow affects public interfaces (a new flag was added or updated, or some behavior has changed) - add the corresponding change to documentation.
-
-
-Examples of good changelog messages:
-
-1. FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol) when [sending / receiving data to / from Kafka](https://docs.victoriametrics.com/vmagent.html#kafka-integration). This protocol allows saving egress network bandwidth costs when sending data from `vmagent` to `Kafka` located in another datacenter or availability zone. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1225).
-
-2. BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmgent.html) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).

.github/workflows/check-licenses.yml

@@ -14,25 +14,13 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
+      - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.21.1
+        id: go
       - name: Code checkout
         uses: actions/checkout@master
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
-
       - name: Check License
-        run: make check-licenses
+        run: |
+          make check-licenses

.github/workflows/codeql-analysis-js.yml

@@ -36,11 +36,11 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v2
         with:
           category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -55,27 +55,16 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v4
         with:
-          go-version: stable
-          cache: false
-        if: ${{ matrix.language == 'go' }}
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
+          go-version: 1.21.1
+          check-latest: true
+          cache: true
         if: ${{ matrix.language == 'go' }}
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -86,7 +75,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+        uses: github/codeql-action/autobuild@v2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -100,4 +89,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -7,8 +7,6 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
   pull_request:
     branches:
       - master
@@ -16,8 +14,6 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
 permissions:
   contents: read
 
@@ -34,55 +30,18 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v4
         with:
-          go-version: stable
-          cache: false
+          go-version: 1.21.1
+          check-latest: true
+          cache: true
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-all-
-
-      - name: Run check-all
+      - name: Dependencies
         run: |
+          make install-golangci-lint
           make check-all
           git diff --exit-code
 
-  build:
-    needs: lint
-    name: build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
-      - name: Build
-        run: make crossbuild
-
   test:
     needs: lint
     strategy:
@@ -95,26 +54,43 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v4
         with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
+          go-version: 1.21.1
+          check-latest: true
+          cache: true
 
       - name: run tests
-        run: make ${{ matrix.scenario}}
+        run: |
+          make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt
+
+  build:
+    needs: test
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21.1
+          check-latest: true
+          cache: true
+
+      - uses: actions/cache@v3
+        with:
+          path: gocache-for-docker
+          key:  gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
+
+      - name: Build
+        run: |
+          make victoria-metrics-crossbuild
+          make vmuitils-crossbuild

.github/workflows/sync-docs.yml

@@ -24,6 +24,7 @@ jobs:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: docs
+
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v5
         with:
@@ -38,6 +39,7 @@ jobs:
           calculatedSha=$(git rev-parse --short ${{ github.sha }})
           echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
         working-directory: main
+
       - name: update code and commit
         run: |
           rm -rf content

.github/workflows/wiki.yml

@@ -0,0 +1,33 @@
+name: wiki
+on:
+  push:
+    paths:
+      - 'docs/*'
+    branches:
+      - master
+permissions:
+  contents: read
+
+jobs:
+  build:
+    permissions:
+      contents: write  # for Git to git push
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
+          cp -r docs/* wiki
+          cd wiki
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add .
+          git commit -m "update wiki pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf wiki

.gitignore

@@ -22,4 +22,3 @@ Gemfile.lock
 /_site
 _site
 *.tmp
-/docs/.jekyll-metadata

CONTRIBUTING.md

@@ -14,8 +14,3 @@ We are open to third-party pull requests provided they follow [KISS design princ
 - Avoid automated decisions, which may hurt cluster availability, consistency or performance.
 
 Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
-
-Before sending a pull request please check the following:
-- [ ] All commits are signed and include `Signed-off-by` line. Use `git commit -s` to include `Signed-off-by` your commits. See this [doc](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) about how to sign your commits.
-- [ ] Tests are passing locally. Use `make test` to run all tests locally.
-- [ ] Linting is passing locally. Use `make check-all` to run all linters locally.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2024 VictoriaMetrics, Inc.
+   Copyright 2019-2023 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,7 +1,5 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
-MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
@@ -17,7 +15,6 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
 include snap/local/Makefile
@@ -28,7 +25,6 @@ all: \
 	victoria-logs-prod \
 	vmagent-prod \
 	vmalert-prod \
-	vmalert-tool-prod \
 	vmauth-prod \
 	vmbackup-prod \
 	vmrestore-prod \
@@ -37,11 +33,10 @@ all: \
 clean:
 	rm -rf bin/*
 
-publish: \
+publish: package-base \
 	publish-victoria-metrics \
 	publish-vmagent \
 	publish-vmalert \
-	publish-vmalert-tool \
 	publish-vmauth \
 	publish-vmbackup \
 	publish-vmrestore \
@@ -52,7 +47,6 @@ package: \
 	package-victoria-logs \
 	package-vmagent \
 	package-vmalert \
-	package-vmalert-tool \
 	package-vmauth \
 	package-vmbackup \
 	package-vmrestore \
@@ -61,7 +55,6 @@ package: \
 vmutils: \
 	vmagent \
 	vmalert \
-	vmalert-tool \
 	vmauth \
 	vmbackup \
 	vmrestore \
@@ -70,7 +63,6 @@ vmutils: \
 vmutils-pure: \
 	vmagent-pure \
 	vmalert-pure \
-	vmalert-tool-pure \
 	vmauth-pure \
 	vmbackup-pure \
 	vmrestore-pure \
@@ -79,7 +71,6 @@ vmutils-pure: \
 vmutils-linux-amd64: \
 	vmagent-linux-amd64 \
 	vmalert-linux-amd64 \
-	vmalert-tool-linux-amd64 \
 	vmauth-linux-amd64 \
 	vmbackup-linux-amd64 \
 	vmrestore-linux-amd64 \
@@ -88,7 +79,6 @@ vmutils-linux-amd64: \
 vmutils-linux-arm64: \
 	vmagent-linux-arm64 \
 	vmalert-linux-arm64 \
-	vmalert-tool-linux-arm64 \
 	vmauth-linux-arm64 \
 	vmbackup-linux-arm64 \
 	vmrestore-linux-arm64 \
@@ -97,7 +87,6 @@ vmutils-linux-arm64: \
 vmutils-linux-arm: \
 	vmagent-linux-arm \
 	vmalert-linux-arm \
-	vmalert-tool-linux-arm \
 	vmauth-linux-arm \
 	vmbackup-linux-arm \
 	vmrestore-linux-arm \
@@ -106,7 +95,6 @@ vmutils-linux-arm: \
 vmutils-linux-386: \
 	vmagent-linux-386 \
 	vmalert-linux-386 \
-	vmalert-tool-linux-386 \
 	vmauth-linux-386 \
 	vmbackup-linux-386 \
 	vmrestore-linux-386 \
@@ -115,7 +103,6 @@ vmutils-linux-386: \
 vmutils-linux-ppc64le: \
 	vmagent-linux-ppc64le \
 	vmalert-linux-ppc64le \
-	vmalert-tool-linux-ppc64le \
 	vmauth-linux-ppc64le \
 	vmbackup-linux-ppc64le \
 	vmrestore-linux-ppc64le \
@@ -124,7 +111,6 @@ vmutils-linux-ppc64le: \
 vmutils-darwin-amd64: \
 	vmagent-darwin-amd64 \
 	vmalert-darwin-amd64 \
-	vmalert-tool-darwin-amd64 \
 	vmauth-darwin-amd64 \
 	vmbackup-darwin-amd64 \
 	vmrestore-darwin-amd64 \
@@ -133,7 +119,6 @@ vmutils-darwin-amd64: \
 vmutils-darwin-arm64: \
 	vmagent-darwin-arm64 \
 	vmalert-darwin-arm64 \
-	vmalert-tool-darwin-arm64 \
 	vmauth-darwin-arm64 \
 	vmbackup-darwin-arm64 \
 	vmrestore-darwin-arm64 \
@@ -142,7 +127,6 @@ vmutils-darwin-arm64: \
 vmutils-freebsd-amd64: \
 	vmagent-freebsd-amd64 \
 	vmalert-freebsd-amd64 \
-	vmalert-tool-freebsd-amd64 \
 	vmauth-freebsd-amd64 \
 	vmbackup-freebsd-amd64 \
 	vmrestore-freebsd-amd64 \
@@ -151,7 +135,6 @@ vmutils-freebsd-amd64: \
 vmutils-openbsd-amd64: \
 	vmagent-openbsd-amd64 \
 	vmalert-openbsd-amd64 \
-	vmalert-tool-openbsd-amd64 \
 	vmauth-openbsd-amd64 \
 	vmbackup-openbsd-amd64 \
 	vmrestore-openbsd-amd64 \
@@ -160,32 +143,29 @@ vmutils-openbsd-amd64: \
 vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
 	vmalert-windows-amd64 \
-	vmalert-tool-windows-amd64 \
 	vmauth-windows-amd64 \
 	vmbackup-windows-amd64 \
 	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
-crossbuild:
-	$(MAKE_PARALLEL) victoria-metrics-crossbuild vmutils-crossbuild
-
 victoria-metrics-crossbuild: \
 	victoria-metrics-linux-386 \
 	victoria-metrics-linux-amd64 \
 	victoria-metrics-linux-arm64 \
 	victoria-metrics-linux-arm \
+	victoria-metrics-linux-386 \
 	victoria-metrics-linux-ppc64le \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
 	victoria-metrics-freebsd-amd64 \
-	victoria-metrics-openbsd-amd64 \
-	victoria-metrics-windows-amd64
+	victoria-metrics-openbsd-amd64
 
 vmutils-crossbuild: \
 	vmutils-linux-386 \
 	vmutils-linux-amd64 \
 	vmutils-linux-arm64 \
 	vmutils-linux-arm \
+	vmutils-linux-386 \
 	vmutils-linux-ppc64le \
 	vmutils-darwin-amd64 \
 	vmutils-darwin-arm64 \
@@ -195,15 +175,14 @@ vmutils-crossbuild: \
 
 publish-release:
 	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
+	git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && LATEST_TAG=enterprise-cluster-stable $(MAKE) release publish
 
-release:
-	$(MAKE_PARALLEL) \
-		release-victoria-metrics \
-		release-vmutils
+release: \
+	release-victoria-metrics \
+	release-vmutils
 
 release-victoria-metrics: \
 	release-victoria-metrics-linux-386 \
@@ -262,16 +241,16 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-metrics-windows-$(GOARCH)-prod.exe
 
-release-victoria-logs:
-	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
-		release-victoria-logs-linux-amd64 \
-		release-victoria-logs-linux-arm \
-		release-victoria-logs-linux-arm64 \
-		release-victoria-logs-darwin-amd64 \
-		release-victoria-logs-darwin-arm64 \
-		release-victoria-logs-freebsd-amd64 \
-		release-victoria-logs-openbsd-amd64 \
-		release-victoria-logs-windows-amd64
+release-victoria-logs: \
+	release-victoria-logs-linux-386 \
+	release-victoria-logs-linux-amd64 \
+	release-victoria-logs-linux-arm \
+	release-victoria-logs-linux-arm64 \
+	release-victoria-logs-darwin-amd64 \
+	release-victoria-logs-darwin-arm64 \
+	release-victoria-logs-freebsd-amd64 \
+	release-victoria-logs-openbsd-amd64 \
+	release-victoria-logs-windows-amd64
 
 release-victoria-logs-linux-386:
 	GOOS=linux GOARCH=386 $(MAKE) release-victoria-logs-goos-goarch
@@ -360,7 +339,6 @@ release-vmutils-windows-amd64:
 release-vmutils-goos-goarch: \
 	vmagent-$(GOOS)-$(GOARCH)-prod \
 	vmalert-$(GOOS)-$(GOARCH)-prod \
-	vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 	vmauth-$(GOOS)-$(GOARCH)-prod \
 	vmbackup-$(GOOS)-$(GOARCH)-prod \
 	vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -369,7 +347,6 @@ release-vmutils-goos-goarch: \
 		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 			vmauth-$(GOOS)-$(GOARCH)-prod \
 			vmbackup-$(GOOS)-$(GOARCH)-prod \
 			vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -377,7 +354,6 @@ release-vmutils-goos-goarch: \
 		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 			vmauth-$(GOOS)-$(GOARCH)-prod \
 			vmbackup-$(GOOS)-$(GOARCH)-prod \
 			vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -386,7 +362,6 @@ release-vmutils-goos-goarch: \
 	cd bin && rm -rf \
 		vmagent-$(GOOS)-$(GOARCH)-prod \
 		vmalert-$(GOOS)-$(GOARCH)-prod \
-		vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 		vmauth-$(GOOS)-$(GOARCH)-prod \
 		vmbackup-$(GOOS)-$(GOARCH)-prod \
 		vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -395,7 +370,6 @@ release-vmutils-goos-goarch: \
 release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
-	vmalert-tool-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
 	vmbackup-windows-$(GOARCH)-prod \
 	vmrestore-windows-$(GOARCH)-prod \
@@ -404,7 +378,6 @@ release-vmutils-windows-goarch: \
 		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
 			vmbackup-windows-$(GOARCH)-prod.exe \
 			vmrestore-windows-$(GOARCH)-prod.exe \
@@ -412,7 +385,6 @@ release-vmutils-windows-goarch: \
 		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
 			vmbackup-windows-$(GOARCH)-prod.exe \
 			vmrestore-windows-$(GOARCH)-prod.exe \
@@ -421,7 +393,6 @@ release-vmutils-windows-goarch: \
 	cd bin && rm -rf \
 		vmagent-windows-$(GOARCH)-prod.exe \
 		vmalert-windows-$(GOARCH)-prod.exe \
-		vmalert-tool-windows-$(GOARCH)-prod.exe \
 		vmauth-windows-$(GOARCH)-prod.exe \
 		vmbackup-windows-$(GOARCH)-prod.exe \
 		vmrestore-windows-$(GOARCH)-prod.exe \
@@ -466,7 +437,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.21
+	go mod tidy -compat=1.20
 	go mod vendor
 
 app-local:
@@ -492,7 +463,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.57.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.54.2
 
 govulncheck: install-govulncheck
 	govulncheck ./...
@@ -535,3 +506,11 @@ copy-docs:
 docs-sync:
 	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
 	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs
+	SRC=app/vmagent/README.md DST=docs/vmagent.md OLD_URL='/vmagent.html' ORDER=3 TITLE=vmagent $(MAKE) copy-docs
+	SRC=app/vmalert/README.md DST=docs/vmalert.md OLD_URL='/vmalert.html' ORDER=4 TITLE=vmalert $(MAKE) copy-docs
+	SRC=app/vmauth/README.md DST=docs/vmauth.md OLD_URL='/vmauth.html' ORDER=5 TITLE=vmauth $(MAKE) copy-docs
+	SRC=app/vmbackup/README.md DST=docs/vmbackup.md OLD_URL='/vmbackup.html' ORDER=6 TITLE=vmbackup $(MAKE) copy-docs
+	SRC=app/vmrestore/README.md DST=docs/vmrestore.md OLD_URL='/vmrestore.html' ORDER=7 TITLE=vmrestore $(MAKE) copy-docs
+	SRC=app/vmctl/README.md DST=docs/vmctl.md OLD_URL='/vmctl.html' ORDER=8 TITLE=vmctl $(MAKE) copy-docs
+	SRC=app/vmgateway/README.md DST=docs/vmgateway.md OLD_URL='/vmgateway.html' ORDER=9 TITLE=vmgateway $(MAKE) copy-docs
+	SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md OLD_URL='/vmbackupmanager.html' ORDER=10 TITLE=vmbackupmanager $(MAKE) copy-docs

SECURITY.md

@@ -2,17 +2,13 @@
 
 ## Supported Versions
 
-The following versions of VictoriaMetrics receive regular security fixes:
-
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
-| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x LTS release | :white_check_mark: |
+| v1.87.x LTS release | :white_check_mark: |
 | other releases  | :x:                |
 
-See [this page](https://victoriametrics.com/security/) for more details.
-
 ## Reporting a Vulnerability
 
 Please report any security issues to security@victoriametrics.com

app/victoria-logs/deployment/Dockerfile

@@ -1,7 +1,7 @@
 ARG base_image
 FROM $base_image
 
-EXPOSE 9428
+EXPOSE 8428
 
 ENTRYPOINT ["/victoria-logs-prod"]
 ARG src_binary

app/victoria-logs/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -21,10 +22,11 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
 )
 
 func main() {
@@ -32,31 +34,27 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	cgroup.SetGOGC(*gogc)
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9428"}
-	}
-	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
+	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
 	startTime := time.Now()
 
 	vlstorage.Init()
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
 
-	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
 
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())

app/victoria-logs/multiarch/Dockerfile

@@ -6,7 +6,7 @@ RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 9428
+EXPOSE 8428
 ENTRYPOINT ["/victoria-logs-prod"]
 ARG TARGETARCH
 COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/main.go

@@ -26,12 +26,12 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP addresses to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/#deduplication")
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
 		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
@@ -48,6 +48,7 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		*dryRun = true
@@ -66,37 +67,30 @@ func main() {
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8428"}
-	}
-	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
+	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-
 	startSelfScraper()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
-	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
 
 	stopSelfScraper()
 
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	vminsert.Stop()
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	vmstorage.Stop()
 	vmselect.Stop()
@@ -128,7 +122,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"api/v1/status/tsdb", "tsdb status page"},
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
-			{"-/reload", "reload configuration"},
 		})
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -7,13 +7,11 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -39,13 +37,11 @@ const (
 )
 
 const (
-	testReadHTTPPath           = "http://127.0.0.1" + testHTTPListenAddr
-	testWriteHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr + "/write"
-	testOpenTSDBWriteHTTPPath  = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
-	testPromWriteHTTPPath      = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
-	testImportCSVWriteHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/import/csv"
-
-	testHealthHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
 )
 
 const (
@@ -58,15 +54,15 @@ var (
 )
 
 type test struct {
-	Name                     string   `json:"name"`
-	Data                     []string `json:"data"`
-	InsertQuery              string   `json:"insert_query"`
-	Query                    []string `json:"query"`
-	ResultMetrics            []Metric `json:"result_metrics"`
-	ResultSeries             Series   `json:"result_series"`
-	ResultQuery              Query    `json:"result_query"`
-	Issue                    string   `json:"issue"`
-	ExpectedResultLinesCount int      `json:"expected_result_lines_count"`
+	Name             string     `json:"name"`
+	Data             []string   `json:"data"`
+	InsertQuery      string     `json:"insert_query"`
+	Query            []string   `json:"query"`
+	ResultMetrics    []Metric   `json:"result_metrics"`
+	ResultSeries     Series     `json:"result_series"`
+	ResultQuery      Query      `json:"result_query"`
+	ResultQueryRange QueryRange `json:"result_query_range"`
+	Issue            string     `json:"issue"`
 }
 
 type Metric struct {
@@ -84,90 +80,42 @@ type Series struct {
 	Status string              `json:"status"`
 	Data   []map[string]string `json:"data"`
 }
-
 type Query struct {
-	Status string `json:"status"`
-	Data   struct {
-		ResultType string          `json:"resultType"`
-		Result     json.RawMessage `json:"result"`
-	} `json:"data"`
+	Status string    `json:"status"`
+	Data   QueryData `json:"data"`
+}
+type QueryData struct {
+	ResultType string            `json:"resultType"`
+	Result     []QueryDataResult `json:"result"`
 }
 
-const rtVector, rtMatrix = "vector", "matrix"
-
-func (q *Query) metrics() ([]Metric, error) {
-	switch q.Data.ResultType {
-	case rtVector:
-		var r QueryInstant
-		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
-			return nil, err
-		}
-		return r.metrics()
-	case rtMatrix:
-		var r QueryRange
-		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
-			return nil, err
-		}
-		return r.metrics()
-	default:
-		return nil, fmt.Errorf("unknown result type %q", q.Data.ResultType)
-	}
+type QueryDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Value  []interface{}     `json:"value"`
 }
 
-type QueryInstant struct {
-	Result []struct {
-		Labels map[string]string `json:"metric"`
-		TV     [2]interface{}    `json:"value"`
-	} `json:"result"`
-}
-
-func (q QueryInstant) metrics() ([]Metric, error) {
-	result := make([]Metric, len(q.Result))
-	for i, res := range q.Result {
-		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
-		if err != nil {
-			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
-		}
-		var m Metric
-		m.Metric = res.Labels
-		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
-		m.Values = append(m.Values, f)
-		result[i] = m
-	}
-	return result, nil
+func (r *QueryDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
 }
 
 type QueryRange struct {
-	Result []struct {
-		Metric map[string]string `json:"metric"`
-		Values [][]interface{}   `json:"values"`
-	} `json:"result"`
+	Status string         `json:"status"`
+	Data   QueryRangeData `json:"data"`
+}
+type QueryRangeData struct {
+	ResultType string                 `json:"resultType"`
+	Result     []QueryRangeDataResult `json:"result"`
 }
 
-func (q QueryRange) metrics() ([]Metric, error) {
-	var result []Metric
-	for i, res := range q.Result {
-		var m Metric
-		for _, tv := range res.Values {
-			f, err := strconv.ParseFloat(tv[1].(string), 64)
-			if err != nil {
-				return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, tv[1], err)
-			}
-			m.Values = append(m.Values, f)
-			m.Timestamps = append(m.Timestamps, int64(tv[0].(float64)))
-		}
-		if len(m.Values) < 1 || len(m.Timestamps) < 1 {
-			return nil, fmt.Errorf("metric %v contains no values", res)
-		}
-		m.Metric = q.Result[i].Metric
-		result = append(result, m)
-	}
-	return result, nil
+type QueryRangeDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Values [][]interface{}   `json:"values"`
 }
 
-func (q *Query) UnmarshalJSON(b []byte) error {
-	type plain Query
-	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(q))
+func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryRangeDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
 }
 
 func TestMain(m *testing.M) {
@@ -184,7 +132,7 @@ func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, false, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
@@ -230,7 +178,7 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
-	if err := httpserver.Stop(*httpListenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		log.Printf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
@@ -241,18 +189,14 @@ func tearDown() {
 
 func TestWriteRead(t *testing.T) {
 	t.Run("write", testWrite)
-	time.Sleep(500 * time.Millisecond)
 	vmstorage.Storage.DebugFlush()
-	time.Sleep(1500 * time.Millisecond)
+	time.Sleep(1 * time.Second)
 	t.Run("read", testRead)
 }
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
 		for _, test := range readIn("prometheus", t, insertionTime) {
-			if test.Data == nil {
-				continue
-			}
 			s := newSuite(t)
 			r := testutil.WriteRequest{}
 			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
@@ -265,14 +209,6 @@ func testWrite(t *testing.T) {
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
-	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", t, insertionTime) {
-			if test.Data == nil {
-				continue
-			}
-			httpWrite(t, testImportCSVWriteHTTPPath, test.InsertQuery, bytes.NewBuffer([]byte(strings.Join(test.Data, "\n"))))
-		}
-	})
 
 	t.Run("influxdb", func(t *testing.T) {
 		for _, x := range readIn("influxdb", t, insertionTime) {
@@ -314,7 +250,7 @@ func testWrite(t *testing.T) {
 }
 
 func testRead(t *testing.T) {
-	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
 			for _, x := range readIn(engine, t, insertionTime) {
 				test := x
@@ -325,12 +261,7 @@ func testRead(t *testing.T) {
 						if test.Issue != "" {
 							test.Issue = "\nRegression in " + test.Issue
 						}
-						switch {
-						case strings.HasPrefix(q, "/api/v1/export/csv"):
-							data := strings.Split(string(httpReadData(t, testReadHTTPPath, q)), "\n")
-							if len(data) == test.ExpectedResultLinesCount {
-								t.Fatalf("not expected number of csv lines want=%d\ngot=%d test=%s.%s\n\response=%q", len(data), test.ExpectedResultLinesCount, q, test.Issue, strings.Join(data, "\n"))
-							}
+						switch true {
 						case strings.HasPrefix(q, "/api/v1/export"):
 							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
 								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
@@ -341,19 +272,17 @@ func testRead(t *testing.T) {
 							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
 								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
 							}
+						case strings.HasPrefix(q, "/api/v1/query_range"):
+							queryResult := QueryRange{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil {
+								t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue)
+							}
 						case strings.HasPrefix(q, "/api/v1/query"):
 							queryResult := Query{}
 							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
-							gotMetrics, err := queryResult.metrics()
-							if err != nil {
-								t.Fatalf("failed to parse query response: %s", err)
-							}
-							expMetrics, err := test.ResultQuery.metrics()
-							if err != nil {
-								t.Fatalf("failed to parse expected response: %s", err)
-							}
-							if err := checkMetricsResult(gotMetrics, expMetrics); err != nil {
-								t.Fatalf("%q fails with error %s.%s", q, err, test.Issue)
+							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
+								t.Fatalf("Query. %s fails with error: %s.%s", q, err, test.Issue)
 							}
 						default:
 							t.Fatalf("unsupported read query %s", q)
@@ -369,7 +298,7 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 	t.Helper()
 	s := newSuite(t)
 	var tt []test
-	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -431,7 +360,6 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	}
 	return rows
 }
-
 func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
 	s := newSuite(t)
@@ -444,20 +372,6 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
-func httpReadData(t *testing.T, address, query string) []byte {
-	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	s.equalInt(resp.StatusCode, 200)
-	data, err := io.ReadAll(resp.Body)
-	s.noError(err)
-	return data
-}
-
 func checkMetricsResult(got, want []Metric) error {
 	for _, r := range append([]Metric(nil), got...) {
 		want = removeIfFoundMetrics(r, want)
@@ -503,6 +417,60 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
+func checkQueryResult(got, want Query) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Value[0], item.Value[0]) && reflect.DeepEqual(r.Value[1], item.Value[1]) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryRangeResult(got, want QueryRange) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryRangeDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryRangeData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
 type suite struct{ t *testing.T }
 
 func newSuite(t *testing.T) *suite { return &suite{t: t} }
@@ -530,73 +498,3 @@ func (s *suite) greaterThan(a, b int) {
 		s.t.FailNow()
 	}
 }
-
-func TestImportJSONLines(t *testing.T) {
-	f := func(labelsCount, labelLen int) {
-		t.Helper()
-
-		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
-		line := generateJSONLine(labelsCount, labelLen)
-		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
-		}
-		if resp.StatusCode != 204 {
-			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
-		}
-	}
-
-	// labels with various lengths
-	for i := 0; i < 500; i++ {
-		f(10, i*5)
-	}
-
-	// Too many labels
-	f(1000, 100)
-
-	// Too long labels
-	f(1, 100_000)
-	f(10, 100_000)
-	f(10, 10_000)
-}
-
-func generateJSONLine(labelsCount, labelLen int) string {
-	m := make(map[string]string, labelsCount)
-	m["__name__"] = generateSizedRandomString(labelLen)
-	for j := 1; j < labelsCount; j++ {
-		labelName := generateSizedRandomString(labelLen)
-		labelValue := generateSizedRandomString(labelLen)
-		m[labelName] = labelValue
-	}
-
-	type jsonLine struct {
-		Metric     map[string]string `json:"metric"`
-		Values     []float64         `json:"values"`
-		Timestamps []int64           `json:"timestamps"`
-	}
-	line := &jsonLine{
-		Metric:     m,
-		Values:     []float64{1.34},
-		Timestamps: []int64{time.Now().UnixNano() / 1e6},
-	}
-	data, err := json.Marshal(&line)
-	if err != nil {
-		panic(fmt.Errorf("cannot marshal JSON: %w", err))
-	}
-	data = append(data, '\n')
-	return string(data)
-}
-
-const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
-
-func generateSizedRandomString(size int) string {
-	dst := make([]byte, size)
-	for i := range dst {
-		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
-	}
-	return string(dst)
-}

app/victoria-metrics/self_scraper.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -50,8 +49,16 @@ func selfScraper(scrapeInterval time.Duration) {
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
-	f := func(currentTime time.Time, sendStaleMarkers bool) {
-		currentTimestamp := currentTime.UnixNano() / 1e6
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
 		bb.Reset()
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
@@ -76,27 +83,12 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr := &mrs[len(mrs)-1]
 			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 			mr.Timestamp = currentTimestamp
-			if sendStaleMarkers {
-				mr.Value = decimal.StaleNaN
-			} else {
-				mr.Value = r.Value
-			}
+			mr.Value = r.Value
 		}
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
 	}
-	for {
-		select {
-		case <-selfScraperStopCh:
-			f(time.Now(), true)
-			t.Stop()
-			logger.Infof("stopped self-scraping `/metrics` page")
-			return
-		case currentTime := <-t.C:
-			f(currentTime, false)
-		}
-	}
 }
 
 func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
@@ -106,7 +98,7 @@ func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
 		dst = append(dst, prompb.Label{})
 	}
 	lb := &dst[len(dst)-1]
-	lb.Name = key
-	lb.Value = value
+	lb.Name = bytesutil.ToUnsafeBytes(key)
+	lb.Value = bytesutil.ToUnsafeBytes(value)
 	return dst
 }

app/victoria-metrics/testdata/csv/basic.json

@@ -1,14 +0,0 @@
-{
-  "name": "csv export",
-  "data": [
-    "rfc3339,4,{TIME_MS}",
-    "rfc3339milli,6,{TIME_MS}",
-    "ts,8,{TIME_MS}",
-    "tsms,10,{TIME_MS},"
-  ],
-  "insert_query": "?format=1:label:tfmt,2:metric:test_csv,3:time:unix_ms",
-  "query": [
-    "/api/v1/export/csv?format=__name__,tfmt,__value__,__timestamp__:rfc3339&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
-  ],
-  "expected_result_lines_count": 4
-}

app/victoria-metrics/testdata/csv/with_extra_labels.json

@@ -1,14 +0,0 @@
-{
-  "name": "csv export with extra_labels",
-  "data": [
-    "location-1,4,{TIME_MS}",
-    "location-2,6,{TIME_MS}",
-    "location-3,8,{TIME_MS}",
-    "location-4,10,{TIME_MS},"
-  ],
-  "insert_query": "?format=1:label:location,2:metric:test_csv_labels,3:time:unix_ms&extra_label=location=location-1",
-  "query": [
-    "/api/v1/export/csv?format=__name__,location,__value__,__timestamp__:unix_ms&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
-  ],
-  "expected_result_lines_count": 4
-}

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -7,7 +7,7 @@
     "not_nan_not_inf;item=y 3 {TIME_S-1m}",
     "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
   "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -6,7 +6,7 @@
     "empty_label_match;foo=bar 2 {TIME_S-1m}",
     "empty_label_match;foo=baz 3 {TIME_S-1m}"],
   "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S-1m}&end={TIME_S}&step=60"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -8,7 +8,7 @@
     "max_lookback_set 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_set"},"values":[

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -8,7 +8,7 @@
     "max_lookback_unset 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -8,7 +8,7 @@
     "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
   ],
   "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/prometheus/instant-matrix.json

@@ -1,12 +0,0 @@
-{
-  "name": "instant query with look-behind window",
-  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
-  "query": ["/api/v1/query?query=foo[5m]"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"matrix",
-        "result":[{"metric":{"__name__":"foo"},"values":[["{TIME_S-60s}", "1"]]}]
-      }
-    }
-}

app/victoria-metrics/testdata/prometheus/instant-scalar.json

@@ -1,11 +0,0 @@
-{
-  "name": "instant scalar query",
-  "query": ["/api/v1/query?query=42&time={TIME_S}"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"vector",
-        "result":[{"metric":{},"value":["{TIME_S}", "42"]}]
-      }
-    }
-}

app/victoria-metrics/testdata/prometheus/issue-5553-too-big-lookback.json

@@ -1,13 +0,0 @@
-{
-  "name": "too big look-behind window",
-  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
-  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"},{\"name\":\"issue\",\"value\":\"5553\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
-  "query": ["/api/v1/query?query=foo{issue=\"5553\"}[100y]"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"matrix",
-        "result":[{"metric":{"__name__":"foo", "issue": "5553"},"values":[["{TIME_S-60s}", "1"]]}]
-      }
-    }
-}

app/victoria-metrics/testdata/prometheus/query-range.json

@@ -1,18 +0,0 @@
-{
-  "name": "query range",
-  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
-  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"bar\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}, {\"value\":2,\"timestamp\":\"{TIME_MS-120s}\"}, {\"value\":1,\"timestamp\":\"{TIME_MS-180s}\"}]}]"],
-  "query": ["/api/v1/query_range?query=bar&step=30s&start={TIME_MS-180s}"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"matrix",
-        "result":[
-          {
-            "metric":{"__name__":"bar"},
-            "values":[["{TIME_S-180s}", "1"],["{TIME_S-150s}", "1"],["{TIME_S-120s}", "2"],["{TIME_S-90s}", "2"], ["{TIME_S-60s}", "1"], ["{TIME_S-30s}", "1"], ["{TIME_S}", "1"]]
-          }
-        ]
-      }
-    }
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -105,7 +105,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		vlstorage.MustAddRows(lr)
 		logstorage.PutLogRows(lr)
 		if err != nil {
-			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
+			logger.Warnf("cannot decode log message #%d in /_bulk request: %s", n, err)
 			return true
 		}
 

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -120,10 +120,10 @@ func compressData(s string) string {
 	var bb bytes.Buffer
 	zw := gzip.NewWriter(&bb)
 	if _, err := zw.Write([]byte(s)); err != nil {
-		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
+		panic(fmt.Errorf("unexpected error when compressing data: %s", err))
 	}
 	if err := zw.Close(); err != nil {
-		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
+		panic(fmt.Errorf("unexpected error when closing gzip writer: %s", err))
 	}
 	return bb.String()
 }

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -33,7 +33,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 
 	timeField := "@timestamp"
 	msgField := "message"
-	processLogMessage := func(_ int64, _ []logstorage.Field) {}
+	processLogMessage := func(timestmap int64, fields []logstorage.Field) {}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))
@@ -43,7 +43,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 			r.Reset(dataBytes)
 			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
 			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
+				panic(fmt.Errorf("unexpected error: %s", err))
 			}
 		}
 	})

app/vlinsert/loki/loki_json_test.go

@@ -11,7 +11,7 @@ import (
 func TestParseJSONRequestFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
 			t.Fatalf("unexpected call to parseJSONRequest callback!")
 		})
 		if err == nil {

app/vlinsert/loki/loki_json_timing_test.go

@@ -27,9 +27,9 @@ func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseJSONRequest(data, func(timestamp int64, fields []logstorage.Field) {})
 			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
+				panic(fmt.Errorf("unexpected error: %s", err))
 			}
 		}
 	})

app/vlinsert/loki/loki_protobuf.go

@@ -84,7 +84,7 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 
 	err = req.Unmarshal(bb.B)
 	if err != nil {
-		return 0, fmt.Errorf("cannot parse request body: %w", err)
+		return 0, fmt.Errorf("cannot parse request body: %s", err)
 	}
 
 	var commonFields []logstorage.Field
@@ -97,7 +97,7 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 		// Labels are same for all entries in the stream.
 		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
 		if err != nil {
-			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
+			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %s", stream.Labels, err)
 		}
 		fields := commonFields
 

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -29,9 +29,9 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseProtobufRequest(body, func(timestamp int64, fields []logstorage.Field) {})
 			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
+				panic(fmt.Errorf("unexpected error: %s", err))
 			}
 		}
 	})

app/vlselect/logsql/logsql.go

@@ -7,7 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
@@ -18,18 +17,13 @@ var (
 )
 
 // ProcessQueryRequest handles /select/logsql/query request
-func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}, cancel func()) {
+func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}) {
 	// Extract tenantID
 	tenantID, err := logstorage.GetTenantIDFromRequest(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
 
 	qStr := r.FormValue("query")
 	q, err := logstorage.ParseQuery(qStr)
@@ -40,7 +34,7 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
 
 	sw := getSortWriter()
-	sw.Init(w, maxSortBufferSize.IntN(), limit)
+	sw.Init(w, maxSortBufferSize.IntN())
 	tenantIDs := []logstorage.TenantID{tenantID}
 	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
 		if len(columns) == 0 {
@@ -48,36 +42,11 @@ func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan s
 		}
 		rowsCount := len(columns[0].Values)
 
-		// skip entries with empty _stream column
-		// _stream is empty in case indexdb entry was not flushed to the storage yet
-		// skipping such entries makes the result more consistent
-		streamCol := 0
-
-		// fast path
-		// _stream column is a built-in column and it is always supposed to be at the same position
-		if len(columns) >= 2 && columns[1].Name == "_stream" {
-			streamCol = 1
-		} else {
-			for i := 1; i < len(columns); i++ {
-				if columns[i].Name == "_stream" {
-					streamCol = i
-					break
-				}
-			}
-		}
-
 		bb := blockResultPool.Get()
 		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
-			if columns[streamCol].Values[rowIdx] == "" {
-				continue
-			}
 			WriteJSONRow(bb, columns, rowIdx)
 		}
-
-		if !sw.TryWrite(bb.B) {
-			cancel()
-		}
-
+		sw.MustWrite(bb.B)
 		blockResultPool.Put(bb)
 	})
 	sw.FinalFlush()

app/vlselect/logsql/sort_writer.go

@@ -36,12 +36,8 @@ var sortWriterPool sync.Pool
 // If the buf isn't empty at FinalFlush() call, then the buffered data
 // is sorted by _time field.
 type sortWriter struct {
-	mu sync.Mutex
-	w  io.Writer
-
-	maxLines     int
-	linesWritten int
-
+	mu         sync.Mutex
+	w          io.Writer
 	maxBufLen  int
 	buf        []byte
 	bufFlushed bool
@@ -51,119 +47,58 @@ type sortWriter struct {
 
 func (sw *sortWriter) reset() {
 	sw.w = nil
-
-	sw.maxLines = 0
-	sw.linesWritten = 0
-
 	sw.maxBufLen = 0
 	sw.buf = sw.buf[:0]
 	sw.bufFlushed = false
 	sw.hasErr = false
 }
 
-// Init initializes sw.
-//
-// If maxLines is set to positive value, then sw accepts up to maxLines
-// and then rejects all the other lines by returning false from TryWrite.
-func (sw *sortWriter) Init(w io.Writer, maxBufLen, maxLines int) {
+func (sw *sortWriter) Init(w io.Writer, maxBufLen int) {
 	sw.reset()
 
 	sw.w = w
 	sw.maxBufLen = maxBufLen
-	sw.maxLines = maxLines
 }
 
-// TryWrite writes p to sw.
-//
-// True is returned on successful write, false otherwise.
-//
-// Unsuccessful write may occur on underlying write error or when maxLines lines are already written to sw.
-func (sw *sortWriter) TryWrite(p []byte) bool {
+func (sw *sortWriter) MustWrite(p []byte) {
 	sw.mu.Lock()
 	defer sw.mu.Unlock()
 
 	if sw.hasErr {
-		return false
+		return
 	}
 
 	if sw.bufFlushed {
-		if !sw.writeToUnderlyingWriterLocked(p) {
+		if _, err := sw.w.Write(p); err != nil {
 			sw.hasErr = true
-			return false
 		}
-		return true
+		return
 	}
-
 	if len(sw.buf)+len(p) < sw.maxBufLen {
 		sw.buf = append(sw.buf, p...)
-		return true
+		return
 	}
-
 	sw.bufFlushed = true
-	if !sw.writeToUnderlyingWriterLocked(sw.buf) {
-		sw.hasErr = true
-		return false
-	}
-	sw.buf = sw.buf[:0]
-
-	if !sw.writeToUnderlyingWriterLocked(p) {
-		sw.hasErr = true
-		return false
-	}
-	return true
-}
-
-func (sw *sortWriter) writeToUnderlyingWriterLocked(p []byte) bool {
-	if len(p) == 0 {
-		return true
-	}
-	if sw.maxLines > 0 {
-		if sw.linesWritten >= sw.maxLines {
-			return false
+	if len(sw.buf) > 0 {
+		if _, err := sw.w.Write(sw.buf); err != nil {
+			sw.hasErr = true
+			return
 		}
-		var linesLeft int
-		p, linesLeft = trimLines(p, sw.maxLines-sw.linesWritten)
-		sw.linesWritten += linesLeft
+		sw.buf = sw.buf[:0]
 	}
 	if _, err := sw.w.Write(p); err != nil {
-		return false
+		sw.hasErr = true
 	}
-	return true
 }
 
-func trimLines(p []byte, maxLines int) ([]byte, int) {
-	if maxLines <= 0 {
-		return nil, 0
-	}
-	n := bytes.Count(p, newline)
-	if n < maxLines {
-		return p, n
-	}
-	for n >= maxLines {
-		idx := bytes.LastIndexByte(p, '\n')
-		p = p[:idx]
-		n--
-	}
-	return p[:len(p)+1], maxLines
-}
-
-var newline = []byte("\n")
-
 func (sw *sortWriter) FinalFlush() {
 	if sw.hasErr || sw.bufFlushed {
 		return
 	}
-
 	rs := getRowsSorter()
 	rs.parseRows(sw.buf)
 	rs.sort()
-
-	rows := rs.rows
-	if sw.maxLines > 0 && len(rows) > sw.maxLines {
-		rows = rows[:sw.maxLines]
-	}
-	WriteJSONRows(sw.w, rows)
-
+	WriteJSONRows(sw.w, rs.rows)
 	putRowsSorter(rs)
 }
 

app/vlselect/logsql/sort_writer_test.go

@@ -7,16 +7,15 @@ import (
 )
 
 func TestSortWriter(t *testing.T) {
-	f := func(maxBufLen, maxLines int, data string, expectedResult string) {
+	f := func(maxBufLen int, data string, expectedResult string) {
 		t.Helper()
 
 		var bb bytes.Buffer
 		sw := getSortWriter()
-		sw.Init(&bb, maxBufLen, maxLines)
+		sw.Init(&bb, maxBufLen)
+
 		for _, s := range strings.Split(data, "\n") {
-			if !sw.TryWrite([]byte(s + "\n")) {
-				break
-			}
+			sw.MustWrite([]byte(s + "\n"))
 		}
 		sw.FinalFlush()
 		putSortWriter(sw)
@@ -27,20 +26,14 @@ func TestSortWriter(t *testing.T) {
 		}
 	}
 
-	f(100, 0, "", "")
-	f(100, 0, "{}", "{}\n")
+	f(100, "", "")
+	f(100, "{}", "{}\n")
 
 	data := `{"_time":"def","_msg":"xxx"}
 {"_time":"abc","_msg":"foo"}`
 	resultExpected := `{"_time":"abc","_msg":"foo"}
 {"_time":"def","_msg":"xxx"}
 `
-	f(100, 0, data, resultExpected)
-	f(10, 0, data, data+"\n")
-
-	// Test with the maxLines
-	f(100, 1, data, `{"_time":"abc","_msg":"foo"}`+"\n")
-	f(10, 1, data, `{"_time":"def","_msg":"xxx"}`+"\n")
-	f(10, 2, data, data+"\n")
-	f(100, 2, data, resultExpected)
+	f(100, data, resultExpected)
+	f(10, data, data+"\n")
 }

app/vlselect/main.go

@@ -1,7 +1,6 @@
 package vlselect
 
 import (
-	"context"
 	"embed"
 	"flag"
 	"fmt"
@@ -89,12 +88,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	if strings.HasPrefix(path, "/vmui/") {
-		if strings.HasPrefix(path, "/vmui/static/") {
-			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
-			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
-			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
-			w.Header().Set("Cache-Control", "max-age=31536000")
-		}
 		r.URL.Path = path
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
@@ -102,8 +95,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	// Limit the number of concurrent queries, which can consume big amounts of CPU.
 	startTime := time.Now()
-	ctx := r.Context()
-	stopCh := ctx.Done()
+	stopCh := r.Context().Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
@@ -141,15 +133,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 	}
 
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	defer cancel()
-	stopCh = ctxWithCancel.Done()
-
 	switch {
 	case path == "/logsql/query":
 		logsqlQueryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(w, r, stopCh, cancel)
+		logsql.ProcessQueryRequest(w, r, stopCh)
 		return true
 	default:
 		return false

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,14 @@
 {
   "files": {
-    "main.css": "./static/css/main.bc07cc78.css",
-    "main.js": "./static/js/main.034044a7.js",
-    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.10add6e7bdf0f1d98cf7.md",
+    "main.css": "./static/css/main.3258b6c4.css",
+    "main.js": "./static/js/main.41b816cc.js",
+    "static/js/522.b5ae4365.chunk.js": "./static/js/522.b5ae4365.chunk.js",
+    "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
+    "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.bc07cc78.css",
-    "static/js/main.034044a7.js"
+    "static/css/main.3258b6c4.css",
+    "static/js/main.41b816cc.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.034044a7.js"></script><link href="./static/css/main.bc07cc78.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.41b816cc.js"></script><link href="./static/css/main.3258b6c4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.41b816cc.js.LICENSE.txt

@@ -4,8 +4,10 @@
 	http://jedwatson.github.io/classnames
 */
 
+/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
+
 /**
- * @remix-run/router v1.15.1
+ * @remix-run/router v1.7.2
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +18,7 @@
  */
 
 /**
- * React Router DOM v6.22.1
+ * React Router DOM v6.14.2
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +29,7 @@
  */
 
 /**
- * React Router v6.22.1
+ * React Router v6.14.2
  *
  * Copyright (c) Remix Software Inc.
  *

app/vmagent/csvimport/request_handler.go

@@ -65,9 +65,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/datadog/request_handler.go

@@ -1,4 +1,4 @@
-package datadogv1
+package datadog
 
 import (
 	"net/http"
@@ -8,32 +8,33 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv1"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv1"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv1"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
 )
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(series []datadogv1.Series) error {
+	return stream.Parse(req.Body, ce, func(series []parser.Series) error {
 		return insertRows(at, series, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -62,7 +63,7 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbm
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := datadogutils.SplitTag(tag)
+			name, value := parser.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
@@ -87,9 +88,7 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbm
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/datadogsketches/request_handler.go

@@ -1,95 +0,0 @@
-package datadogsketches
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
-		return insertRows(at, sketches, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for _, sketch := range sketches {
-		ms := sketch.ToSummary()
-		for _, m := range ms {
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: m.Name,
-			})
-			for _, label := range m.Labels {
-				labels = append(labels, prompbmarshal.Label{
-					Name:  label.Name,
-					Value: label.Value,
-				})
-			}
-			for _, tag := range sketch.Tags {
-				name, value := datadogutils.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
-				labels = append(labels, prompbmarshal.Label{
-					Name:  name,
-					Value: value,
-				})
-			}
-			labels = append(labels, extraLabels...)
-			samplesLen := len(samples)
-			for _, p := range m.Points {
-				samples = append(samples, prompbmarshal.Sample{
-					Timestamp: p.Timestamp,
-					Value:     p.Value,
-				})
-			}
-			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[samplesLen:],
-			})
-		}
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/datadogv2/request_handler.go

@@ -1,102 +0,0 @@
-package datadogv2
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv2"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv2"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv2"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/v2/series request.
-//
-// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ct := req.Header.Get("Content-Type")
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, ct, func(series []datadogv2.Series) error {
-		return insertRows(at, series, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range series {
-		ss := &series[i]
-		rowsTotal += len(ss.Points)
-		labelsLen := len(labels)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: ss.Metric,
-		})
-		for _, rs := range ss.Resources {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  rs.Type,
-				Value: rs.Name,
-			})
-		}
-		if ss.SourceTypeName != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "source_type_name",
-				Value: ss.SourceTypeName,
-			})
-		}
-		for _, tag := range ss.Tags {
-			name, value := datadogutils.SplitTag(tag)
-			if name == "host" {
-				name = "exported_host"
-			}
-			labels = append(labels, prompbmarshal.Label{
-				Name:  name,
-				Value: value,
-			})
-		}
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		for _, pt := range ss.Points {
-			samples = append(samples, prompbmarshal.Sample{
-				Timestamp: pt.Timestamp * 1000,
-				Value:     pt.Value,
-			})
-		}
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/graphite/request_handler.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
@@ -21,12 +20,10 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
-		return insertRows(nil, rows)
-	})
+	return stream.Parse(r, insertRows)
 }
 
-func insertRows(at *auth.Token, rows []parser.Row) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -59,9 +56,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -36,9 +36,9 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
 	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
-		return insertRows(at, db, rows, nil)
+		return insertRows(nil, db, rows, nil)
 	})
 }
 
@@ -130,9 +130,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	if !remotewrite.TryPush(at, &ctx.ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/main.go

@@ -8,16 +8,16 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/newrelic"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
@@ -40,16 +40,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
@@ -70,8 +68,7 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -98,6 +95,7 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -120,18 +118,13 @@ func main() {
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8429"}
-	}
-	logger.Infof("starting vmagent at %q...", listenAddrs)
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
-	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, false)
+			return influx.InsertHandlerForReader(r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
@@ -146,23 +139,24 @@ func main() {
 		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
 	}
 
-	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
+	promscrape.Init(remotewrite.Push)
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
-	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	remotewrite.StopIngestionRateLimiter()
-	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -264,7 +258,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		path = strings.TrimSuffix(path, "/")
 	}
 	switch path {
-	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
+	case "/prometheus/api/v1/write", "/api/v1/write":
 		if common.HandleVMProtoServerHandshake(w, r) {
 			return true
 		}
@@ -316,53 +310,19 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
+	case "/opentelemetry/api/v1/push":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
 			opentelemetryPushErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "/newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "/newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(nil, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
+		w.WriteHeader(http.StatusOK)
 		return true
 	case "/datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv2WriteErrors.Inc()
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -371,15 +331,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -434,7 +385,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -443,7 +394,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -453,15 +404,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
-			return true
-		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
 	case "/ready":
-		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
 			http.Error(w, errMsg, http.StatusTooEarly)
 		} else {
@@ -513,7 +461,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
 	}
 	switch p.Suffix {
-	case "prometheus/", "prometheus", "prometheus/api/v1/write", "prometheus/api/v1/push":
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(at, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -562,52 +510,19 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
+	case "opentelemetry/api/v1/push":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
 			opentelemetryPushErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(at, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
+		w.WriteHeader(http.StatusOK)
 		return true
 	case "datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv2WriteErrors.Inc()
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -615,15 +530,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -674,28 +580,16 @@ var (
 
 	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
-	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-
-	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-
-	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-
-	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-
-	newrelicInventoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/inventory/deltas", protocol="newrelic"}`)
-	newrelicCheckRequest      = metrics.NewCounter(`vm_http_requests_total{path="/newrelic", protocol="newrelic"}`)
+	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
+	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
 
 	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)

app/vmagent/native/request_handler.go

@@ -84,8 +84,6 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/newrelic/request_handler.go

@@ -1,88 +0,0 @@
-package newrelic
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="newrelic"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="newrelic"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="newrelic"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	isGzip := ce == "gzip"
-	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
-		return insertRows(at, rows, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	samplesCount := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		tags := r.Tags
-		srcSamples := r.Samples
-		for j := range srcSamples {
-			s := &srcSamples[j]
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: bytesutil.ToUnsafeString(s.Name),
-			})
-			for k := range tags {
-				t := &tags[k]
-				labels = append(labels, prompbmarshal.Label{
-					Name:  bytesutil.ToUnsafeString(t.Key),
-					Value: bytesutil.ToUnsafeString(t.Value),
-				})
-			}
-			samples = append(samples, prompbmarshal.Sample{
-				Value:     s.Value,
-				Timestamp: r.Timestamp,
-			})
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[len(samples)-1:],
-			})
-			labels = append(labels, extraLabels...)
-		}
-		samplesCount += len(srcSamples)
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(samplesCount)
-	}
-	rowsPerInsert.Update(float64(samplesCount))
-	return nil
-}

app/vmagent/opentelemetry/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -28,15 +27,10 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
-		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
-			processBody = firehose.ProcessRequestBody
-		} else {
-			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-		}
+		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 	}
-	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }
@@ -65,9 +59,7 @@ func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/opentsdb/request_handler.go

@@ -56,9 +56,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(nil, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/opentsdbhttp/request_handler.go

@@ -64,9 +64,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -73,9 +73,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/promremotewrite/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -47,8 +48,8 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
 			labels = append(labels, prompbmarshal.Label{
-				Name:  label.Name,
-				Value: label.Value,
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
 			})
 		}
 		labels = append(labels, extraLabels...)
@@ -68,9 +69,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/remotewrite/client.go

@@ -17,9 +17,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -31,12 +29,11 @@ var (
 
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
-		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
+		"is sent after temporary unavailability of the remote storage")
 	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -61,10 +58,8 @@ var (
 	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
-	oauth2EndpointParams   = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
-		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
-	oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
-	oauth2Scopes   = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
 
 	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
 		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
@@ -92,7 +87,7 @@ type client struct {
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
 
-	rl *ratelimiter.RateLimiter
+	rl rateLimiter
 
 	bytesSent       *metrics.Counter
 	blocksSent      *metrics.Counter
@@ -111,15 +106,17 @@ type client struct {
 func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
-		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
+	tlsCfg := authCfg.NewTLSConfig()
 	awsCfg, err := getAWSAPIConfig(argIdx)
 	if err != nil {
-		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
 		DialContext:         statDial,
-		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
+		TLSClientConfig:     tlsCfg,
+		TLSHandshakeTimeout: 10 * time.Second,
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -137,7 +134,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		tr.Proxy = http.ProxyURL(pu)
 	}
 	hc := &http.Client{
-		Transport: authCfg.NewRoundTripper(tr),
+		Transport: tr,
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
@@ -173,11 +170,12 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 }
 
 func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
-	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
 	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
-		c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
+		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
+	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
+
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
 	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
@@ -233,16 +231,10 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
 	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
 	if clientSecretFile != "" || clientSecret != "" {
-		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
-		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
-		}
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
 			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
-			EndpointParams:   endpointParams,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
 		}
@@ -310,7 +302,7 @@ func (c *client) runWorker() {
 				continue
 			}
 			// Return unsent block to the queue.
-			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			c.fq.MustWriteBlock(block)
 			return
 		case <-c.stopCh:
 			// c must be stopped. Wait for a while in the hope the block will be sent.
@@ -319,11 +311,11 @@ func (c *client) runWorker() {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
-					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+					c.fq.MustWriteBlock(block)
 				}
 			case <-time.After(graceDuration):
 				// Return unsent block to the queue.
-				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+				c.fq.MustWriteBlock(block)
 			}
 			return
 		}
@@ -331,42 +323,26 @@ func (c *client) runWorker() {
 }
 
 func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
-	req, err := c.newRequest(url, body)
-	if err != nil {
-		return nil, err
-	}
+	req := c.newRequest(url, body)
 	resp, err := c.hc.Do(req)
-	if err == nil {
-		return resp, nil
+	if err != nil && errors.Is(err, io.EOF) {
+		// it is likely connection became stale.
+		// So we do one more attempt in hope request will succeed.
+		// If not, the error should be handled by the caller as usual.
+		// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
+		req = c.newRequest(url, body)
+		resp, err = c.hc.Do(req)
 	}
-	if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
-		return nil, err
-	}
-	// It is likely connection became stale or timed out during the first request.
-	// Make another attempt in hope request will succeed.
-	// If not, the error should be handled by the caller as usual.
-	// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
-	req, err = c.newRequest(url, body)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	resp, err = c.hc.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	return resp, nil
+	return resp, err
 }
 
-func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
+func (c *client) newRequest(url string, body []byte) *http.Request {
 	reqBody := bytes.NewBuffer(body)
 	req, err := http.NewRequest(http.MethodPost, url, reqBody)
 	if err != nil {
 		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
 	}
-	err = c.authCfg.SetHeaders(req, true)
-	if err != nil {
-		return nil, err
-	}
+	c.authCfg.SetHeaders(req, true)
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
@@ -380,10 +356,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 	if c.awsCfg != nil {
 		sigv4Hash := awsapi.HashHex(body)
 		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
-			return nil, fmt.Errorf("cannot sign remoteWrite request with AWS sigv4: %w", err)
+			// there is no need in retry, request will be rejected by client.Do and retried by code below
+			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
 		}
 	}
-	return req, nil
+	return req
 }
 
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
@@ -391,9 +368,8 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // The function returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
-	retryDuration := timeutil.AddJitterToDuration(time.Second)
+	c.rl.register(len(block), c.stopCh)
+	retryDuration := time.Second
 	retriesCount := 0
 
 again:
@@ -403,8 +379,8 @@ again:
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > maxRetryDuration {
-			retryDuration = maxRetryDuration
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
 		}
 		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
@@ -450,8 +426,8 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
+	if retryDuration > time.Minute {
+		retryDuration = time.Minute
 	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
@@ -474,3 +450,45 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+
+type rateLimiter struct {
+	perSecondLimit int64
+
+	// mu protects budget and deadline from concurrent access.
+	mu sync.Mutex
+
+	// The current budget. It is increased by perSecondLimit every second.
+	budget int64
+
+	// The next deadline for increasing the budget by perSecondLimit
+	deadline time.Time
+
+	limitReached *metrics.Counter
+}
+
+func (rl *rateLimiter) register(dataLen int, stopCh <-chan struct{}) {
+	limit := rl.perSecondLimit
+	if limit <= 0 {
+		return
+	}
+
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+
+	for rl.budget <= 0 {
+		if d := time.Until(rl.deadline); d > 0 {
+			rl.limitReached.Inc()
+			t := timerpool.Get(d)
+			select {
+			case <-stopCh:
+				timerpool.Put(t)
+				return
+			case <-t.C:
+				timerpool.Put(t)
+			}
+		}
+		rl.budget += limit
+		rl.deadline = time.Now().Add(time.Second)
+	}
+	rl.budget -= int64(dataLen)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -16,7 +15,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -39,9 +37,9 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(pushBlock func(block []byte), isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
-	ps.wr.fq = fq
+	ps.wr.pushBlock = pushBlock
 	ps.wr.isVMRemoteWrite = isVMRemoteWrite
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
@@ -59,11 +57,10 @@ func (ps *pendingSeries) MustStop() {
 	ps.periodicFlusherWG.Wait()
 }
 
-func (ps *pendingSeries) TryPush(tss []prompbmarshal.TimeSeries) bool {
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
 	ps.mu.Lock()
-	ok := ps.wr.tryPush(tss)
+	ps.wr.push(tss)
 	ps.mu.Unlock()
-	return ok
 }
 
 func (ps *pendingSeries) periodicFlusher() {
@@ -71,54 +68,51 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	d := timeutil.AddJitterToDuration(*flushInterval)
-	ticker := time.NewTicker(d)
+	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
-	for {
+	mustStop := false
+	for !mustStop {
 		select {
 		case <-ps.stopCh:
-			ps.mu.Lock()
-			ps.wr.mustFlushOnStop()
-			ps.mu.Unlock()
-			return
+			mustStop = true
 		case <-ticker.C:
-			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime.Load() < uint64(flushSeconds) {
+			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
 				continue
 			}
 		}
 		ps.mu.Lock()
-		_ = ps.wr.tryFlush()
+		ps.wr.flush()
 		ps.mu.Unlock()
 	}
 }
 
 type writeRequest struct {
-	lastFlushTime atomic.Uint64
+	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
+	lastFlushTime uint64
 
-	// The queue to send blocks to.
-	fq *persistentqueue.FastQueue
+	// pushBlock is called when whe write request is ready to be sent.
+	pushBlock func(block []byte)
 
 	// Whether to encode the write request with VictoriaMetrics remote write protocol.
 	isVMRemoteWrite bool
 
-	// How many significant figures must be left before sending the writeRequest to fq.
+	// How many significant figures must be left before sending the writeRequest to pushBlock.
 	significantFigures int
 
-	// How many decimal digits after point must be left before sending the writeRequest to fq.
+	// How many decimal digits after point must be left before sending the writeRequest to pushBlock.
 	roundDigits int
 
 	wr prompbmarshal.WriteRequest
 
-	tss     []prompbmarshal.TimeSeries
+	tss []prompbmarshal.TimeSeries
+
 	labels  []prompbmarshal.Label
 	samples []prompbmarshal.Sample
-
-	// buf holds labels data
-	buf []byte
+	buf     []byte
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
+	// Do not reset lastFlushTime, pushBlock, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
 
 	wr.wr.Timeseries = nil
 
@@ -136,40 +130,23 @@ func (wr *writeRequest) reset() {
 	wr.buf = wr.buf[:0]
 }
 
-// mustFlushOnStop force pushes wr data into wr.fq
-//
-// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
-func (wr *writeRequest) mustFlushOnStop() {
+func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
-		logger.Panicf("BUG: final flush must always return true")
-	}
+	wr.adjustSampleValues()
+	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
+	pushWriteRequest(&wr.wr, wr.pushBlock, wr.isVMRemoteWrite)
 	wr.reset()
 }
 
-func (wr *writeRequest) mustWriteBlock(block []byte) bool {
-	wr.fq.MustWriteBlockIgnoreDisabledPQ(block)
-	return true
-}
-
-func (wr *writeRequest) tryFlush() bool {
-	wr.wr.Timeseries = wr.tss
-	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
-		return false
-	}
-	wr.reset()
-	return true
-}
-
-func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roundDigits int) {
-	if n := significantFigures; n > 0 {
+func (wr *writeRequest) adjustSampleValues() {
+	samples := wr.samples
+	if n := wr.significantFigures; n > 0 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToSignificantFigures(s.Value, n)
 		}
 	}
-	if n := roundDigits; n < 100 {
+	if n := wr.roundDigits; n < 100 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToDecimalDigits(s.Value, n)
@@ -177,27 +154,21 @@ func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roun
 	}
 }
 
-func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
 		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
-			if !wr.tryFlush() {
-				return false
-			}
+			wr.flush()
 			tssDst = wr.tss
 		}
-		tsSrc := &src[i]
-		adjustSampleValues(tsSrc.Samples, wr.significantFigures, wr.roundDigits)
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
-		wr.copyTimeSeries(&tssDst[len(tssDst)-1], tsSrc)
 	}
-
 	wr.tss = tssDst
-	return true
 }
 
 func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
@@ -225,45 +196,31 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
-// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
-var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
-
-func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte), isVMRemoteWrite bool) {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
-		return true
+		return
 	}
-
-	marshalConcurrencyCh <- struct{}{}
-
 	bb := writeRequestBufPool.Get()
-	bb.B = wr.MarshalProtobuf(bb.B[:0])
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := compressBufPool.Get()
+		zb := snappyBufPool.Get()
 		if isVMRemoteWrite {
 			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
 		} else {
 			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		}
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
-
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			zbLen := len(zb.B)
-			ok := tryPushBlock(zb.B)
-			compressBufPool.Put(zb)
-			if ok {
-				blockSizeRows.Update(float64(len(wr.Timeseries)))
-				blockSizeBytes.Update(float64(zbLen))
-			}
-			return ok
+			pushBlock(zb.B)
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return
 		}
-		compressBufPool.Put(zb)
+		snappyBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
 	}
 
 	// Too big block. Recursively split it into smaller parts if possible.
@@ -272,36 +229,23 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 		samples := wr.Timeseries[0].Samples
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
-			return true
+			return
 		}
 		n := len(samples) / 2
 		wr.Timeseries[0].Samples = samples[:n]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			return false
-		}
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		wr.Timeseries[0].Samples = samples[n:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			return false
-		}
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		wr.Timeseries[0].Samples = samples
-		return true
+		return
 	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 	wr.Timeseries = timeseries[n:]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 	wr.Timeseries = timeseries
-	return true
 }
 
 var (
@@ -309,7 +253,5 @@ var (
 	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var (
-	writeRequestBufPool bytesutil.ByteBufferPool
-	compressBufPool     bytesutil.ByteBufferPool
-)
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/pendingseries_test.go

@@ -26,16 +26,13 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 		t.Helper()
 		wr := newTestWriteRequest(rowsCount, 20)
 		pushBlockLen := 0
-		pushBlock := func(block []byte) bool {
+		pushBlock := func(block []byte) {
 			if pushBlockLen > 0 {
 				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
 			}
 			pushBlockLen = len(block)
-			return true
-		}
-		if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
-			t.Fatalf("cannot push data to to remote storage")
 		}
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
 		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
 			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
 				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
@@ -43,7 +40,7 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 	}
 
 	// Check Prometheus remote write
-	f(false, expectedBlockLenProm, 3)
+	f(false, expectedBlockLenProm, 0)
 
 	// Check VictoriaMetrics remote write
 	f(true, expectedBlockLenVM, 15)

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/golang/snappy"
 	"github.com/klauspost/compress/s2"
 )
@@ -21,7 +22,7 @@ func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []by
 	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
 		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
 			wr := newTestWriteRequest(rowsCount, 10)
-			data := wr.MarshalProtobuf(nil)
+			data := prompbmarshal.MarshalWriteRequest(nil, wr)
 			b.ReportAllocs()
 			b.SetBytes(int64(rowsCount))
 			b.RunParallel(func(pb *testing.PB) {

app/vmagent/remotewrite/relabel.go

@@ -3,7 +3,6 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -93,7 +92,6 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *pro
 		// Nothing to change.
 		return tss
 	}
-	rctx.reset()
 	tssDst := tss[:0]
 	labels := rctx.labels[:0]
 	for i := range tss {
@@ -122,7 +120,6 @@ func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraL
 	if len(extraLabels) == 0 {
 		return
 	}
-	rctx.reset()
 	labels := rctx.labels[:0]
 	for i := range tss {
 		ts := &tss[i]
@@ -142,34 +139,6 @@ func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraL
 	rctx.labels = labels
 }
 
-func (rctx *relabelCtx) tenantToLabels(tss []prompbmarshal.TimeSeries, accountID, projectID uint32) {
-	rctx.reset()
-	accountIDStr := strconv.FormatUint(uint64(accountID), 10)
-	projectIDStr := strconv.FormatUint(uint64(projectID), 10)
-	labels := rctx.labels[:0]
-	for i := range tss {
-		ts := &tss[i]
-		labelsLen := len(labels)
-		for _, label := range ts.Labels {
-			labelName := label.Name
-			if labelName == "vm_account_id" || labelName == "vm_project_id" {
-				continue
-			}
-			labels = append(labels, label)
-		}
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "vm_account_id",
-			Value: accountIDStr,
-		})
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "vm_project_id",
-			Value: projectIDStr,
-		})
-		ts.Labels = labels[labelsLen:]
-	}
-	rctx.labels = labels
-}
-
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
 	labels []prompbmarshal.Label
@@ -191,7 +160,7 @@ func getRelabelCtx() *relabelCtx {
 }
 
 func putRelabelCtx(rctx *relabelCtx) {
-	rctx.reset()
+	rctx.labels = rctx.labels[:0]
 	relabelCtxPool.Put(rctx)
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -3,7 +3,6 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"net/url"
 	"path/filepath"
 	"strconv"
@@ -11,8 +10,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -26,8 +23,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -38,30 +33,19 @@ var (
 	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
 		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
 		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
-		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set")
+		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set. "+
+		"See also -remoteWrite.multitenantURL")
 	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
-		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. "+
-		"This flag is deprecated in favor of -enableMultitenantHandlers . See https://docs.victoriametrics.com/vmagent.html#multitenancy")
-	enableMultitenantHandlers = flag.Bool("enableMultitenantHandlers", false, "Whether to process incoming data via multitenant insert handlers according to "+
-		"https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format . By default incoming data is processed via single-node insert handlers "+
-		"according to https://docs.victoriametrics.com/#how-to-import-time-series-data ."+
-		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details")
+		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
 	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
 		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages")
-	shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
-		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
-		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.ignoreLabels")
-	shardByURLIgnoreLabels = flagutil.NewArrayString("remoteWrite.shardByURL.ignoreLabels", "Optional list of labels, which must be ignored when sharding outgoing samples "+
-		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
-		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.labels")
-	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
-		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
+	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
+		"See also -remoteWrite.maxDiskUsagePerURL")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
+		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
 	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
@@ -84,8 +68,6 @@ var (
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
 		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
-	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
-		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
@@ -96,18 +78,8 @@ var (
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation.html#deduplication")
-	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current aggregation interval "+
-		"for the corresponding -remoteWrite.streamAggr.config . See https://docs.victoriametrics.com/stream-aggregation.html#ignoring-old-samples")
-	streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation.html#dropping-unneeded-labels")
-
-	disableOnDiskQueue = flag.Bool("remoteWrite.disableOnDiskQueue", false, "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence ."+
-		"See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence")
+	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before being aggregated. "+
+		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
 )
 
 var (
@@ -120,19 +92,11 @@ var (
 
 	// Data without tenant id is written to defaultAuthToken if -remoteWrite.multitenantURL is specified.
 	defaultAuthToken = &auth.Token{}
-
-	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
-	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
-		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
-			"or remove -remoteWrite.disableOnDiskQueue from vmagent command-line flags, so it could save pending data to -remoteWrite.tmpDataPath; " +
-			"see https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence"),
-		StatusCode: http.StatusTooManyRequests,
-	}
 )
 
-// MultitenancyEnabled returns true if -enableMultitenantHandlers or -remoteWrite.multitenantURL is specified.
+// MultitenancyEnabled returns true if -remoteWrite.multitenantURL is specified.
 func MultitenancyEnabled() bool {
-	return *enableMultitenantHandlers || len(*remoteWriteMultitenantURLs) > 0
+	return len(*remoteWriteMultitenantURLs) > 0
 }
 
 // Contains the current relabelConfigs.
@@ -152,11 +116,6 @@ func InitSecretFlags() {
 	}
 }
 
-var (
-	shardByURLLabelsMap       map[string]struct{}
-	shardByURLIgnoreLabelsMap map[string]struct{}
-)
-
 // Init initializes remotewrite.
 //
 // It must be called after flag.Parse().
@@ -187,21 +146,12 @@ func Init() {
 			return float64(dailySeriesLimiter.CurrentItems())
 		})
 	}
-
 	if *queues > maxQueues {
 		*queues = maxQueues
 	}
 	if *queues <= 0 {
 		*queues = 1
 	}
-
-	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
-		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
-			"see https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
-	}
-	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
-	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
-
 	initLabelsGlobal()
 
 	// Register SIGHUP handler for config reload before loadRelabelConfigs.
@@ -220,7 +170,6 @@ func Init() {
 	if len(*remoteWriteURLs) > 0 {
 		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 	}
-	dropDanglingQueues()
 
 	// Start config reloader.
 	configReloaderWG.Add(1)
@@ -238,42 +187,6 @@ func Init() {
 	}()
 }
 
-func dropDanglingQueues() {
-	if *keepDanglingQueues {
-		return
-	}
-	if len(*remoteWriteMultitenantURLs) > 0 {
-		// Do not drop dangling queues for *remoteWriteMultitenantURLs, since it is impossible to determine
-		// unused queues for multitenant urls - they are created on demand when new sample for the given
-		// tenant is pushed to remote storage.
-		return
-	}
-	// Remove dangling persistent queues, if any.
-	// This is required for the case when the number of queues has been changed or URL have been changed.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
-	//
-	existingQueues := make(map[string]struct{}, len(rwctxsDefault))
-	for _, rwctx := range rwctxsDefault {
-		existingQueues[rwctx.fq.Dirname()] = struct{}{}
-	}
-
-	queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
-	files := fs.MustReadDir(queuesDir)
-	removed := 0
-	for _, f := range files {
-		dirname := f.Name()
-		if _, ok := existingQueues[dirname]; !ok {
-			logger.Infof("removing dangling queue %q", dirname)
-			fullPath := filepath.Join(queuesDir, dirname)
-			fs.MustRemoveAll(fullPath)
-			removed++
-		}
-	}
-	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsDefault))
-	}
-}
-
 func reloadRelabelConfigs() {
 	relabelConfigReloads.Inc()
 	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
@@ -293,7 +206,7 @@ func reloadRelabelConfigs() {
 var (
 	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
 	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess      = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
+	relabelConfigSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
 	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
@@ -347,44 +260,42 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		}
 		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
+
+	if !*keepDanglingQueues {
+		// Remove dangling queues, if any.
+		// This is required for the case when the number of queues has been changed or URL have been changed.
+		// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
+		existingQueues := make(map[string]struct{}, len(rwctxs))
+		for _, rwctx := range rwctxs {
+			existingQueues[rwctx.fq.Dirname()] = struct{}{}
+		}
+
+		queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
+		files := fs.MustReadDir(queuesDir)
+		removed := 0
+		for _, f := range files {
+			dirname := f.Name()
+			if _, ok := existingQueues[dirname]; !ok {
+				logger.Infof("removing dangling queue %q", dirname)
+				fullPath := filepath.Join(queuesDir, dirname)
+				fs.MustRemoveAll(fullPath)
+				removed++
+			}
+		}
+		if removed > 0 {
+			logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
+		}
+	}
+
 	return rwctxs
 }
 
 var configReloaderStopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
-// StartIngestionRateLimiter starts ingestion rate limiter.
-//
-// Ingestion rate limiter must be started before Init() call.
-//
-// StopIngestionRateLimiter must be called before Stop() call in order to unblock all the callers
-// to ingestion rate limiter. Otherwise deadlock may occur at Stop() call.
-func StartIngestionRateLimiter() {
-	if *maxIngestionRate <= 0 {
-		return
-	}
-	ingestionRateLimitReached := metrics.NewCounter(`vmagent_max_ingestion_rate_limit_reached_total`)
-	ingestionRateLimiterStopCh = make(chan struct{})
-	ingestionRateLimiter = ratelimiter.New(int64(*maxIngestionRate), ingestionRateLimitReached, ingestionRateLimiterStopCh)
-}
-
-// StopIngestionRateLimiter stops ingestion rate limiter.
-func StopIngestionRateLimiter() {
-	if ingestionRateLimiterStopCh == nil {
-		return
-	}
-	close(ingestionRateLimiterStopCh)
-	ingestionRateLimiterStopCh = nil
-}
-
-var (
-	ingestionRateLimiter       *ratelimiter.RateLimiter
-	ingestionRateLimiterStopCh chan struct{}
-)
-
 // Stop stops remotewrite.
 //
-// It is expected that nobody calls TryPush during and after the call to this func.
+// It is expected that nobody calls Push during and after the call to this func.
 func Stop() {
 	close(configReloaderStopCh)
 	configReloaderWG.Wait()
@@ -394,7 +305,7 @@ func Stop() {
 	}
 	rwctxsDefault = nil
 
-	// There is no need in locking rwctxsMapLock here, since nobody should call TryPush during the Stop call.
+	// There is no need in locking rwctxsMapLock here, since nobody should call Push during the Stop call.
 	for _, rwctxs := range rwctxsMap {
 		for _, rwctx := range rwctxs {
 			rwctx.MustStop()
@@ -410,47 +321,24 @@ func Stop() {
 	}
 }
 
-// PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
+// Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
-// If at is nil, then the data is pushed to the configured -remoteWrite.url.
-// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
+// If at is nil, then the data is pushed to the configured `-remoteWrite.url`.
+// If at isn't nil, the data is pushed to the configured `-remoteWrite.multitenantURL`.
 //
-// PushDropSamplesOnFailure can modify wr contents.
-func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
-	_ = tryPush(at, wr, true)
-}
-
-// TryPush tries sending wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
-//
-// If at is nil, then the data is pushed to the configured -remoteWrite.url.
-// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
-//
-// TryPush can modify wr contents, so the caller must re-initialize wr before calling TryPush() after unsuccessful attempt.
-// TryPush may send partial data from wr on unsuccessful attempt, so repeated call for the same wr may send the data multiple times.
-//
-// The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
-func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, *dropSamplesOnOverload)
-}
-
-func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailure bool) bool {
-	tss := wr.Timeseries
-
-	if at == nil && MultitenancyEnabled() {
-		// Write data to default tenant if at isn't set when multitenancy is enabled.
+// Note that wr may be modified by Push because of relabeling and rounding.
+func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
+		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
 		at = defaultAuthToken
 	}
-
-	var tenantRctx *relabelCtx
 	var rwctxs []*remoteWriteCtx
 	if at == nil {
 		rwctxs = rwctxsDefault
-	} else if len(*remoteWriteMultitenantURLs) == 0 {
-		// Convert at to (vm_account_id, vm_project_id) labels.
-		tenantRctx = getRelabelCtx()
-		defer putRelabelCtx(tenantRctx)
-		rwctxs = rwctxsDefault
 	} else {
+		if len(*remoteWriteMultitenantURLs) == 0 {
+			logger.Panicf("BUG: -remoteWrite.multitenantURL command-line flag must be set when __tenant_id__=%q label is set", at)
+		}
 		rwctxsMapLock.Lock()
 		tenantID := tenantmetrics.TenantID{
 			AccountID: at.AccountID,
@@ -464,37 +352,18 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 		rwctxsMapLock.Unlock()
 	}
 
-	rowsCount := getRowsCount(tss)
-
-	if *disableOnDiskQueue {
-		// Quick check whether writes to configured remote storage systems are blocked.
-		// This allows saving CPU time spent on relabeling and block compression
-		// if some of remote storage systems cannot keep up with the data ingestion rate.
-		for _, rwctx := range rwctxs {
-			if rwctx.fq.IsWriteBlocked() {
-				pushFailures.Inc()
-				if dropSamplesOnFailure {
-					// Just drop samples
-					samplesDropped.Add(rowsCount)
-					return true
-				}
-				return false
-			}
-		}
-	}
-
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
 	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
-		defer putRelabelCtx(rctx)
 	}
+	tss := wr.Timeseries
+	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
-
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -502,15 +371,12 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 		i := 0
 		for i < len(tss) {
 			samplesCount += len(tss[i].Samples)
-			labelsCount += len(tss[i].Samples) * len(tss[i].Labels)
+			labelsCount += len(tss[i].Labels)
 			i++
 			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
 				break
 			}
 		}
-
-		ingestionRateLimiter.Register(samplesCount)
-
 		tssBlock := tss
 		if i < len(tss) {
 			tssBlock = tss[:i]
@@ -518,9 +384,6 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 		} else {
 			tss = nil
 		}
-		if tenantRctx != nil {
-			tenantRctx.tenantToLabels(tssBlock, at.AccountID, at.ProjectID)
-		}
 		if rctx != nil {
 			rowsCountBeforeRelabel := getRowsCount(tssBlock)
 			tssBlock = rctx.applyRelabeling(tssBlock, pcsGlobal)
@@ -529,35 +392,25 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock) {
-			if !*disableOnDiskQueue {
-				logger.Panicf("BUG: tryPushBlockToRemoteStorages must return true if -remoteWrite.disableOnDiskQueue isn't set")
-			}
-			pushFailures.Inc()
-			if dropSamplesOnFailure {
-				samplesDropped.Add(rowsCount)
-				return true
-			}
-			return false
+		pushBlockToRemoteStorages(rwctxs, tssBlock)
+		if rctx != nil {
+			rctx.reset()
 		}
 	}
-	return true
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
 }
 
-var (
-	samplesDropped = metrics.NewCounter(`vmagent_remotewrite_samples_dropped_total`)
-	pushFailures   = metrics.NewCounter(`vmagent_remotewrite_push_failures_total`)
-)
-
-func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) bool {
+func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
 	if len(tssBlock) == 0 {
 		// Nothing to push
-		return true
+		return
 	}
-
 	if len(rwctxs) == 1 {
 		// Fast path - just push data to the configured single remote storage
-		return rwctxs[0].TryPush(tssBlock)
+		rwctxs[0].Push(tssBlock)
+		return
 	}
 
 	// We need to push tssBlock to multiple remote storages.
@@ -565,51 +418,27 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 	if *shardByURL {
 		// Shard the data among rwctxs
 		tssByURL := make([][]prompbmarshal.TimeSeries, len(rwctxs))
-		tmpLabels := promutils.GetLabels()
 		for _, ts := range tssBlock {
-			hashLabels := ts.Labels
-			if len(shardByURLLabelsMap) > 0 {
-				hashLabels = tmpLabels.Labels[:0]
-				for _, label := range ts.Labels {
-					if _, ok := shardByURLLabelsMap[label.Name]; ok {
-						hashLabels = append(hashLabels, label)
-					}
-				}
-				tmpLabels.Labels = hashLabels
-			} else if len(shardByURLIgnoreLabelsMap) > 0 {
-				hashLabels = tmpLabels.Labels[:0]
-				for _, label := range ts.Labels {
-					if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
-						hashLabels = append(hashLabels, label)
-					}
-				}
-				tmpLabels.Labels = hashLabels
-			}
-			h := getLabelsHash(hashLabels)
+			h := getLabelsHash(ts.Labels)
 			idx := h % uint64(len(tssByURL))
 			tssByURL[idx] = append(tssByURL[idx], ts)
 		}
-		promutils.PutLabels(tmpLabels)
-
 		// Push sharded data to remote storages in parallel in order to reduce
 		// the time needed for sending the data to multiple remote storage systems.
 		var wg sync.WaitGroup
-		var anyPushFailed atomic.Bool
+		wg.Add(len(rwctxs))
 		for i, rwctx := range rwctxs {
 			tssShard := tssByURL[i]
 			if len(tssShard) == 0 {
 				continue
 			}
-			wg.Add(1)
 			go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
 				defer wg.Done()
-				if !rwctx.TryPush(tss) {
-					anyPushFailed.Store(true)
-				}
+				rwctx.Push(tss)
 			}(rwctx, tssShard)
 		}
 		wg.Wait()
-		return !anyPushFailed.Load()
+		return
 	}
 
 	// Replicate data among rwctxs.
@@ -617,17 +446,13 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
 	wg.Add(len(rwctxs))
-	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
-			if !rwctx.TryPush(tssBlock) {
-				anyPushFailed.Store(true)
-			}
+			rwctx.Push(tssBlock)
 		}(rwctx)
 	}
 	wg.Wait()
-	return !anyPushFailed.Load()
 }
 
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
@@ -723,14 +548,12 @@ type remoteWriteCtx struct {
 	fq  *persistentqueue.FastQueue
 	c   *client
 
-	sas          atomic.Pointer[streamaggr.Aggregators]
-	deduplicator *streamaggr.Deduplicator
-
+	sas                 atomic.Pointer[streamaggr.Aggregators]
 	streamAggrKeepInput bool
 	streamAggrDropInput bool
 
 	pss        []*pendingSeries
-	pssNextIdx atomic.Uint64
+	pssNextIdx uint64
 
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
@@ -749,19 +572,13 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, *disableOnDiskQueue)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queue_blocked{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
-		if fq.IsWriteBlocked() {
-			return 1
-		}
-		return 0
-	})
 
 	var c *client
 	switch remoteWriteURL.Scheme {
@@ -783,7 +600,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
+		pss[i] = newPendingSeries(fq.MustWriteBlock, c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{
@@ -798,15 +615,9 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 
 	// Initialize sas
 	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
-	ignoreOldSamples := streamAggrIgnoreOldSamples.GetOptionalArg(argIdx)
 	if sasFile != "" {
-		opts := &streamaggr.Options{
-			DedupInterval:    dedupInterval,
-			DropInputLabels:  *streamAggrDropInputLabels,
-			IgnoreOldSamples: ignoreOldSamples,
-		}
-		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
+		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
 		if err != nil {
 			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
 		}
@@ -815,24 +626,17 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
-	} else if dedupInterval > 0 {
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels)
 	}
 
 	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
-	// sas and deduplicator must be stopped before rwctx is closed
+	// sas must be stopped before rwctx is closed
 	// because sas can write pending series to rwctx.pss if there are any
 	sas := rwctx.sas.Swap(nil)
 	sas.MustStop()
 
-	if rwctx.deduplicator != nil {
-		rwctx.deduplicator.MustStop()
-		rwctx.deduplicator = nil
-	}
-
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
 	}
@@ -849,7 +653,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.rowsDroppedByRelabel = nil
 }
 
-func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
+func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
@@ -871,7 +675,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 	rowsCount := getRowsCount(tss)
 	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
 
-	// Apply stream aggregation or deduplication if they are configured
+	// Apply stream aggregation if any
 	sas := rwctx.sas.Load()
 	if sas != nil {
 		matchIdxs := matchIdxsPool.Get()
@@ -886,14 +690,8 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		}
 		matchIdxsPool.Put(matchIdxs)
-	} else if rwctx.deduplicator != nil {
-		rwctx.deduplicator.Push(tss)
-		clear(tss)
-		tss = tss[:0]
 	}
-
-	// Try pushing the data to remote storage
-	ok := rwctx.tryPushInternal(tss)
+	rwctx.pushInternal(tss)
 
 	// Return back relabeling contexts to the pool
 	if rctx != nil {
@@ -901,8 +699,6 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 		tssPool.Put(v)
 		putRelabelCtx(rctx)
 	}
-
-	return ok
 }
 
 var matchIdxsPool bytesutil.ByteBufferPool
@@ -918,25 +714,11 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 		}
 	}
 	tail := src[len(dst):]
-	clear(tail)
+	_ = prompbmarshal.ResetTimeSeries(tail)
 	return dst
 }
 
-func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSeries) {
-	if rwctx.tryPushInternal(tss) {
-		return
-	}
-	if !*disableOnDiskQueue {
-		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
-	}
-	pushFailures.Inc()
-	if *dropSamplesOnOverload {
-		rowsCount := getRowsCount(tss)
-		samplesDropped.Add(rowsCount)
-	}
-}
-
-func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
+func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
 	if len(labelsGlobal) > 0 {
@@ -949,41 +731,34 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 	}
 
 	pss := rwctx.pss
-	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
-
-	ok := pss[idx].TryPush(tss)
+	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	pss[idx].Push(tss)
 
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
 		tssPool.Put(v)
 		putRelabelCtx(rctx)
 	}
-
-	return ok
 }
 
 func (rwctx *remoteWriteCtx) reinitStreamAggr() {
-	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if sasFile == "" {
+	sas := rwctx.sas.Load()
+	if sas == nil {
 		// There is no stream aggregation for rwctx
 		return
 	}
 
+	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
-	opts := &streamaggr.Options{
-		DedupInterval:    streamAggrDedupInterval.GetOptionalArg(rwctx.idx),
-		DropInputLabels:  *streamAggrDropInputLabels,
-		IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(rwctx.idx),
-	}
-	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
+	dedupInterval := streamAggrDedupInterval.GetOptionalArg(rwctx.idx)
+	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
 		logger.Errorf("cannot reload stream aggregation config from -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", sasFile, err)
 		return
 	}
-	sas := rwctx.sas.Load()
 	if !sasNew.Equal(sas) {
 		sasOld := rwctx.sas.Swap(sasNew)
 		sasOld.MustStop()
@@ -1013,17 +788,13 @@ func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 
 // CheckStreamAggrConfigs checks configs pointed by -remoteWrite.streamAggr.config
 func CheckStreamAggrConfigs() error {
-	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
 	for idx, sasFile := range *streamAggrConfig {
 		if sasFile == "" {
 			continue
 		}
-		opts := &streamaggr.Options{
-			DedupInterval:    streamAggrDedupInterval.GetOptionalArg(idx),
-			DropInputLabels:  *streamAggrDropInputLabels,
-			IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		}
-		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, opts)
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, dedupInterval)
 		if err != nil {
 			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)
 		}
@@ -1031,11 +802,3 @@ func CheckStreamAggrConfigs() error {
 	}
 	return nil
 }
-
-func newMapFromStrings(a []string) map[string]struct{} {
-	m := make(map[string]struct{}, len(a))
-	for _, s := range a {
-		m[s] = struct{}{}
-	}
-	return m
-}

app/vmagent/remotewrite/statconn.go

@@ -50,7 +50,7 @@ var (
 )
 
 type statConn struct {
-	closed atomic.Int32
+	closed uint64
 	net.Conn
 }
 
@@ -76,7 +76,7 @@ func (sc *statConn) Write(p []byte) (int, error) {
 
 func (sc *statConn) Close() error {
 	err := sc.Conn.Close()
-	if sc.closed.Add(1) == 1 {
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
 		conns.Dec()
 	}
 	return err

app/vmagent/vmimport/request_handler.go

@@ -76,9 +76,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmalert-tool/Makefile

@@ -1,103 +0,0 @@
-# All these commands must run from repository root.
-
-vmalert-tool:
-	APP_NAME=vmalert-tool $(MAKE) app-local
-
-vmalert-tool-race:
-	APP_NAME=vmalert-tool RACE=-race $(MAKE) app-local
-
-vmalert-tool-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker
-
-vmalert-tool-pure-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-pure
-
-vmalert-tool-linux-amd64-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-amd64
-
-vmalert-tool-linux-arm-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm
-
-vmalert-tool-linux-arm64-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm64
-
-vmalert-tool-linux-ppc64le-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-ppc64le
-
-vmalert-tool-linux-386-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-386
-
-vmalert-tool-darwin-amd64-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-amd64
-
-vmalert-tool-darwin-arm64-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-arm64
-
-vmalert-tool-freebsd-amd64-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-freebsd-amd64
-
-vmalert-tool-openbsd-amd64-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-openbsd-amd64
-
-vmalert-tool-windows-amd64-prod:
-	APP_NAME=vmalert-tool $(MAKE) app-via-docker-windows-amd64
-
-package-vmalert-tool:
-	APP_NAME=vmalert-tool $(MAKE) package-via-docker
-
-package-vmalert-tool-pure:
-	APP_NAME=vmalert-tool $(MAKE) package-via-docker-pure
-
-package-vmalert-tool-amd64:
-	APP_NAME=vmalert-tool $(MAKE) package-via-docker-amd64
-
-package-vmalert-tool-arm:
-	APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm
-
-package-vmalert-tool-arm64:
-	APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm64
-
-package-vmalert-tool-ppc64le:
-	APP_NAME=vmalert-tool $(MAKE) package-via-docker-ppc64le
-
-package-vmalert-tool-386:
-	APP_NAME=vmalert-tool $(MAKE) package-via-docker-386
-
-publish-vmalert-tool:
-	APP_NAME=vmalert-tool $(MAKE) publish-via-docker
-
-vmalert-tool-linux-amd64:
-	APP_NAME=vmalert-tool CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmalert-tool-linux-arm:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-vmalert-tool-linux-arm64:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vmalert-tool-linux-ppc64le:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vmalert-tool-linux-s390x:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vmalert-tool-linux-386:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vmalert-tool-darwin-amd64:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmalert-tool-darwin-arm64:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vmalert-tool-freebsd-amd64:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmalert-tool-openbsd-amd64:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmalert-tool-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmalert-tool $(MAKE) app-local-windows-goarch
-
-vmalert-tool-pure:
-	APP_NAME=vmalert-tool $(MAKE) app-local-pure

app/vmalert-tool/README.md

@@ -1,3 +0,0 @@
-See vmalert-tool docs [here](https://docs.victoriametrics.com/vmalert-tool.html).
-
-vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert-tool.md).

app/vmalert-tool/main.go

@@ -1,54 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"time"
-
-	"github.com/urfave/cli/v2"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert-tool/unittest"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-)
-
-func main() {
-	start := time.Now()
-	app := &cli.App{
-		Name:      "vmalert-tool",
-		Usage:     "VMAlert command-line tool",
-		UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html",
-		Version:   buildinfo.Version,
-		Commands: []*cli.Command{
-			{
-				Name:      "unittest",
-				Usage:     "Run unittest for alerting and recording rules.",
-				UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html#Unit-testing-for-rules",
-				Flags: []cli.Flag{
-					&cli.StringSliceFlag{
-						Name:     "files",
-						Usage:    "files to run unittest with. Supports an array of values separated by comma or specified via multiple flags.",
-						Required: true,
-					},
-					&cli.BoolFlag{
-						Name:     "disableAlertgroupLabel",
-						Usage:    "disable adding group's Name as label to generated alerts and time series.",
-						Required: false,
-					},
-				},
-				Action: func(c *cli.Context) error {
-					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
-						return fmt.Errorf("unittest failed")
-					}
-					return nil
-				},
-			},
-		},
-	}
-
-	err := app.Run(os.Args)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	log.Printf("Total time: %v", time.Since(start))
-}

app/vmalert-tool/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
-FROM $certs_image as certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 8429
-ENTRYPOINT ["/vmalert-tool-prod"]
-ARG TARGETARCH
-COPY vmalert-tool-linux-${TARGETARCH}-prod ./vmalert-tool-prod

app/vmalert-tool/unittest/alerting.go

@@ -1,19 +0,0 @@
-package unittest
-
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-// alertTestCase holds alert_rule_test cases defined in test file
-type alertTestCase struct {
-	EvalTime  *promutils.Duration `yaml:"eval_time"`
-	GroupName string              `yaml:"groupname"`
-	Alertname string              `yaml:"alertname"`
-	ExpAlerts []expAlert          `yaml:"exp_alerts"`
-}
-
-// expAlert holds exp_alerts defined in test file
-type expAlert struct {
-	ExpLabels      map[string]string `yaml:"exp_labels"`
-	ExpAnnotations map[string]string `yaml:"exp_annotations"`
-}

app/vmalert-tool/unittest/input.go

@@ -1,182 +0,0 @@
-package unittest
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"github.com/VictoriaMetrics/metricsql"
-)
-
-// series holds input_series defined in the test file
-type series struct {
-	Series string `yaml:"series"`
-	Values string `yaml:"values"`
-}
-
-// sequenceValue is an omittable value in a sequence of time series values.
-type sequenceValue struct {
-	Value   float64
-	Omitted bool
-}
-
-func httpWrite(address string, r io.Reader) {
-	resp, err := http.Post(address, "", r)
-	if err != nil {
-		logger.Fatalf("failed to send to storage: %v", err)
-	}
-	resp.Body.Close()
-}
-
-// writeInputSeries send input series to vmstorage and flush them
-func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error {
-	r := testutil.WriteRequest{}
-	for _, data := range input {
-		expr, err := metricsql.Parse(data.Series)
-		if err != nil {
-			return fmt.Errorf("failed to parse series %s: %v", data.Series, err)
-		}
-		promvals, err := parseInputValue(data.Values, true)
-		if err != nil {
-			return fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
-		}
-		metricExpr, ok := expr.(*metricsql.MetricExpr)
-		if !ok {
-			return fmt.Errorf("failed to parse series %s to metric expr: %v", data.Series, err)
-		}
-		samples := make([]testutil.Sample, 0, len(promvals))
-		ts := startStamp
-		for _, v := range promvals {
-			if !v.Omitted {
-				samples = append(samples, testutil.Sample{
-					Timestamp: ts.UnixMilli(),
-					Value:     v.Value,
-				})
-			}
-			ts = ts.Add(interval.Duration())
-		}
-		var ls []testutil.Label
-		for _, filter := range metricExpr.LabelFilterss[0] {
-			ls = append(ls, testutil.Label{Name: filter.Label, Value: filter.Value})
-		}
-		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
-	}
-
-	data, err := testutil.Compress(r)
-	if err != nil {
-		return fmt.Errorf("failed to compress data: %v", err)
-	}
-	// write input series to vm
-	httpWrite(dst, bytes.NewBuffer(data))
-	vmstorage.Storage.DebugFlush()
-	return nil
-}
-
-// parseInputValue support input like "1", "1+1x1 _ -4 3+20x1", see more examples in test.
-func parseInputValue(input string, origin bool) ([]sequenceValue, error) {
-	var res []sequenceValue
-	items := strings.Split(input, " ")
-	reg := regexp.MustCompile(`\D?\d*\D?`)
-	for _, item := range items {
-		if item == "stale" {
-			res = append(res, sequenceValue{Value: decimal.StaleNaN})
-			continue
-		}
-		vals := reg.FindAllString(item, -1)
-		switch len(vals) {
-		case 1:
-			if vals[0] == "_" {
-				res = append(res, sequenceValue{Omitted: true})
-				continue
-			}
-			v, err := strconv.ParseFloat(vals[0], 64)
-			if err != nil {
-				return nil, err
-			}
-			res = append(res, sequenceValue{Value: v})
-			continue
-		case 2:
-			p1 := vals[0][:len(vals[0])-1]
-			v2, err := strconv.ParseInt(vals[1], 10, 64)
-			if err != nil {
-				return nil, err
-			}
-			option := vals[0][len(vals[0])-1]
-			switch option {
-			case '+':
-				v1, err := strconv.ParseFloat(p1, 64)
-				if err != nil {
-					return nil, err
-				}
-				res = append(res, sequenceValue{Value: v1 + float64(v2)})
-			case 'x':
-				for i := int64(0); i <= v2; i++ {
-					if p1 == "_" {
-						if i == 0 {
-							i = 1
-						}
-						res = append(res, sequenceValue{Omitted: true})
-						continue
-					}
-					v1, err := strconv.ParseFloat(p1, 64)
-					if err != nil {
-						return nil, err
-					}
-					if !origin || v1 == 0 {
-						res = append(res, sequenceValue{Value: v1 * float64(i)})
-						continue
-					}
-					newVal := fmt.Sprintf("%s+0x%s", p1, vals[1])
-					newRes, err := parseInputValue(newVal, false)
-					if err != nil {
-						return nil, err
-					}
-					res = append(res, newRes...)
-					break
-				}
-
-			default:
-				return nil, fmt.Errorf("got invalid operation %b", option)
-			}
-		case 3:
-			r1, err := parseInputValue(fmt.Sprintf("%s%s", vals[1], vals[2]), false)
-			if err != nil {
-				return nil, err
-			}
-			p1 := vals[0][:len(vals[0])-1]
-			v1, err := strconv.ParseFloat(p1, 64)
-			if err != nil {
-				return nil, err
-			}
-			option := vals[0][len(vals[0])-1]
-			var isAdd bool
-			if option == '+' {
-				isAdd = true
-			}
-			for _, r := range r1 {
-				if isAdd {
-					res = append(res, sequenceValue{
-						Value: r.Value + v1,
-					})
-				} else {
-					res = append(res, sequenceValue{
-						Value: v1 - r.Value,
-					})
-				}
-			}
-		default:
-			return nil, fmt.Errorf("unsupported input %s", input)
-		}
-	}
-	return res, nil
-}

app/vmalert-tool/unittest/input_test.go

@@ -1,93 +0,0 @@
-package unittest
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-)
-
-func TestParseInputValue(t *testing.T) {
-	testCases := []struct {
-		input  string
-		exp    []sequenceValue
-		failed bool
-	}{
-		{
-			"",
-			nil,
-			true,
-		},
-		{
-			"testfailed",
-			nil,
-			true,
-		},
-		// stale doesn't support operations
-		{
-			"stalex3",
-			nil,
-			true,
-		},
-		{
-			"-4",
-			[]sequenceValue{{Value: -4}},
-			false,
-		},
-		{
-			"_",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"stale",
-			[]sequenceValue{{Value: decimal.StaleNaN}},
-			false,
-		},
-		{
-			"-4x1",
-			[]sequenceValue{{Value: -4}, {Value: -4}},
-			false,
-		},
-		{
-			"_x1",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"1+1x4",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 4}, {Value: 5}},
-			false,
-		},
-		{
-			"2-1x4",
-			[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
-			false,
-		},
-		{
-			"1+1x1 _ -4 stale 3+20x1",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
-			false,
-		},
-	}
-
-	for _, tc := range testCases {
-		output, err := parseInputValue(tc.input, true)
-		if err != nil != tc.failed {
-			t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
-		}
-		if len(tc.exp) != len(output) {
-			t.Fatalf("expect %v, got %v", tc.exp, output)
-		}
-		for i := 0; i < len(tc.exp); i++ {
-			if tc.exp[i].Omitted != output[i].Omitted {
-				t.Fatalf("expect %v, got %v", tc.exp, output)
-			}
-			if tc.exp[i].Value != output[i].Value {
-				if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
-					continue
-				}
-				t.Fatalf("expect %v, got %v", tc.exp, output)
-			}
-		}
-	}
-}

app/vmalert-tool/unittest/recording.go

@@ -1,100 +0,0 @@
-package unittest
-
-import (
-	"context"
-	"fmt"
-	"net/url"
-	"reflect"
-	"sort"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"github.com/VictoriaMetrics/metricsql"
-)
-
-// metricsqlTestCase holds metricsql_expr_test cases defined in test file
-type metricsqlTestCase struct {
-	Expr       string              `yaml:"expr"`
-	EvalTime   *promutils.Duration `yaml:"eval_time"`
-	ExpSamples []expSample         `yaml:"exp_samples"`
-}
-
-type expSample struct {
-	Labels string  `yaml:"labels"`
-	Value  float64 `yaml:"value"`
-}
-
-// checkMetricsqlCase will check metricsql_expr_test cases
-func checkMetricsqlCase(cases []metricsqlTestCase, q datasource.QuerierBuilder) (checkErrs []error) {
-	queries := q.BuildWithParams(datasource.QuerierParams{QueryParams: url.Values{"nocache": {"1"}, "latency_offset": {"1ms"}}, DataSourceType: "prometheus"})
-Outer:
-	for _, mt := range cases {
-		result, _, err := queries.Query(context.Background(), mt.Expr, durationToTime(mt.EvalTime))
-		if err != nil {
-			checkErrs = append(checkErrs, fmt.Errorf("    expr: %q, time: %s, err: %w", mt.Expr,
-				mt.EvalTime.Duration().String(), err))
-			continue
-		}
-		var gotSamples []parsedSample
-		for _, s := range result.Data {
-			sort.Slice(s.Labels, func(i, j int) bool {
-				return s.Labels[i].Name < s.Labels[j].Name
-			})
-			gotSamples = append(gotSamples, parsedSample{
-				Labels: s.Labels,
-				Value:  s.Values[0],
-			})
-		}
-		var expSamples []parsedSample
-		for _, s := range mt.ExpSamples {
-			expLb := datasource.Labels{}
-			if s.Labels != "" {
-				metricsqlExpr, err := metricsql.Parse(s.Labels)
-				if err != nil {
-					checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s, err: %v", mt.Expr,
-						mt.EvalTime.Duration().String(), fmt.Errorf("failed to parse labels %q: %w", s.Labels, err)))
-					continue Outer
-				}
-				metricsqlMetricExpr, ok := metricsqlExpr.(*metricsql.MetricExpr)
-				if !ok {
-					checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s, err: %v", mt.Expr,
-						mt.EvalTime.Duration().String(), fmt.Errorf("got unsupported metricsql type")))
-					continue Outer
-				}
-				for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
-					expLb = append(expLb, datasource.Label{
-						Name:  l.Label,
-						Value: l.Value,
-					})
-				}
-			}
-			sort.Slice(expLb, func(i, j int) bool {
-				return expLb[i].Name < expLb[j].Name
-			})
-			expSamples = append(expSamples, parsedSample{
-				Labels: expLb,
-				Value:  s.Value,
-			})
-		}
-		sort.Slice(expSamples, func(i, j int) bool {
-			return datasource.LabelCompare(expSamples[i].Labels, expSamples[j].Labels) <= 0
-		})
-		sort.Slice(gotSamples, func(i, j int) bool {
-			return datasource.LabelCompare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
-		})
-		if !reflect.DeepEqual(expSamples, gotSamples) {
-			checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s,\n        exp: %v\n        got: %v", mt.Expr,
-				mt.EvalTime.Duration().String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
-		}
-
-	}
-	return
-}
-
-func durationToTime(pd *promutils.Duration) time.Time {
-	if pd == nil {
-		return time.Time{}
-	}
-	return time.UnixMilli(pd.Duration().Milliseconds())
-}

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -1,43 +0,0 @@
-rule_files:
-  - rules.yaml
-
-evaluation_interval: 1m
-
-tests:
-  - interval: 1m
-    input_series:
-      - series: 'up{job="vmagent2", instance="localhost:9090"}'
-        values: "0+0x1440"
-
-    metricsql_expr_test:
-      - expr: suquery_interval_test
-        eval_time: 4m
-        exp_samples:
-          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
-            value: 1
-
-    alert_rule_test:
-      - eval_time: 2h
-        alertname: InstanceDown
-        exp_alerts:
-          - exp_labels:
-              job: vmagent2
-              severity: page
-              instance: localhost:9090
-              datacenter: dc-123
-            exp_annotations:
-              summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
-
-      - eval_time: 0
-        alertname: AlwaysFiring
-        exp_alerts:
-          - exp_labels:
-              datacenter: dc-123
-
-      - eval_time: 0
-        alertname: InstanceDown
-        exp_alerts: []
-
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/testdata/failed-test.yaml

@@ -1,49 +0,0 @@
-rule_files:
-  - rules.yaml
-
-tests:
-  - interval: 1m
-    name: "Failing test"
-    input_series:
-      - series: test
-        values: "0"
-
-    metricsql_expr_test:
-      - expr: test
-        eval_time: 0m
-        exp_samples:
-          - value: 0
-            labels: test
-
-    # will failed cause there is no "Test" group and rule defined
-    alert_rule_test:
-      - eval_time: 0m
-        groupname: Test
-        alertname: Test
-        exp_alerts:
-          - exp_labels: {}
-
-  - interval: 1m
-    name: Failing alert test
-    input_series:
-      - series: 'up{job="test"}'
-        values: 0x10
-
-    alert_rule_test:
-      # will failed cause rule is firing
-      - eval_time: 5m
-        groupname: group1
-        alertname: InstanceDown
-        exp_alerts: []
-
-  - interval: 1m
-    name: Failing alert test with missing groupname
-    input_series:
-      - series: 'up{job="test"}'
-        values: 0x10
-
-    alert_rule_test:
-      # will failed cause missing groupname
-      - eval_time: 5m
-        alertname: AlwaysFiring
-        exp_alerts: []

app/vmalert-tool/unittest/testdata/long-period.yaml

@@ -1,30 +0,0 @@
-# can be executed successfully but will take more than 1 minute
-# not included in unit test now
-evaluation_interval: 100d
-
-rule_files:
-  - rules.yaml
-
-tests:
-  - interval: 1d
-    input_series:
-      - series: test
-        # Max time in time.Duration is 106751d from 1970 (2^63/10^9), i.e. 2262.
-        # But VictoriaMetrics supports maxTimestamp value +2 days from now. see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/827.
-        # We input series to 2024-01-01T00:00:00 here.
-        values: "0+1x19723"
-
-    metricsql_expr_test:
-      - expr: timestamp(test)
-        eval_time: 0m
-        exp_samples:
-          - value: 0
-      - expr: test
-        eval_time: 100d
-        exp_samples:
-          - labels: test
-            value: 100
-      - expr: timestamp(test)
-        eval_time: 19000d
-        exp_samples:
-          - value: 1641600000 # 19000d -> seconds.

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -1,39 +0,0 @@
-groups:
-  - name: group1
-    rules:
-      - alert: InstanceDown
-        expr: up == 0
-        for: 5m
-        labels:
-          severity: page
-        annotations:
-          summary: "Instance {{ $labels.instance }} down"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
-      - alert: AlwaysFiring
-        expr: 1
-      - alert: SameAlertNameWithDifferentGroup
-        expr: absent(test)
-        for: 1m
-
-  - name: group2
-    rules:
-      - record: t1
-        expr: test
-      - record: job:test:count_over_time1m
-        expr: sum without(instance) (count_over_time(test[1m]))
-      - record: suquery_interval_test
-        expr: count_over_time(up[5m:])
-
-      - alert: SameAlertNameWithDifferentGroup
-        expr: absent(test)
-        for: 5m
-
-  - name: group3
-    rules:
-      - record: t2
-        expr: t1
-
-  - name: group4
-    rules:
-      - record: t3
-        expr: t1

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -1,104 +0,0 @@
-rule_files:
-  - rules.yaml
-
-evaluation_interval: 1m
-group_eval_order: ["group4", "group2", "group3"]
-
-tests:
-  - interval: 1m
-    name: "basic test"
-    input_series:
-      - series: "test"
-        values: "_x5 1x5 _ stale"
-
-    alert_rule_test:
-      - eval_time: 1m
-        groupname: group1
-        alertname: SameAlertNameWithDifferentGroup
-        exp_alerts:
-          - {}
-      - eval_time: 1m
-        groupname: group2
-        alertname: SameAlertNameWithDifferentGroup
-        exp_alerts: []
-      - eval_time: 150s
-        groupname: group1
-        alertname: SameAlertNameWithDifferentGroup
-        exp_alerts:
-          - {}
-      - eval_time: 6m
-        groupname: group1
-        alertname: SameAlertNameWithDifferentGroup
-        exp_alerts: []
-
-    metricsql_expr_test:
-      - expr: test
-        eval_time: 11m
-        exp_samples:
-          - labels: '{__name__="test"}'
-            value: 1
-      - expr: test
-        eval_time: 12m
-        exp_samples: []
-
-  - interval: 1m
-    name: "basic test2"
-    input_series:
-      - series: 'up{job="vmagent1", instance="localhost:9090"}'
-        values: "0+0x1440"
-      - series: "test"
-        values: "0+1x1440"
-
-    metricsql_expr_test:
-      - expr: count(ALERTS) by (alertgroup, alertname, alertstate)
-        eval_time: 4m
-        exp_samples:
-          - labels: '{alertgroup="group1", alertname="AlwaysFiring", alertstate="firing"}'
-            value: 1
-          - labels: '{alertgroup="group1", alertname="InstanceDown", alertstate="pending"}'
-            value: 1
-      - expr: t1
-        eval_time: 4m
-        exp_samples:
-          - value: 4
-            labels: '{__name__="t1", datacenter="dc-123"}'
-      - expr: t2
-        eval_time: 4m
-        exp_samples:
-          - value: 4
-            labels: '{__name__="t2", datacenter="dc-123"}'
-      - expr: t3
-        eval_time: 4m
-        exp_samples:
-          # t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
-          - value: 3
-            labels: '{__name__="t3", datacenter="dc-123"}'
-
-    alert_rule_test:
-      - eval_time: 10m
-        groupname: group1
-        alertname: InstanceDown
-        exp_alerts:
-          - exp_labels:
-              job: vmagent1
-              severity: page
-              instance: localhost:9090
-              datacenter: dc-123
-            exp_annotations:
-              summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
-
-      - eval_time: 0
-        groupname: group1
-        alertname: AlwaysFiring
-        exp_alerts:
-          - exp_labels:
-              datacenter: dc-123
-
-      - eval_time: 0
-        groupname: alerts
-        alertname: InstanceDown
-        exp_alerts: []
-
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -1,46 +0,0 @@
-rule_files:
-  - rules.yaml
-
-evaluation_interval: 1m
-
-tests:
-  - interval: 1m
-    input_series:
-      - series: 'up{job="vmagent2", instance="localhost:9090"}'
-        values: "0+0x1440"
-
-    metricsql_expr_test:
-      - expr: suquery_interval_test
-        eval_time: 4m
-        exp_samples:
-          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
-            value: 1
-
-    alert_rule_test:
-      - eval_time: 2h
-        groupname: group1
-        alertname: InstanceDown
-        exp_alerts:
-          - exp_labels:
-              job: vmagent2
-              severity: page
-              instance: localhost:9090
-              datacenter: dc-123
-            exp_annotations:
-              summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
-
-      - eval_time: 0
-        groupname: group1
-        alertname: AlwaysFiring
-        exp_alerts:
-          - exp_labels:
-              datacenter: dc-123
-
-      - eval_time: 0
-        groupname: group1
-        alertname: InstanceDown
-        exp_alerts: []
-
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/type.go

@@ -1,83 +0,0 @@
-package unittest
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-)
-
-// parsedSample is a sample with parsed Labels
-type parsedSample struct {
-	Labels datasource.Labels
-	Value  float64
-}
-
-func (ps *parsedSample) String() string {
-	return ps.Labels.String() + " " + strconv.FormatFloat(ps.Value, 'E', -1, 64)
-}
-
-func parsedSamplesString(pss []parsedSample) string {
-	if len(pss) == 0 {
-		return "nil"
-	}
-	s := pss[0].String()
-	for _, ps := range pss[1:] {
-		s += ", " + ps.String()
-	}
-	return s
-}
-
-// labelAndAnnotation holds labels and annotations
-type labelAndAnnotation struct {
-	Labels      datasource.Labels
-	Annotations datasource.Labels
-}
-
-func (la *labelAndAnnotation) String() string {
-	return "Labels:" + la.Labels.String() + "\nAnnotations:" + la.Annotations.String()
-}
-
-// labelsAndAnnotations is collection of LabelAndAnnotation
-type labelsAndAnnotations []labelAndAnnotation
-
-func (la labelsAndAnnotations) Len() int { return len(la) }
-
-func (la labelsAndAnnotations) Swap(i, j int) { la[i], la[j] = la[j], la[i] }
-func (la labelsAndAnnotations) Less(i, j int) bool {
-	diff := datasource.LabelCompare(la[i].Labels, la[j].Labels)
-	if diff != 0 {
-		return diff < 0
-	}
-	return datasource.LabelCompare(la[i].Annotations, la[j].Annotations) < 0
-}
-
-func (la labelsAndAnnotations) String() string {
-	if len(la) == 0 {
-		return "[]"
-	}
-	s := "[\n0:" + indentLines("\n"+la[0].String(), "  ")
-	for i, l := range la[1:] {
-		s += ",\n" + fmt.Sprintf("%d", i+1) + ":" + indentLines("\n"+l.String(), "  ")
-	}
-	s += "\n]"
-
-	return s
-}
-
-// indentLines prefixes each line in the supplied string with the given "indent" string.
-func indentLines(lines, indent string) string {
-	sb := strings.Builder{}
-	n := strings.Split(lines, "\n")
-	for i, l := range n {
-		if i > 0 {
-			sb.WriteString(indent)
-		}
-		sb.WriteString(l)
-		if i != len(n)-1 {
-			sb.WriteRune('\n')
-		}
-	}
-	return sb.String()
-}

app/vmalert-tool/unittest/unittest.go

@@ -1,445 +0,0 @@
-package unittest
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"path/filepath"
-	"reflect"
-	"sort"
-	"time"
-
-	"gopkg.in/yaml.v2"
-
-	vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	storagePath    string
-	httpListenAddr = ":8880"
-	// insert series from 1970-01-01T00:00:00
-	testStartTime = time.Unix(0, 0).UTC()
-
-	testPromWriteHTTPPath = "http://127.0.0.1" + httpListenAddr + "/api/v1/write"
-	testDataSourcePath    = "http://127.0.0.1" + httpListenAddr + "/prometheus"
-	testRemoteWritePath   = "http://127.0.0.1" + httpListenAddr
-	testHealthHTTPPath    = "http://127.0.0.1" + httpListenAddr + "/health"
-
-	disableAlertgroupLabel bool
-)
-
-const (
-	testStoragePath = "vmalert-unittest"
-	testLogLevel    = "ERROR"
-)
-
-// UnitTest runs unittest for files
-func UnitTest(files []string, disableGroupLabel bool) bool {
-	if err := templates.Load([]string{}, true); err != nil {
-		logger.Fatalf("failed to load template: %v", err)
-	}
-	storagePath = filepath.Join(os.TempDir(), testStoragePath)
-	processFlags()
-	vminsert.Init()
-	vmselect.Init()
-	// storagePath will be created again when closing vmselect, so remove it again.
-	defer fs.MustRemoveAll(storagePath)
-	defer vminsert.Stop()
-	defer vmselect.Stop()
-	disableAlertgroupLabel = disableGroupLabel
-	return rulesUnitTest(files)
-}
-
-func rulesUnitTest(files []string) bool {
-	var failed bool
-	for _, f := range files {
-		if err := ruleUnitTest(f); err != nil {
-			fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", f, err)
-			failed = true
-		} else {
-			fmt.Println("  SUCCESS")
-		}
-	}
-	return failed
-}
-
-func ruleUnitTest(filename string) []error {
-	fmt.Println("\nUnit Testing: ", filename)
-	b, err := os.ReadFile(filename)
-	if err != nil {
-		return []error{fmt.Errorf("failed to read file: %w", err)}
-	}
-
-	var unitTestInp unitTestFile
-	if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
-		return []error{fmt.Errorf("failed to unmarshal file: %w", err)}
-	}
-	if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
-		return []error{fmt.Errorf("failed to resolve path for `rule_files`: %w", err)}
-	}
-
-	if unitTestInp.EvaluationInterval.Duration() == 0 {
-		fmt.Println("evaluation_interval set to 1m by default")
-		unitTestInp.EvaluationInterval = &promutils.Duration{D: 1 * time.Minute}
-	}
-
-	groupOrderMap := make(map[string]int)
-	for i, gn := range unitTestInp.GroupEvalOrder {
-		if _, ok := groupOrderMap[gn]; ok {
-			return []error{fmt.Errorf("group name repeated in `group_eval_order`: %s", gn)}
-		}
-		groupOrderMap[gn] = i
-	}
-
-	testGroups, err := vmalertconfig.Parse(unitTestInp.RuleFiles, nil, true)
-	if err != nil {
-		return []error{fmt.Errorf("failed to parse `rule_files`: %w", err)}
-	}
-
-	var errs []error
-	for _, t := range unitTestInp.Tests {
-		if err := verifyTestGroup(t); err != nil {
-			errs = append(errs, err)
-			continue
-		}
-		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
-		errs = append(errs, testErrs...)
-	}
-
-	if len(errs) > 0 {
-		return errs
-	}
-	return nil
-}
-
-func verifyTestGroup(group testGroup) error {
-	var testGroupName string
-	if group.TestGroupName != "" {
-		testGroupName = fmt.Sprintf("testGroupName: %s\n", group.TestGroupName)
-	}
-	for _, at := range group.AlertRuleTests {
-		if at.Alertname == "" {
-			return fmt.Errorf("\n%s    missing required filed \"alertname\"", testGroupName)
-		}
-		if !disableAlertgroupLabel && at.GroupName == "" {
-			return fmt.Errorf("\n%s    missing required filed \"groupname\" when flag \"disableAlertGroupLabel\" is false", testGroupName)
-		}
-		if disableAlertgroupLabel && at.GroupName != "" {
-			return fmt.Errorf("\n%s    shouldn't set filed \"groupname\" when flag \"disableAlertGroupLabel\" is true", testGroupName)
-		}
-		if at.EvalTime == nil {
-			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
-		}
-	}
-	for _, et := range group.MetricsqlExprTests {
-		if et.Expr == "" {
-			return fmt.Errorf("\n%s    missing required filed \"expr\"", testGroupName)
-		}
-		if et.EvalTime == nil {
-			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
-		}
-	}
-	return nil
-}
-
-func processFlags() {
-	flag.Parse()
-	for _, fv := range []struct {
-		flag  string
-		value string
-	}{
-		{flag: "storageDataPath", value: storagePath},
-		{flag: "loggerLevel", value: testLogLevel},
-		{flag: "search.disableCache", value: "true"},
-		// set storage retention time to 100 years, allow to store series from 1970-01-01T00:00:00.
-		{flag: "retentionPeriod", value: "100y"},
-		{flag: "datasource.url", value: testDataSourcePath},
-		{flag: "remoteWrite.url", value: testRemoteWritePath},
-	} {
-		// panics if flag doesn't exist
-		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
-			logger.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
-		}
-	}
-}
-
-func setUp() {
-	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	var ab flagutil.ArrayBool
-	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
-		switch r.URL.Path {
-		case "/prometheus/api/v1/query":
-			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return true
-		case "/prometheus/api/v1/write", "/api/v1/write":
-			if err := promremotewrite.InsertHandler(r); err != nil {
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return true
-		default:
-		}
-		return false
-	})
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-	readyCheckFunc := func() bool {
-		resp, err := http.Get(testHealthHTTPPath)
-		if err != nil {
-			return false
-		}
-		_ = resp.Body.Close()
-		return resp.StatusCode == 200
-	}
-checkCheck:
-	for {
-		select {
-		case <-ctx.Done():
-			logger.Fatalf("http server can't be ready in 30s")
-		default:
-			if readyCheckFunc() {
-				break checkCheck
-			}
-			time.Sleep(3 * time.Second)
-		}
-	}
-}
-
-func tearDown() {
-	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
-		logger.Errorf("cannot stop the webservice: %s", err)
-	}
-	vmstorage.Stop()
-	metrics.UnregisterAllMetrics()
-	fs.MustRemoveAll(storagePath)
-}
-
-// resolveAndGlobFilepaths joins all relative paths in a configuration
-// with a given base directory and replaces all globs with matching files.
-func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
-	for i, rf := range utf.RuleFiles {
-		if rf != "" && !filepath.IsAbs(rf) {
-			utf.RuleFiles[i] = filepath.Join(baseDir, rf)
-		}
-	}
-
-	var globbedFiles []string
-	for _, rf := range utf.RuleFiles {
-		m, err := filepath.Glob(rf)
-		if err != nil {
-			return err
-		}
-		if len(m) == 0 {
-			fmt.Fprintln(os.Stderr, "  WARNING: no file match pattern", rf)
-		}
-		globbedFiles = append(globbedFiles, m...)
-	}
-	utf.RuleFiles = globbedFiles
-	return nil
-}
-
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
-	// set up vmstorage and http server for ingest and read queries
-	setUp()
-	// tear down vmstorage and clean the data dir
-	defer tearDown()
-
-	err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, testPromWriteHTTPPath)
-	if err != nil {
-		return []error{err}
-	}
-
-	q, err := datasource.Init(nil)
-	if err != nil {
-		return []error{fmt.Errorf("failed to init datasource: %v", err)}
-	}
-	rw, err := remotewrite.NewDebugClient()
-	if err != nil {
-		return []error{fmt.Errorf("failed to init wr: %v", err)}
-	}
-
-	alertEvalTimesMap := map[time.Duration]struct{}{}
-	alertExpResultMap := map[time.Duration]map[string]map[string][]expAlert{}
-	for _, at := range tg.AlertRuleTests {
-		et := at.EvalTime.Duration()
-		alertEvalTimesMap[et] = struct{}{}
-		if _, ok := alertExpResultMap[et]; !ok {
-			alertExpResultMap[et] = make(map[string]map[string][]expAlert)
-		}
-		if _, ok := alertExpResultMap[et][at.GroupName]; !ok {
-			alertExpResultMap[et][at.GroupName] = make(map[string][]expAlert)
-		}
-		alertExpResultMap[et][at.GroupName][at.Alertname] = at.ExpAlerts
-	}
-	alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
-	for k := range alertEvalTimesMap {
-		alertEvalTimes = append(alertEvalTimes, k)
-	}
-	sort.Slice(alertEvalTimes, func(i, j int) bool {
-		return alertEvalTimes[i] < alertEvalTimes[j]
-	})
-
-	// sort group eval order according to the given "group_eval_order".
-	sort.Slice(testGroups, func(i, j int) bool {
-		return groupOrderMap[testGroups[i].Name] < groupOrderMap[testGroups[j].Name]
-	})
-
-	// create groups with given rule
-	var groups []*rule.Group
-	for _, group := range testGroups {
-		ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
-		groups = append(groups, ng)
-	}
-
-	evalIndex := 0
-	maxEvalTime := testStartTime.Add(tg.maxEvalTime())
-	for ts := testStartTime; ts.Before(maxEvalTime) || ts.Equal(maxEvalTime); ts = ts.Add(evalInterval) {
-		for _, g := range groups {
-			errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts)
-			for err := range errs {
-				if err != nil {
-					checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name,
-						ts, err))
-				}
-			}
-			// flush series after each group evaluation
-			vmstorage.Storage.DebugFlush()
-		}
-
-		// check alert_rule_test case at every eval time
-		for evalIndex < len(alertEvalTimes) {
-			if ts.Sub(testStartTime) > alertEvalTimes[evalIndex] ||
-				alertEvalTimes[evalIndex] >= ts.Add(evalInterval).Sub(testStartTime) {
-				break
-			}
-			gotAlertsMap := map[string]map[string]labelsAndAnnotations{}
-			for _, g := range groups {
-				if disableAlertgroupLabel {
-					g.Name = ""
-				}
-				if _, ok := alertExpResultMap[alertEvalTimes[evalIndex]][g.Name]; !ok {
-					continue
-				}
-				if _, ok := gotAlertsMap[g.Name]; !ok {
-					gotAlertsMap[g.Name] = make(map[string]labelsAndAnnotations)
-				}
-				for _, r := range g.Rules {
-					ar, isAlertRule := r.(*rule.AlertingRule)
-					if !isAlertRule {
-						continue
-					}
-					if _, ok := alertExpResultMap[alertEvalTimes[evalIndex]][g.Name][ar.Name]; ok {
-						for _, got := range ar.GetAlerts() {
-							if got.State != notifier.StateFiring {
-								continue
-							}
-							if disableAlertgroupLabel {
-								delete(got.Labels, "alertgroup")
-							}
-							laa := labelAndAnnotation{
-								Labels:      datasource.ConvertToLabels(got.Labels),
-								Annotations: datasource.ConvertToLabels(got.Annotations),
-							}
-							gotAlertsMap[g.Name][ar.Name] = append(gotAlertsMap[g.Name][ar.Name], laa)
-						}
-					}
-
-				}
-			}
-			for groupname, gres := range alertExpResultMap[alertEvalTimes[evalIndex]] {
-				for alertname, res := range gres {
-					var expAlerts labelsAndAnnotations
-					for _, expAlert := range res {
-						if expAlert.ExpLabels == nil {
-							expAlert.ExpLabels = make(map[string]string)
-						}
-						// alertGroupNameLabel is added as additional labels when `disableAlertGroupLabel` is false
-						if !disableAlertgroupLabel {
-							expAlert.ExpLabels["alertgroup"] = groupname
-						}
-						// alertNameLabel is added as additional labels in vmalert.
-						expAlert.ExpLabels["alertname"] = alertname
-						expAlerts = append(expAlerts, labelAndAnnotation{
-							Labels:      datasource.ConvertToLabels(expAlert.ExpLabels),
-							Annotations: datasource.ConvertToLabels(expAlert.ExpAnnotations),
-						})
-					}
-					sort.Sort(expAlerts)
-
-					gotAlerts := gotAlertsMap[groupname][alertname]
-					sort.Sort(gotAlerts)
-					if !reflect.DeepEqual(expAlerts, gotAlerts) {
-						var testGroupName string
-						if tg.TestGroupName != "" {
-							testGroupName = fmt.Sprintf("testGroupName: %s,\n", tg.TestGroupName)
-						}
-						expString := indentLines(expAlerts.String(), "            ")
-						gotString := indentLines(gotAlerts.String(), "            ")
-						checkErrs = append(checkErrs, fmt.Errorf("\n%s    groupname: %s, alertname: %s, time: %s, \n        exp:%v, \n        got:%v ",
-							testGroupName, groupname, alertname, alertEvalTimes[evalIndex].String(), expString, gotString))
-					}
-				}
-			}
-			evalIndex++
-		}
-
-	}
-
-	checkErrs = append(checkErrs, checkMetricsqlCase(tg.MetricsqlExprTests, q)...)
-	return checkErrs
-}
-
-// unitTestFile holds the contents of a single unit test file
-type unitTestFile struct {
-	RuleFiles          []string            `yaml:"rule_files"`
-	EvaluationInterval *promutils.Duration `yaml:"evaluation_interval"`
-	GroupEvalOrder     []string            `yaml:"group_eval_order"`
-	Tests              []testGroup         `yaml:"tests"`
-}
-
-// testGroup is a group of input series and test cases associated with it
-type testGroup struct {
-	Interval           *promutils.Duration `yaml:"interval"`
-	InputSeries        []series            `yaml:"input_series"`
-	AlertRuleTests     []alertTestCase     `yaml:"alert_rule_test"`
-	MetricsqlExprTests []metricsqlTestCase `yaml:"metricsql_expr_test"`
-	ExternalLabels     map[string]string   `yaml:"external_labels"`
-	TestGroupName      string              `yaml:"name"`
-}
-
-// maxEvalTime returns the max eval time among all alert_rule_test and metricsql_expr_test
-func (tg *testGroup) maxEvalTime() time.Duration {
-	var maxd time.Duration
-	for _, alert := range tg.AlertRuleTests {
-		if alert.EvalTime.Duration() > maxd {
-			maxd = alert.EvalTime.Duration()
-		}
-	}
-	for _, met := range tg.MetricsqlExprTests {
-		if met.EvalTime.Duration() > maxd {
-			maxd = met.EvalTime.Duration()
-		}
-	}
-	return maxd
-}

app/vmalert-tool/unittest/unittest_test.go

@@ -1,47 +0,0 @@
-package unittest
-
-import (
-	"os"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
-)
-
-func TestMain(m *testing.M) {
-	if err := templates.Load([]string{}, true); err != nil {
-		os.Exit(1)
-	}
-	os.Exit(m.Run())
-}
-
-func TestUnitRule(t *testing.T) {
-	testCases := []struct {
-		name              string
-		disableGroupLabel bool
-		files             []string
-		failed            bool
-	}{
-		{
-			name:   "run multi files",
-			files:  []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
-			failed: false,
-		},
-		{
-			name:              "disable group label",
-			disableGroupLabel: true,
-			files:             []string{"./testdata/disable-group-label.yaml"},
-			failed:            false,
-		},
-		{
-			name:   "failing test",
-			files:  []string{"./testdata/failed-test.yaml"},
-			failed: true,
-		},
-	}
-	for _, tc := range testCases {
-		fail := UnitTest(tc.files, tc.disableGroupLabel)
-		if fail != tc.failed {
-			t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
-		}
-	}
-}

app/vmalert/Makefile

@@ -68,7 +68,6 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
-	go test -v -race -cover ./app/vmalert/rule
 	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier

app/vmalert/alerting.go

@@ -1,10 +1,11 @@
-package rule
+package main
 
 import (
 	"context"
 	"fmt"
 	"hash/fnv"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -30,7 +31,6 @@ type AlertingRule struct {
 	Annotations   map[string]string
 	GroupID       uint64
 	GroupName     string
-	File          string
 	EvalInterval  time.Duration
 	Debug         bool
 
@@ -48,15 +48,14 @@ type AlertingRule struct {
 }
 
 type alertingRuleMetrics struct {
-	errors        *utils.Counter
+	errors        *utils.Gauge
 	pending       *utils.Gauge
 	active        *utils.Gauge
 	samples       *utils.Gauge
 	seriesFetched *utils.Gauge
 }
 
-// NewAlertingRule creates a new AlertingRule
-func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
+func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
 		Type:          group.Type,
 		RuleID:        cfg.ID,
@@ -68,12 +67,12 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		Annotations:   cfg.Annotations,
 		GroupID:       group.ID(),
 		GroupName:     group.Name,
-		File:          group.File,
 		EvalInterval:  group.Interval,
 		Debug:         cfg.Debug,
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
+			EvalOffset:         group.EvalOffset,
 			QueryParams:        group.Params,
 			Headers:            group.Headers,
 			Debug:              cfg.Debug,
@@ -82,18 +81,13 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		metrics: &alertingRuleMetrics{},
 	}
 
-	entrySize := *ruleUpdateEntriesLimit
 	if cfg.UpdateEntriesLimit != nil {
-		entrySize = *cfg.UpdateEntriesLimit
-	}
-	if entrySize < 1 {
-		entrySize = 1
-	}
-	ar.state = &ruleState{
-		entries: make([]StateEntry, entrySize),
+		ar.state = newRuleState(*cfg.UpdateEntriesLimit)
+	} else {
+		ar.state = newRuleState(*ruleUpdateEntriesLimit)
 	}
 
-	labels := fmt.Sprintf(`alertname=%q, group=%q, file=%q, id="%d"`, ar.Name, group.Name, group.File, ar.ID())
+	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
 	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
 			ar.alertsMu.RLock()
@@ -118,32 +112,39 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return float64(num)
 		})
-	ar.metrics.errors = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_alerting_rules_errors_total{%s}`, labels))
+	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
+		func() float64 {
+			e := ar.state.getLast()
+			if e.err == nil {
+				return 0
+			}
+			return 1
+		})
 	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
 		func() float64 {
 			e := ar.state.getLast()
-			return float64(e.Samples)
+			return float64(e.samples)
 		})
 	ar.metrics.seriesFetched = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_series_fetched{%s}`, labels),
 		func() float64 {
 			e := ar.state.getLast()
-			if e.SeriesFetched == nil {
+			if e.seriesFetched == nil {
 				// means seriesFetched is unsupported
 				return -1
 			}
-			seriesFetched := float64(*e.SeriesFetched)
-			if seriesFetched == 0 && e.Samples > 0 {
+			seriesFetched := float64(*e.seriesFetched)
+			if seriesFetched == 0 && e.samples > 0 {
 				// `alert: 0.95` will fetch no series
 				// but will get one time series in response.
-				seriesFetched = float64(e.Samples)
+				seriesFetched = float64(e.samples)
 			}
 			return seriesFetched
 		})
 	return ar
 }
 
-// close unregisters rule metrics
-func (ar *AlertingRule) close() {
+// Close unregisters rule metrics
+func (ar *AlertingRule) Close() {
 	ar.metrics.active.Unregister()
 	ar.metrics.pending.Unregister()
 	ar.metrics.errors.Unregister()
@@ -162,27 +163,6 @@ func (ar *AlertingRule) ID() uint64 {
 	return ar.RuleID
 }
 
-// GetAlerts returns active alerts of rule
-func (ar *AlertingRule) GetAlerts() []*notifier.Alert {
-	ar.alertsMu.RLock()
-	defer ar.alertsMu.RUnlock()
-	var alerts []*notifier.Alert
-	for _, a := range ar.alerts {
-		alerts = append(alerts, a)
-	}
-	return alerts
-}
-
-// GetAlert returns alert if id exists
-func (ar *AlertingRule) GetAlert(id uint64) *notifier.Alert {
-	ar.alertsMu.RLock()
-	defer ar.alertsMu.RUnlock()
-	if ar.alerts == nil {
-		return nil
-	}
-	return ar.alerts[id]
-}
-
 func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...interface{}) {
 	if !ar.Debug {
 		return
@@ -209,26 +189,6 @@ func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string
 	logger.Infof("%s", prefix+msg)
 }
 
-// updateWith copies all significant fields.
-// alerts state isn't copied since
-// it should be updated in next 2 Execs
-func (ar *AlertingRule) updateWith(r Rule) error {
-	nr, ok := r.(*AlertingRule)
-	if !ok {
-		return fmt.Errorf("BUG: attempt to update alerting rule with wrong type %#v", r)
-	}
-	ar.Expr = nr.Expr
-	ar.For = nr.For
-	ar.KeepFiringFor = nr.KeepFiringFor
-	ar.Labels = nr.Labels
-	ar.Annotations = nr.Annotations
-	ar.EvalInterval = nr.EvalInterval
-	ar.Debug = nr.Debug
-	ar.q = nr.q
-	ar.state = nr.state
-	return nil
-}
-
 type labelSet struct {
 	// origin labels extracted from received time series
 	// plus extra labels (group labels, service labels like alertNameLabel).
@@ -237,30 +197,11 @@ type labelSet struct {
 	origin map[string]string
 	// processed labels includes origin labels
 	// plus extra labels (group labels, service labels like alertNameLabel).
-	// in case of key conflicts, origin labels are renamed with prefix `exported_` and extra labels are preferred.
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5161
+	// in case of conflicts, extra labels are preferred.
 	// used as labels attached to notifier.Alert and ALERTS series written to remote storage.
 	processed map[string]string
 }
 
-// add adds a value v with key k to origin and processed label sets.
-// On k conflicts in processed set, the passed v is preferred.
-// On k conflicts in origin set, the original value is preferred and copied
-// to processed with `exported_%k` key. The copy happens only if passed v isn't equal to origin[k] value.
-func (ls *labelSet) add(k, v string) {
-	ls.processed[k] = v
-	ov, ok := ls.origin[k]
-	if !ok {
-		ls.origin[k] = v
-		return
-	}
-	if ov != v {
-		// copy value only if v and ov are different
-		key := fmt.Sprintf("exported_%s", k)
-		ls.processed[key] = ov
-	}
-}
-
 // toLabels converts labels from given Metric
 // to labelSet which contains original and processed labels.
 func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
@@ -283,80 +224,73 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		Expr:   ar.Expr,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to expand labels: %w", err)
+		return nil, fmt.Errorf("failed to expand labels: %s", err)
 	}
 	for k, v := range extraLabels {
-		ls.add(k, v)
+		ls.processed[k] = v
+		if _, ok := ls.origin[k]; !ok {
+			ls.origin[k] = v
+		}
 	}
+
 	// set additional labels to identify group and rule name
 	if ar.Name != "" {
-		ls.add(alertNameLabel, ar.Name)
+		ls.processed[alertNameLabel] = ar.Name
+		if _, ok := ls.origin[alertNameLabel]; !ok {
+			ls.origin[alertNameLabel] = ar.Name
+		}
 	}
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
-		ls.add(alertGroupNameLabel, ar.GroupName)
+		ls.processed[alertGroupNameLabel] = ar.GroupName
+		if _, ok := ls.origin[alertGroupNameLabel]; !ok {
+			ls.origin[alertGroupNameLabel] = ar.GroupName
+		}
 	}
 	return ls, nil
 }
 
-// execRange executes alerting rule on the given time range similarly to exec.
-// When making consecutive calls make sure to respect time linearity for start and end params,
-// as this function modifies AlertingRule alerts state.
-// It is not thread safe.
-// It returns ALERT and ALERT_FOR_STATE time series as a result.
-func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error) {
+// ExecRange executes alerting rule on the given time range similarly to Exec.
+// It doesn't update internal states of the Rule and meant to be used just
+// to get time series for backfilling.
+// It returns ALERT and ALERT_FOR_STATE time series as result.
+func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error) {
 	res, err := ar.q.QueryRange(ctx, ar.Expr, start, end)
 	if err != nil {
 		return nil, err
 	}
 	var result []prompbmarshal.TimeSeries
-	holdAlertState := make(map[uint64]*notifier.Alert)
-	qFn := func(_ string) ([]datasource.Metric, error) {
+	qFn := func(query string) ([]datasource.Metric, error) {
 		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
 	}
 	for _, s := range res.Data {
-		ls, err := ar.toLabels(s, qFn)
-		if err != nil {
-			return nil, fmt.Errorf("failed to expand labels: %s", err)
-		}
-		h := hash(ls.processed)
 		a, err := ar.newAlert(s, nil, time.Time{}, qFn) // initial alert
 		if err != nil {
-			return nil, fmt.Errorf("failed to create alert: %w", err)
+			return nil, fmt.Errorf("failed to create alert: %s", err)
+		}
+		if ar.For == 0 { // if alert is instant
+			a.State = notifier.StateFiring
+			for i := range s.Values {
+				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+			}
+			continue
 		}
 
+		// if alert with For > 0
 		prevT := time.Time{}
 		for i := range s.Values {
 			at := time.Unix(s.Timestamps[i], 0)
-			// try to restore alert's state on the first iteration
-			if at.Equal(start) {
-				if _, ok := ar.alerts[h]; ok {
-					a = ar.alerts[h]
-					prevT = at
-				}
-			}
 			if at.Sub(prevT) > ar.EvalInterval {
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
 				a.ActiveAt = at
-				a.Start = time.Time{}
-			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
+			} else if at.Sub(a.ActiveAt) >= ar.For {
 				a.State = notifier.StateFiring
 				a.Start = at
 			}
 			prevT = at
-			if ar.For == 0 {
-				// rules with `for: 0` are always firing when they have Value
-				a.State = notifier.StateFiring
-			}
 			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
-
-			// save alert's state on last iteration, so it can be used on the next execRange call
-			if at.Equal(end) {
-				holdAlertState[h] = a
-			}
 		}
 	}
-	ar.alerts = holdAlertState
 	return result, nil
 }
 
@@ -364,26 +298,23 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 // is kept in memory state and consequently repeatedly sent to the AlertManager.
 const resolvedRetention = 15 * time.Minute
 
-// exec executes AlertingRule expression via the given Querier.
+// Exec executes AlertingRule expression via the given Querier.
 // Based on the Querier results AlertingRule maintains notifier.Alerts
-func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
 	start := time.Now()
 	res, req, err := ar.q.Query(ctx, ar.Expr, ts)
-	curState := StateEntry{
-		Time:          start,
-		At:            ts,
-		Duration:      time.Since(start),
-		Samples:       len(res.Data),
-		SeriesFetched: res.SeriesFetched,
-		Err:           err,
-		Curl:          requestToCurl(req),
+	curState := ruleStateEntry{
+		time:          start,
+		at:            ts,
+		duration:      time.Since(start),
+		samples:       len(res.Data),
+		seriesFetched: res.SeriesFetched,
+		err:           err,
+		curl:          requestToCurl(req),
 	}
 
 	defer func() {
 		ar.state.add(curState)
-		if curState.Err != nil {
-			ar.metrics.errors.Inc()
-		}
 	}()
 
 	ar.alertsMu.Lock()
@@ -393,7 +324,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
-	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s)", curState.Samples, curState.Duration)
+	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s)", curState.samples, curState.duration)
 
 	for h, a := range ar.alerts {
 		// cleanup inactive alerts from previous Exec
@@ -412,14 +343,15 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	for _, m := range res.Data {
 		ls, err := ar.toLabels(m, qFn)
 		if err != nil {
-			curState.Err = fmt.Errorf("failed to expand labels: %w", err)
-			return nil, curState.Err
+			curState.err = fmt.Errorf("failed to expand labels: %s", err)
+			return nil, curState.err
 		}
 		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
-			// duplicate may be caused the removal of `__name__` label
-			curState.Err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
-			return nil, curState.Err
+			// duplicate may be caused by extra labels
+			// conflicting with the metric labels
+			curState.err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, curState.err
 		}
 		updated[h] = struct{}{}
 		if a, ok := ar.alerts[h]; ok {
@@ -440,13 +372,14 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.KeepFiringSince = time.Time{}
 			continue
 		}
-		a, err := ar.newAlert(m, ls, ts, qFn)
+		a, err := ar.newAlert(m, ls, start, qFn)
 		if err != nil {
-			curState.Err = fmt.Errorf("failed to create alert: %w", err)
-			return nil, curState.Err
+			curState.err = fmt.Errorf("failed to create alert: %w", err)
+			return nil, curState.err
 		}
 		a.ID = h
 		a.State = notifier.StatePending
+		a.ActiveAt = ts
 		ar.alerts[h] = a
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
@@ -472,7 +405,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				}
 				// alerts with ar.KeepFiringFor>0 may remain FIRING
 				// even if their expression isn't true anymore
-				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
+				if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
@@ -491,8 +424,8 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	}
 	if limit > 0 && numActivePending > limit {
 		ar.alerts = map[uint64]*notifier.Alert{}
-		curState.Err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
-		return nil, curState.Err
+		curState.err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
+		return nil, curState.err
 	}
 	return ar.toTimeSeries(ts.Unix()), nil
 }
@@ -509,6 +442,26 @@ func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries
 	return tss
 }
 
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (ar *AlertingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*AlertingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update alerting rule with wrong type %#v", r)
+	}
+	ar.Expr = nr.Expr
+	ar.For = nr.For
+	ar.KeepFiringFor = nr.KeepFiringFor
+	ar.Labels = nr.Labels
+	ar.Annotations = nr.Annotations
+	ar.EvalInterval = nr.EvalInterval
+	ar.Debug = nr.Debug
+	ar.q = nr.q
+	ar.state = nr.state
+	return nil
+}
+
 // TODO: consider hashing algorithm in VM
 func hash(labels map[string]string) uint64 {
 	hash := fnv.New64a()
@@ -535,7 +488,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T
 	if ls == nil {
 		ls, err = ar.toLabels(m, qFn)
 		if err != nil {
-			return nil, fmt.Errorf("failed to expand labels: %w", err)
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
 		}
 	}
 	a := &notifier.Alert{
@@ -551,10 +504,106 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.T
 	return a, err
 }
 
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
+	ar.alertsMu.RLock()
+	defer ar.alertsMu.RUnlock()
+	a, ok := ar.alerts[id]
+	if !ok {
+		return nil
+	}
+	return ar.newAlertAPI(*a)
+}
+
+// ToAPI returns Rule representation in form of APIRule
+// Isn't thread-safe. Call must be protected by AlertingRule mutex.
+func (ar *AlertingRule) ToAPI() APIRule {
+	lastState := ar.state.getLast()
+	r := APIRule{
+		Type:              "alerting",
+		DatasourceType:    ar.Type.String(),
+		Name:              ar.Name,
+		Query:             ar.Expr,
+		Duration:          ar.For.Seconds(),
+		KeepFiringFor:     ar.KeepFiringFor.Seconds(),
+		Labels:            ar.Labels,
+		Annotations:       ar.Annotations,
+		LastEvaluation:    lastState.time,
+		EvaluationTime:    lastState.duration.Seconds(),
+		Health:            "ok",
+		State:             "inactive",
+		Alerts:            ar.AlertsToAPI(),
+		LastSamples:       lastState.samples,
+		LastSeriesFetched: lastState.seriesFetched,
+		MaxUpdates:        ar.state.size(),
+		Updates:           ar.state.getAll(),
+		Debug:             ar.Debug,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
+	if lastState.err != nil {
+		r.LastError = lastState.err.Error()
+		r.Health = "err"
+	}
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
+	}
+	return r
+}
+
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
+	var alerts []*APIAlert
+	ar.alertsMu.RLock()
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		alerts = append(alerts, ar.newAlertAPI(*a))
+	}
+	ar.alertsMu.RUnlock()
+	return alerts
+}
+
+func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
+	aa := &APIAlert{
+		// encode as strings to avoid rounding
+		ID:      fmt.Sprintf("%d", a.ID),
+		GroupID: fmt.Sprintf("%d", a.GroupID),
+		RuleID:  fmt.Sprintf("%d", ar.RuleID),
+
+		Name:        a.Name,
+		Expression:  ar.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.ActiveAt,
+		Restored:    a.Restored,
+		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
+	}
+	if alertURLGeneratorFn != nil {
+		aa.SourceLink = alertURLGeneratorFn(a)
+	}
+	if a.State == notifier.StateFiring && !a.KeepFiringSince.IsZero() {
+		aa.Stabilizing = true
+	}
+	return aa
+}
+
 const (
-	// alertMetricName is the metric name for time series reflecting the alert state.
+	// alertMetricName is the metric name for synthetic alert timeseries.
 	alertMetricName = "ALERTS"
-	// alertForStateMetricName is the metric name for time series reflecting the moment of time when alert became active.
+	// alertForStateMetricName is the metric name for 'for' state of alert.
 	alertForStateMetricName = "ALERTS_FOR_STATE"
 
 	// alertNameLabel is the label name indicating the name of an alert.
@@ -569,10 +618,12 @@ const (
 
 // alertToTimeSeries converts the given alert with the given timestamp to time series
 func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
-	return []prompbmarshal.TimeSeries{
-		alertToTimeSeries(a, timestamp),
-		alertForToTimeSeries(a, timestamp),
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
 	}
+	return tss
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
@@ -596,10 +647,10 @@ func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.Time
 	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
-// restore restores the value of ActiveAt field for active alerts,
+// Restore restores the value of ActiveAt field for active alerts,
 // based on previously written time series `alertForStateMetricName`.
 // Only rules with For > 0 can be restored.
-func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts time.Time, lookback time.Duration) error {
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, ts time.Time, lookback time.Duration) error {
 	if ar.For < 1 {
 		return nil
 	}
@@ -611,41 +662,44 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 		return nil
 	}
 
-	nameStr := fmt.Sprintf("%s=%q", alertNameLabel, ar.Name)
-	if !*disableAlertGroupLabel {
-		nameStr = fmt.Sprintf("%s=%q,%s=%q", alertGroupNameLabel, ar.GroupName, alertNameLabel, ar.Name)
-	}
-	var labelsFilter string
-	for k, v := range ar.Labels {
-		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
-	}
-	expr := fmt.Sprintf("last_over_time(%s{%s%s}[%ds])",
-		alertForStateMetricName, nameStr, labelsFilter, int(lookback.Seconds()))
-
-	res, _, err := q.Query(ctx, expr, ts)
-	if err != nil {
-		return fmt.Errorf("failed to execute restore query %q: %w ", expr, err)
-	}
-
-	if len(res.Data) < 1 {
-		ar.logDebugf(ts, nil, "no response was received from restore query")
-		return nil
-	}
-	for _, series := range res.Data {
-		series.DelLabel("__name__")
-		labelSet := make(map[string]string, len(series.Labels))
-		for _, v := range series.Labels {
-			labelSet[v.Name] = v.Value
-		}
-		id := hash(labelSet)
-		a, ok := ar.alerts[id]
-		if !ok {
-			continue
-		}
+	for _, a := range ar.alerts {
 		if a.Restored || a.State != notifier.StatePending {
 			continue
 		}
-		a.ActiveAt = time.Unix(int64(series.Values[0]), 0)
+
+		var labelsFilter []string
+		for k, v := range a.Labels {
+			labelsFilter = append(labelsFilter, fmt.Sprintf("%s=%q", k, v))
+		}
+		sort.Strings(labelsFilter)
+		expr := fmt.Sprintf("last_over_time(%s{%s}[%ds])",
+			alertForStateMetricName, strings.Join(labelsFilter, ","), int(lookback.Seconds()))
+
+		ar.logDebugf(ts, nil, "restoring alert state via query %q", expr)
+
+		res, _, err := q.Query(ctx, expr, ts)
+		if err != nil {
+			return err
+		}
+
+		qMetrics := res.Data
+		if len(qMetrics) < 1 {
+			ar.logDebugf(ts, nil, "no response was received from restore query")
+			continue
+		}
+
+		// only one series expected in response
+		m := qMetrics[0]
+		// __name__ supposed to be alertForStateMetricName
+		m.DelLabel("__name__")
+
+		// we assume that restore query contains all label matchers,
+		// so all received labels will match anyway if their number is equal.
+		if len(m.Labels) != len(a.Labels) {
+			ar.logDebugf(ts, nil, "state restore query returned not expected label-set %v", m.Labels)
+			continue
+		}
+		a.ActiveAt = time.Unix(int64(m.Values[0]), 0)
 		a.Restored = true
 		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
 	}
@@ -655,19 +709,15 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 // alertsToSend walks through the current alerts of AlertingRule
 // and returns only those which should be sent to notifier.
 // Isn't concurrent safe.
-func (ar *AlertingRule) alertsToSend(resolveDuration, resendDelay time.Duration) []notifier.Alert {
-	currentTime := time.Now()
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
 	needsSending := func(a *notifier.Alert) bool {
 		if a.State == notifier.StatePending {
 			return false
 		}
-		if a.State == notifier.StateFiring && a.End.Before(a.LastSent) {
+		if a.ResolvedAt.After(a.LastSent) {
 			return true
 		}
-		if a.State == notifier.StateInactive && a.ResolvedAt.After(a.LastSent) {
-			return true
-		}
-		return a.LastSent.Add(resendDelay).Before(currentTime)
+		return a.LastSent.Add(resendDelay).Before(ts)
 	}
 
 	var alerts []notifier.Alert
@@ -675,11 +725,11 @@ func (ar *AlertingRule) alertsToSend(resolveDuration, resendDelay time.Duration)
 		if !needsSending(a) {
 			continue
 		}
-		a.End = currentTime.Add(resolveDuration)
+		a.End = ts.Add(resolveDuration)
 		if a.State == notifier.StateInactive {
 			a.End = a.ResolvedAt
 		}
-		a.LastSent = currentTime
+		a.LastSent = ts
 		alerts = append(alerts, *a)
 	}
 	return alerts

app/vmalert/alerting_test.go

@@ -1,9 +1,8 @@
-package rule
+package main
 
 import (
 	"context"
 	"errors"
-	"fmt"
 	"reflect"
 	"sort"
 	"strings"
@@ -14,7 +13,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -28,28 +26,20 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 	}{
 		{
 			newTestAlertingRule("instant", 0),
-			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
+			&notifier.Alert{State: notifier.StateFiring},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__": alertForStateMetricName,
-					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant extra labels", 0),
-			&notifier.Alert{
-				State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
-				Labels: map[string]string{
-					"job":      "foo",
-					"instance": "bar",
-				},
-			},
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
@@ -57,35 +47,19 @@ func TestAlertingRule_ToTimeSeries(t *testing.T) {
 					"job":           "foo",
 					"instance":      "bar",
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__": alertForStateMetricName,
-						"job":      "foo",
-						"instance": "bar",
-					}),
 			},
 		},
 		{
 			newTestAlertingRule("instant labels override", 0),
-			&notifier.Alert{
-				State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
-				Labels: map[string]string{
-					alertStateLabel: "foo",
-					"__name__":      "bar",
-				},
-			},
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
 					"__name__":      alertMetricName,
 					alertStateLabel: notifier.StateFiring.String(),
 				}),
-				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
-					[]int64{timestamp.UnixNano()},
-					map[string]string{
-						"__name__":      alertForStateMetricName,
-						alertStateLabel: "foo",
-					}),
 			},
 		},
 		{
@@ -329,20 +303,17 @@ func TestAlertingRule_Exec(t *testing.T) {
 	fakeGroup := Group{Name: "TestRule_Exec"}
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
-			fq := &datasource.FakeQuerier{}
+			fq := &fakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
-			ts := time.Now()
 			for i, step := range tc.steps {
-				fq.Reset()
-				fq.Add(step...)
-				if _, err := tc.rule.exec(context.TODO(), ts, 0); err != nil {
+				fq.reset()
+				fq.add(step...)
+				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 					t.Fatalf("unexpected err: %s", err)
 				}
-
-				// shift the execution timestamp before the next iteration
-				ts = ts.Add(defaultStep)
-
+				// artificial delay between applying steps
+				time.Sleep(defaultStep)
 				if _, ok := tc.expAlerts[i]; !ok {
 					continue
 				}
@@ -375,18 +346,15 @@ func TestAlertingRule_Exec(t *testing.T) {
 }
 
 func TestAlertingRule_ExecRange(t *testing.T) {
-	fakeGroup := Group{Name: "TestRule_ExecRange"}
 	testCases := []struct {
-		rule                    *AlertingRule
-		data                    []datasource.Metric
-		expAlerts               []*notifier.Alert
-		expHoldAlertStateAlerts map[uint64]*notifier.Alert
+		rule      *AlertingRule
+		data      []datasource.Metric
+		expAlerts []*notifier.Alert
 	}{
 		{
 			newTestAlertingRule("empty", 0),
 			[]datasource.Metric{},
 			nil,
-			nil,
 		},
 		{
 			newTestAlertingRule("empty labels", 0),
@@ -394,9 +362,8 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1}, Timestamps: []int64{1}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring},
 			},
-			nil,
 		},
 		{
 			newTestAlertingRule("single-firing", 0),
@@ -405,12 +372,10 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			},
 			[]*notifier.Alert{
 				{
-					Labels:   map[string]string{"name": "foo"},
-					State:    notifier.StateFiring,
-					ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{"name": "foo"},
+					State:  notifier.StateFiring,
 				},
 			},
-			nil,
 		},
 		{
 			newTestAlertingRule("single-firing-on-range", 0),
@@ -418,11 +383,10 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
 			},
 			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1e3, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(2e3, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(3e3, 0)},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
 			},
-			nil,
 		},
 		{
 			newTestAlertingRule("for-pending", time.Second),
@@ -434,16 +398,6 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{State: notifier.StatePending, ActiveAt: time.Unix(3, 0)},
 				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
 			},
-			map[uint64]*notifier.Alert{hash(map[string]string{"alertname": "for-pending"}): {
-				GroupID:     fakeGroup.ID(),
-				Name:        "for-pending",
-				Labels:      map[string]string{"alertname": "for-pending"},
-				Annotations: map[string]string{},
-				State:       notifier.StatePending,
-				ActiveAt:    time.Unix(5, 0),
-				Value:       1,
-				For:         time.Second,
-			}},
 		},
 		{
 			newTestAlertingRule("for-firing", 3*time.Second),
@@ -455,52 +409,6 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
 				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 			},
-			map[uint64]*notifier.Alert{hash(map[string]string{"alertname": "for-firing"}): {
-				GroupID:     fakeGroup.ID(),
-				Name:        "for-firing",
-				Labels:      map[string]string{"alertname": "for-firing"},
-				Annotations: map[string]string{},
-				State:       notifier.StateFiring,
-				ActiveAt:    time.Unix(1, 0),
-				Start:       time.Unix(5, 0),
-				Value:       1,
-				For:         3 * time.Second,
-			}},
-		},
-		{
-			newTestAlertingRule("for-hold-pending", time.Second),
-			[]datasource.Metric{
-				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 2, 5}},
-			},
-			[]*notifier.Alert{
-				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
-				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
-			},
-			map[uint64]*notifier.Alert{hash(map[string]string{"alertname": "for-hold-pending"}): {
-				GroupID:     fakeGroup.ID(),
-				Name:        "for-hold-pending",
-				Labels:      map[string]string{"alertname": "for-hold-pending"},
-				Annotations: map[string]string{},
-				State:       notifier.StatePending,
-				ActiveAt:    time.Unix(5, 0),
-				Value:       1,
-				For:         time.Second,
-			}},
-		},
-		{
-			newTestAlertingRuleWithEvalInterval("firing=>inactive=>inactive=>firing=>firing", 0, time.Second),
-			[]datasource.Metric{
-				{Values: []float64{1, 1, 1, 1}, Timestamps: []int64{1, 4, 5, 6}},
-			},
-			[]*notifier.Alert{
-				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
-				// It is expected for ActiveAT to remain the same while rule continues to fire in each iteration
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-				{State: notifier.StateFiring, ActiveAt: time.Unix(4, 0)},
-			},
-			nil,
 		},
 		{
 			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
@@ -514,10 +422,9 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0)},
 				{State: notifier.StatePending, ActiveAt: time.Unix(20, 0)},
 			},
-			nil,
 		},
 		{
-			newTestAlertingRule("multi-series", 3*time.Second),
+			newTestAlertingRule("multi-series-for=>pending=>pending=>firing", 3*time.Second),
 			[]datasource.Metric{
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 				{
@@ -529,6 +436,7 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
 				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
 				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				//
 				{
 					State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
 					Labels: map[string]string{
@@ -542,29 +450,6 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 					},
 				},
 			},
-			map[uint64]*notifier.Alert{
-				hash(map[string]string{"alertname": "multi-series"}): {
-					GroupID:     fakeGroup.ID(),
-					Name:        "multi-series",
-					Labels:      map[string]string{"alertname": "multi-series"},
-					Annotations: map[string]string{},
-					State:       notifier.StateFiring,
-					ActiveAt:    time.Unix(1, 0),
-					Start:       time.Unix(5, 0),
-					Value:       1,
-					For:         3 * time.Second,
-				},
-				hash(map[string]string{"alertname": "multi-series", "foo": "bar"}): {
-					GroupID:     fakeGroup.ID(),
-					Name:        "multi-series",
-					Labels:      map[string]string{"alertname": "multi-series", "foo": "bar"},
-					Annotations: map[string]string{},
-					State:       notifier.StatePending,
-					ActiveAt:    time.Unix(5, 0),
-					Value:       1,
-					For:         3 * time.Second,
-				},
-			},
 		},
 		{
 			newTestRuleWithLabels("multi-series-firing", "source", "vm"),
@@ -576,44 +461,32 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				},
 			},
 			[]*notifier.Alert{
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
-					Labels: map[string]string{
-						"source": "vm",
-					},
-				},
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(100, 0),
-					Labels: map[string]string{
-						"source": "vm",
-					},
-				},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
 				//
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
-					Labels: map[string]string{
-						"foo":    "bar",
-						"source": "vm",
-					},
-				},
-				{
-					State: notifier.StateFiring, ActiveAt: time.Unix(5, 0),
-					Labels: map[string]string{
-						"foo":    "bar",
-						"source": "vm",
-					},
-				},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
 			},
-			nil,
 		},
 	}
+	fakeGroup := Group{Name: "TestRule_ExecRange"}
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
-			fq := &datasource.FakeQuerier{}
+			fq := &fakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
-			fq.Add(tc.data...)
-			gotTS, err := tc.rule.execRange(context.TODO(), time.Unix(1, 0), time.Unix(5, 0))
+			fq.add(tc.data...)
+			gotTS, err := tc.rule.ExecRange(context.TODO(), time.Now(), time.Now())
 			if err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
@@ -639,35 +512,30 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 					t.Fatalf("%d: expected \n%v but got \n%v", i, exp, got)
 				}
 			}
-			if tc.expHoldAlertStateAlerts != nil {
-				if !reflect.DeepEqual(tc.expHoldAlertStateAlerts, tc.rule.alerts) {
-					t.Fatalf("expected hold alerts state: \n%v but got \n%v", tc.expHoldAlertStateAlerts, tc.rule.alerts)
-				}
-			}
 		})
 	}
 }
 
 func TestGroup_Restore(t *testing.T) {
 	defaultTS := time.Now()
-	fqr := &datasource.FakeQuerierWithRegistry{}
+	fqr := &fakeQuerierWithRegistry{}
 	fn := func(rules []config.Rule, expAlerts map[uint64]*notifier.Alert) {
 		t.Helper()
-		defer fqr.Reset()
+		defer fqr.reset()
 
 		for _, r := range rules {
-			fqr.Set(r.Expr, metricWithValueAndLabels(t, 0, "__name__", r.Alert))
+			fqr.set(r.Expr, metricWithValueAndLabels(t, 0, "__name__", r.Alert))
 		}
 
-		fg := NewGroup(config.Group{Name: "TestRestore", Rules: rules}, fqr, time.Second, nil)
+		fg := newGroup(config.Group{Name: "TestRestore", Rules: rules}, fqr, time.Second, nil)
 		wg := sync.WaitGroup{}
 		wg.Add(1)
 		go func() {
-			nts := func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} }
-			fg.Start(context.Background(), nts, nil, fqr)
+			nts := func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} }
+			fg.start(context.Background(), nts, nil, fqr)
 			wg.Done()
 		}()
-		fg.Close()
+		fg.close()
 		wg.Wait()
 
 		gotAlerts := make(map[uint64]*notifier.Alert)
@@ -696,9 +564,6 @@ func TestGroup_Restore(t *testing.T) {
 			if got.ActiveAt != exp.ActiveAt {
 				t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
 			}
-			if got.Name != exp.Name {
-				t.Fatalf("expected alertname %q; got %q", exp.Name, got.Name)
-			}
 		}
 	}
 
@@ -714,29 +579,27 @@ func TestGroup_Restore(t *testing.T) {
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
-				Name:     "foo",
 				ActiveAt: defaultTS,
 			},
 		})
-	fqr.Reset()
+	fqr.reset()
 
 	// one active alert with state restore
 	ts := time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
 		stateMetric("foo", ts))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
-				Name:     "foo",
 				ActiveAt: ts,
 			},
 		})
 
 	// two rules, two active alerts, one with state restored
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
-		stateMetric("bar", ts))
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+		stateMetric("foo", ts))
 	fn(
 		[]config.Rule{
 			{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)},
@@ -744,20 +607,18 @@ func TestGroup_Restore(t *testing.T) {
 		},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
-				Name:     "foo",
 				ActiveAt: defaultTS,
 			},
 			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
-				Name:     "bar",
 				ActiveAt: ts,
 			},
 		})
 
 	// two rules, two active alerts, two with state restored
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
 		stateMetric("foo", ts))
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
 		stateMetric("bar", ts))
 	fn(
 		[]config.Rule{
@@ -766,82 +627,75 @@ func TestGroup_Restore(t *testing.T) {
 		},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
-				Name:     "foo",
 				ActiveAt: ts,
 			},
 			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
-				Name:     "bar",
 				ActiveAt: ts,
 			},
 		})
 
 	// one active alert but wrong state restore
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
 		stateMetric("wrong alert", ts))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
-				Name:     "foo",
 				ActiveAt: defaultTS,
 			},
 		})
 
 	// one active alert with labels
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
 		stateMetric("foo", ts, "env", "dev"))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
-				Name:     "foo",
 				ActiveAt: ts,
 			},
 		})
 
 	// one active alert with restore labels missmatch
 	ts = time.Now().Truncate(time.Hour)
-	fqr.Set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
 		stateMetric("foo", ts, "env", "dev", "team", "foo"))
 	fn(
 		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
-				Name:     "foo",
 				ActiveAt: defaultTS,
 			},
 		})
 }
 
 func TestAlertingRule_Exec_Negative(t *testing.T) {
-	fq := &datasource.FakeQuerier{}
+	fq := &fakeQuerier{}
 	ar := newTestAlertingRule("test", 0)
 	ar.Labels = map[string]string{"job": "test"}
 	ar.q = fq
 
 	// successful attempt
-	// label `job` will be overridden by rule extra label, the original value will be reserved by "exported_job"
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
-	_, err := ar.exec(context.TODO(), time.Now(), 0)
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	_, err := ar.Exec(context.TODO(), time.Now(), 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// label `__name__` will be omitted and get duplicated results here
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo_1", "job", "bar"))
-	_, err = ar.exec(context.TODO(), time.Now(), 0)
+	// label `job` will collide with rule extra label and will make both time series equal
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
 	}
 
-	fq.Reset()
+	fq.reset()
 
 	expErr := "connection reset by peer"
-	fq.SetErr(errors.New(expErr))
-	_, err = ar.exec(context.TODO(), time.Now(), 0)
+	fq.setErr(errors.New(expErr))
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
 	if err == nil {
 		t.Fatalf("expected to get err; got nil")
 	}
@@ -851,7 +705,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 }
 
 func TestAlertingRuleLimit(t *testing.T) {
-	fq := &datasource.FakeQuerier{}
+	fq := &fakeQuerier{}
 	ar := newTestAlertingRule("test", 0)
 	ar.Labels = map[string]string{"job": "test"}
 	ar.q = fq
@@ -883,15 +737,15 @@ func TestAlertingRuleLimit(t *testing.T) {
 		err       error
 		timestamp = time.Now()
 	)
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
 	for _, testCase := range testCases {
-		_, err = ar.exec(context.TODO(), timestamp, testCase.limit)
+		_, err = ar.Exec(context.TODO(), timestamp, testCase.limit)
 		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
 			t.Fatal(err)
 		}
 	}
-	fq.Reset()
+	fq.reset()
 }
 
 func TestAlertingRule_Template(t *testing.T) {
@@ -955,22 +809,20 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
 					Labels: map[string]string{
-						alertNameLabel:       "override label",
-						"exported_alertname": "override",
-						"instance":           "foo",
+						alertNameLabel: "override label",
+						"instance":     "foo",
 					},
 					Annotations: map[string]string{
 						"summary":     `first: Too high connection number for "foo"`,
 						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
-				hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "bar"}): {
+				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
 					Labels: map[string]string{
-						alertNameLabel:       "override label",
-						"exported_alertname": "override",
-						"instance":           "bar",
+						alertNameLabel: "override label",
+						"instance":     "bar",
 					},
 					Annotations: map[string]string{
 						"summary":     `second: Too high connection number for "bar"`,
@@ -999,18 +851,14 @@ func TestAlertingRule_Template(t *testing.T) {
 			},
 			map[uint64]*notifier.Alert{
 				hash(map[string]string{
-					alertNameLabel:        "OriginLabels",
-					"exported_alertname":  "originAlertname",
-					alertGroupNameLabel:   "Testing",
-					"exported_alertgroup": "originGroupname",
-					"instance":            "foo",
+					alertNameLabel:      "OriginLabels",
+					alertGroupNameLabel: "Testing",
+					"instance":          "foo",
 				}): {
 					Labels: map[string]string{
-						alertNameLabel:        "OriginLabels",
-						"exported_alertname":  "originAlertname",
-						alertGroupNameLabel:   "Testing",
-						"exported_alertgroup": "originGroupname",
-						"instance":            "foo",
+						alertNameLabel:      "OriginLabels",
+						alertGroupNameLabel: "Testing",
+						"instance":          "foo",
 					},
 					Annotations: map[string]string{
 						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
@@ -1022,12 +870,12 @@ func TestAlertingRule_Template(t *testing.T) {
 	fakeGroup := Group{Name: "TestRule_Exec"}
 	for _, tc := range testCases {
 		t.Run(tc.rule.Name, func(t *testing.T) {
-			fq := &datasource.FakeQuerier{}
+			fq := &fakeQuerier{}
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
-			tc.rule.state = &ruleState{entries: make([]StateEntry, 10)}
-			fq.Add(tc.metrics...)
-			if _, err := tc.rule.exec(context.TODO(), time.Now(), 0); err != nil {
+			tc.rule.state = newRuleState(10)
+			fq.add(tc.metrics...)
+			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 				t.Fatalf("unexpected err: %s", err)
 			}
 			for hash, expAlert := range tc.expAlerts {
@@ -1054,7 +902,7 @@ func TestAlertsToSend(t *testing.T) {
 		for i, a := range alerts {
 			ar.alerts[uint64(i)] = a
 		}
-		gotAlerts := ar.alertsToSend(resolveDuration, resendDelay)
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
 		if gotAlerts == nil && expAlerts == nil {
 			return
 		}
@@ -1070,36 +918,60 @@ func TestAlertsToSend(t *testing.T) {
 		})
 		for i, exp := range expAlerts {
 			got := gotAlerts[i]
-			if got.Name != exp.Name {
-				t.Fatalf("expected Name to be %v; got %v", exp.Name, got.Name)
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
 			}
 		}
 	}
 
-	f( // check if firing alerts need to be sent with non-zero resendDelay
-		[]*notifier.Alert{
-			{Name: "a", State: notifier.StateFiring, Start: ts},
-			// no need to resend firing
-			{Name: "b", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-30 * time.Second), End: ts.Add(5 * time.Minute)},
-			// last message is for resolved, send firing message this time
-			{Name: "c", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-30 * time.Second), End: ts.Add(-1 * time.Minute)},
-			// resend firing
-			{Name: "d", State: notifier.StateFiring, Start: ts, LastSent: ts.Add(-1 * time.Minute)},
-		},
-		[]*notifier.Alert{{Name: "a"}, {Name: "c"}, {Name: "d"}},
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
 		5*time.Minute, time.Minute,
 	)
-	f( // check if resolved alerts need to be sent with non-zero resendDelay
-		[]*notifier.Alert{
-			{Name: "a", State: notifier.StateInactive, ResolvedAt: ts, LastSent: ts.Add(-30 * time.Second)},
-			// no need to resend resolved
-			{Name: "b", State: notifier.StateInactive, ResolvedAt: ts, LastSent: ts},
-			// resend resolved
-			{Name: "c", State: notifier.StateInactive, ResolvedAt: ts.Add(-1 * time.Minute), LastSent: ts.Add(-1 * time.Minute)},
-		},
-		[]*notifier.Alert{{Name: "a"}, {Name: "c"}},
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
 		5*time.Minute, time.Minute,
 	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+	f( // inactive alert which has been sent already
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // inactive alert which has been resolved after last send
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
 }
 
 func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
@@ -1117,73 +989,13 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 		For:          waitFor,
 		EvalInterval: waitFor,
 		alerts:       make(map[uint64]*notifier.Alert),
-		state:        &ruleState{entries: make([]StateEntry, 10)},
-		metrics: &alertingRuleMetrics{
-			errors: utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_alerting_rules_errors_total{alertname=%q}`, name)),
-		},
+		state:        newRuleState(10),
 	}
 	return &rule
 }
 
-func newTestAlertingRuleWithEvalInterval(name string, waitFor, evalInterval time.Duration) *AlertingRule {
-	rule := newTestAlertingRule(name, waitFor)
-	rule.EvalInterval = evalInterval
-	return rule
-}
-
 func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	rule.KeepFiringFor = keepFiringFor
 	return rule
 }
-
-func TestAlertingRule_ToLabels(t *testing.T) {
-	metric := datasource.Metric{
-		Labels: []datasource.Label{
-			{Name: "instance", Value: "0.0.0.0:8800"},
-			{Name: "group", Value: "vmalert"},
-			{Name: "alertname", Value: "ConfigurationReloadFailure"},
-		},
-		Values:     []float64{1},
-		Timestamps: []int64{time.Now().UnixNano()},
-	}
-
-	ar := &AlertingRule{
-		Labels: map[string]string{
-			"instance": "override", // this should override instance with new value
-			"group":    "vmalert",  // this shouldn't have effect since value in metric is equal
-		},
-		Expr:      "sum(vmalert_alerting_rules_error) by(instance, group, alertname) > 0",
-		Name:      "AlertingRulesError",
-		GroupName: "vmalert",
-	}
-
-	expectedOriginLabels := map[string]string{
-		"instance":   "0.0.0.0:8800",
-		"group":      "vmalert",
-		"alertname":  "ConfigurationReloadFailure",
-		"alertgroup": "vmalert",
-	}
-
-	expectedProcessedLabels := map[string]string{
-		"instance":           "override",
-		"exported_instance":  "0.0.0.0:8800",
-		"alertname":          "AlertingRulesError",
-		"exported_alertname": "ConfigurationReloadFailure",
-		"group":              "vmalert",
-		"alertgroup":         "vmalert",
-	}
-
-	ls, err := ar.toLabels(metric, nil)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-
-	if !reflect.DeepEqual(ls.origin, expectedOriginLabels) {
-		t.Errorf("origin labels mismatch, got: %v, want: %v", ls.origin, expectedOriginLabels)
-	}
-
-	if !reflect.DeepEqual(ls.processed, expectedProcessedLabels) {
-		t.Errorf("processed labels mismatch, got: %v, want: %v", ls.processed, expectedProcessedLabels)
-	}
-}