deployment/docker: update alpine base image from 3.16.0 to 3.16.1

See https://alpinelinux.org/posts/Alpine-3.16.1-released.html

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,86 +0,0 @@
-name: Bug report
-description: Create a report to help us improve
-labels: [bug]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
-        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-        and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
-  - type: textarea
-    id: describe-the-bug
-    attributes:
-      label: Describe the bug
-      description: |
-        A clear and concise description of what the bug is.
-      placeholder: |
-        When I do `A` VictoriaMetrics does `B`. I expect it to do `C`.
-    validations:
-      required: true
-  - type: textarea
-    id: to-reproduce
-    attributes:
-      label: To Reproduce
-      description: |
-        Steps to reproduce the behavior.
-        If reproducing an issue requires some specific configuration file, please paste it here.
-      placeholder: |
-        Steps to reproduce the behavior.
-    validations:
-      required: true
-  - type: textarea
-    id: version
-    attributes:
-      label: Version
-      description: |
-        The line returned when passing `--version` command line flag to the binary. For example:
-        ```
-        $ ./victoria-metrics-prod --version
-        victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
-        ```
-    validations:
-      required: true
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs
-      description: |
-        Check if any warnings or errors were logged by VictoriaMetrics components
-        or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
-    validations:
-      required: false
-  - type: textarea
-    id: screenshots
-    attributes:
-      label: Screenshots
-      description: |
-        If applicable, add screenshots to help explain your problem.
-  
-        For VictoriaMetrics health-state issues please provide full-length screenshots
-        of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
-        
-        See how to setup monitoring here:
-          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
-    validations:
-      required: false
-  - type: textarea
-    id: flags
-    attributes:
-      label: Used command-line flags
-      description: |
-        Please provide the command-line flags used for running VictoriaMetrics and its components.
-    validations:
-      required: false
-  - type: textarea
-    id: additional-info
-    attributes:
-      label: Additional information
-      placeholder: |
-        Additional information that doesn't fit elsewhere
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/configuration.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Ask on Slack
-    url: https://slack.victoriametrics.com/
-    about: You can ask for help here!

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,43 +0,0 @@
-name: Feature request
-description: Suggest an idea for this project
-labels: [enhancement]
-body:
-  - type: textarea
-    id: describe-the-problem
-    attributes:
-      label: Is your feature request related to a problem? Please describe
-      description: |
-        A clear and concise description of what the problem is.
-      placeholder: |
-        Ex. I'm always frustrated when [...]
-    validations:
-      required: false
-  - type: textarea
-    id: describe-the-solution
-    attributes:
-      label: Describe the solution you'd like
-      description: |
-        A clear and concise description of what you want to happen.
-    validations:
-      required: true
-  - type: textarea
-    id: alternative-solutions
-    attributes:
-      label: Describe alternatives you've considered
-      description: |
-        A clear and concise description of any alternative solutions or features you've considered.
-      placeholder: |
-        I have tried to do `A`, but that doesn't solve a problem completely.
-        I have tried to do `A` and `B`, but implementing this would be better.
-    validations:
-      required: false
-  - type: textarea
-    id: feature-additional-info
-    attributes:
-      label: Additional information
-      description: |
-        Additional information which you consider helpful for implementing this feature.
-      placeholder: |
-        Add any other context or screenshots about the feature request here.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/question.yml

@@ -1,32 +0,0 @@
-name: Question
-description: Ask a question regarding VictoriaMetrics or its components
-labels: [question]
-body:
-  - type: textarea
-    id: describe-the-component
-    attributes:
-      label: Is your question request related to a specific component?
-      placeholder: |
-        VictoriaMetrics, vmagent, vmalert, vmui, etc...
-    validations:
-      required: false
-  - type: textarea
-    id: describe-the-question
-    attributes:
-      label: Describe the question in detail
-      description: |
-        A clear and concise description of the issue and the question.
-    validations:
-      required: true
-  - type: checkboxes
-    id: troubleshooting
-    attributes:
-      label: Troubleshooting docs
-      description: I am familiar with the following troubleshooting docs
-      options:
-        - label: General - https://docs.victoriametrics.com/Troubleshooting.html
-          required: false
-        - label: vmagent - https://docs.victoriametrics.com/vmagent.html#troubleshooting
-          required: false
-        - label: vmalert - https://docs.victoriametrics.com/vmalert.html#troubleshooting
-          required: false

.github/dependabot.yml

@@ -8,12 +8,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    open-pull-requests-limit: 0
   - package-ecosystem: "bundler"
     directory: "/docs"
     schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 0
+      interval: "daily"
   - package-ecosystem: "gomod"
     directory: "/app/vmui/packages/vmui/web"
     schedule:

.github/workflows/check-licenses.yml

@@ -14,25 +14,13 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
+      - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.17
+        id: go
       - name: Code checkout
         uses: actions/checkout@master
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
-
       - name: Check License
-        run: make check-licenses
+        run: |
+          make check-licenses

.github/workflows/codeql-analysis-js.yml

@@ -1,46 +0,0 @@
-name: "CodeQL - JS"
-
-on:
-  push:
-    branches: [master, cluster]
-    paths:
-      - "**.js"
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master, cluster]
-    paths:
-      - "**.js"
-  schedule:
-    - cron: "30 18 * * 2"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-  
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["javascript"]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
-        with:
-          category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -13,26 +13,12 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
+    branches: [ master, cluster ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
+    branches: [ master, cluster ]
   schedule:
-    - cron: "30 18 * * 2"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+    - cron: '30 18 * * 2'
 
 jobs:
   analyze:
@@ -46,58 +32,39 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ["go"]
+        language: [ 'go', 'javascript' ]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Learn more about CodeQL language support at https://git.io/codeql-language-support
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+    - name: Checkout repository
+      uses: actions/checkout@v3
 
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-          cache: false
-        if: ${{ matrix.language == 'go' }}
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-        if: ${{ matrix.language == 'go' }}
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
 
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
 
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
 
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
+    #- run: |
+    #   make bootstrap
+    #   make release
 
-      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-      #    and modify them (or add more) to build your code if your project
-      #    uses a compiled language
-
-      #- run: |
-      #   make bootstrap
-      #   make release
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -1,120 +1,41 @@
 name: main
 on:
   push:
-    branches:
-      - master
-      - cluster
     paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - 'docs/**'
+      - '**.md'
   pull_request:
-    branches:
-      - master
-      - cluster
     paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - 'docs/**'
+      - '**.md'
 permissions:
   contents: read
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-all-
-
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
-
   build:
-    needs: lint
-    name: build
+    name: Build
     runs-on: ubuntu-latest
     steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
       - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.17
         id: go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
+      - name: Code checkout
+        uses: actions/checkout@master
+      - name: Dependencies
+        run: |
+          make install-golint
+          make install-errcheck
+          make install-golangci-lint
       - name: Build
-        run: make crossbuild
-
-  test:
-    needs: lint
-    strategy:
-      matrix:
-        scenario: ["test-full", "test-pure", "test-full-386"]
-    name: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: run tests
-        run: make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.txt
+        env:
+          GO111MODULE: on
+        run: |
+            export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+            make check-all
+            git diff --exit-code
+            make test-full
+            make test-pure
+            make test-full-386
+            make vmcluster-crossbuild

.github/workflows/sync-docs.yml

@@ -1,66 +0,0 @@
-name: publish-docs
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'docs/**'
-  workflow_dispatch: {}
-env:
-  PAGEFIND_VERSION: "1.0.4"
-  HUGO_VERSION: "latest"
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-        with:
-          path: main
-      - name: Checkout private code
-        uses: actions/checkout@v4
-        with:
-          repository: VictoriaMetrics/vmdocs
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: docs
-      - uses: peaceiris/actions-hugo@v2
-        with:
-          hugo-version: ${{env.HUGO_VERSION}}
-          extended: true
-      - name: Install PageFind #install the static search engine for index build
-        uses: supplypike/setup-bin@v3
-        with:
-          uri: "https://github.com/CloudCannon/pagefind/releases/download/v${{env.PAGEFIND_VERSION}}/pagefind-v${{env.PAGEFIND_VERSION}}-x86_64-unknown-linux-musl.tar.gz"
-          name: "pagefind"
-          version: ${{env.PAGEFIND_VERSION}}
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v5
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-          workdir: docs
-      - name: Set short git commit SHA
-        id: vars
-        run: |
-          calculatedSha=$(git rev-parse --short ${{ github.sha }})
-          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
-        working-directory: main
-
-      - name: update code and commit
-        run: |
-          rm -rf content
-          cp -r ../main/docs content
-          make clean-after-copy
-          make build-search-index
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          git add .
-          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
-          git push
-        working-directory: docs

.gitignore

@@ -1,4 +1,3 @@
-/tmp
 /tags
 /pkg
 *.pprof
@@ -8,17 +7,17 @@
 *.test
 *.swp
 /gocache-for-docker
-/victoria-logs-data
 /victoria-metrics-data
 /vmagent-remotewrite-data
 /vmstorage-data
 /vmselect-cache
-/package/temp-deb-*
-/package/temp-rpm-*
-/package/*.deb
-/package/*.rpm
-.DS_store
+.DS_Store
+
+
+### terraform
+terraform.tfstate
+terraform.tfstate.*
+.terraform/
 Gemfile.lock
 /_site
 _site
-*.tmp

.golangci.yml

@@ -1,19 +0,0 @@
-run:
-  timeout: 2m
-
-linters:
-  enable:
-  - revive
-
-issues:
-  exclude-rules:
-  - linters:
-    - staticcheck
-    text: "SA(4003|1019|5011):"
-  include:
-  - EXC0012
-  - EXC0014
-
-linters-settings:
-  errcheck:
-    exclude: ./errcheck_excludes.txt

.wwhrd.yml

@@ -3,4 +3,3 @@ allowlist:
   - MIT
   - BSD-3-Clause
   - BSD-2-Clause
-  - ISC

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2023 VictoriaMetrics, Inc.
+   Copyright 2019-2022 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,11 +1,8 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell cat /proc/cpuinfo | grep -c processor)
-MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
-LATEST_TAG ?= cluster-latest
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -17,10 +14,7 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include docs/Makefile
 include deployment/*/Makefile
-include dashboards/Makefile
-include package/release/Makefile
 
 all: \
 	vminsert \
@@ -70,21 +64,14 @@ vmcluster-openbsd-amd64: \
 	vmselect-openbsd-amd64 \
 	vmstorage-openbsd-amd64
 
-vmcluster-windows-amd64: \
-	vminsert-windows-amd64 \
-	vmselect-windows-amd64 \
-	vmstorage-windows-amd64
-
-crossbuild: vmcluster-crossbuild
-
-vmcluster-crossbuild:
-	$(MAKE_PARALLEL) vmcluster-linux-amd64 \
-		vmcluster-linux-arm64 \
-		vmcluster-linux-arm \
-		vmcluster-linux-ppc64le \
-		vmcluster-linux-386 \
-		vmcluster-freebsd-amd64 \
-		vmcluster-openbsd-amd64
+vmcluster-crossbuild: \
+	vmcluster-linux-amd64 \
+	vmcluster-linux-arm64 \
+	vmcluster-linux-arm \
+	vmcluster-linux-ppc64le \
+	vmcluster-linux-386 \
+	vmcluster-freebsd-amd64 \
+	vmcluster-openbsd-amd64
 
 publish: \
 	publish-vminsert \
@@ -97,21 +84,19 @@ package: \
 	package-vmstorage
 
 publish-release:
-	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
+	git checkout $(TAG) && $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
 
-release:
-	$(MAKE_PARALLEL) release-vmcluster
+release: \
+	release-vmcluster
 
 release-vmcluster: \
 	release-vmcluster-linux-amd64 \
 	release-vmcluster-linux-arm64 \
 	release-vmcluster-freebsd-amd64 \
-	release-vmcluster-openbsd-amd64 \
-	release-vmcluster-windows-amd64
+	release-vmcluster-openbsd-amd64
 
 release-vmcluster-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
@@ -125,9 +110,6 @@ release-vmcluster-freebsd-amd64:
 release-vmcluster-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
 
-release-vmcluster-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmcluster-windows-goarch
-
 release-vmcluster-goos-goarch: \
 	vminsert-$(GOOS)-$(GOARCH)-prod \
 	vmselect-$(GOOS)-$(GOARCH)-prod \
@@ -147,25 +129,6 @@ release-vmcluster-goos-goarch: \
 		vmselect-$(GOOS)-$(GOARCH)-prod \
 		vmstorage-$(GOOS)-$(GOARCH)-prod
 
-release-vmcluster-windows-goarch: \
-	vminsert-windows-$(GOARCH)-prod \
-	vmselect-windows-$(GOARCH)-prod \
-	vmstorage-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vminsert-windows-$(GOARCH)-prod.exe \
-			vmselect-windows-$(GOARCH)-prod.exe \
-			vmstorage-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vminsert-windows-$(GOARCH)-prod.exe \
-			vmselect-windows-$(GOARCH)-prod.exe \
-			vmstorage-windows-$(GOARCH)-prod.exe \
-		> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vminsert-windows-$(GOARCH)-prod.exe \
-		vmselect-windows-$(GOARCH)-prod.exe \
-		vmstorage-windows-$(GOARCH)-prod.exe
-
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -174,102 +137,107 @@ fmt:
 	gofmt -l -w -s ./app
 
 vet:
-	go vet ./lib/...
-	go vet ./app/...
+	go vet -mod=vendor ./lib/...
+	go vet -mod=vendor ./app/...
 
-check-all: fmt vet golangci-lint govulncheck
+lint: install-golint
+	golint lib/...
+	golint app/...
+
+install-golint:
+	which golint || GO111MODULE=off go get golang.org/x/lint/golint
+
+errcheck: install-errcheck
+	errcheck -exclude=errcheck_excludes.txt ./lib/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmctl/...
+
+install-errcheck:
+	which errcheck || GO111MODULE=off go get github.com/kisielk/errcheck
+
+check-all: fmt vet lint errcheck golangci-lint
 
 test:
-	go test ./lib/... ./app/...
+	go test -mod=vendor ./lib/... ./app/...
 
 test-race:
-	go test -race ./lib/... ./app/...
+	go test -mod=vendor -race ./lib/... ./app/...
 
 test-pure:
-	CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
 
 test-full:
-	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	go test -bench=. ./lib/...
-	go test -bench=. ./app/...
+	go test -mod=vendor -bench=. ./lib/...
+	go test -mod=vendor -bench=. ./app/...
 
 benchmark-pure:
-	CGO_ENABLED=0 go test -bench=. ./lib/...
-	CGO_ENABLED=0 go test -bench=. ./app/...
+	CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
+	CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
 
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.20
+	go mod tidy -compat=1.17
 	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-goos-goarch:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-windows-goarch:
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
-	which qtc || go install github.com/valyala/quicktemplate/qtc@latest
+	which qtc || GO111MODULE=off go get github.com/valyala/quicktemplate/qtc
 
 
 golangci-lint: install-golangci-lint
-	golangci-lint run
+	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.55.1
-
-govulncheck: install-govulncheck
-	govulncheck ./...
-
-install-govulncheck:
-	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.46.2
 
 install-wwhrd:
-	which wwhrd || go install github.com/frapposelli/wwhrd@latest
+	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
 
 copy-docs:
-# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
-# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
-	echo "---" > ${DST}
-	@if [ ${ORDER} -ne 0 ]; then \
-		echo "sort: ${ORDER}" >> ${DST}; \
-		echo "weight: ${ORDER}" >> ${DST}; \
-		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
-	fi
-
-	echo "title: ${TITLE}" >> ${DST}
-	@if [ ${OLD_URL} ]; then \
-		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
-	fi
-	echo "---" >> ${DST}
+	echo "---\nsort: ${ORDER}\n---\n" > ${DST}
 	cat ${SRC} >> ${DST}
-	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
-	rm -rf docs/*.tmp
 
-# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
-# For ORDER=0 it adds no order tag/weight tag.
-# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
-# Images starting with <img src="docs/ are replaced with <img src="
-# Cluster docs are supposed to be ordered as 2nd.
-# The rest of docs is ordered manually.
+# Copies docs for all components and adds the order tag.
+# Cluster docs are supposed to be ordered as 9th.
+# For The rest of docs is ordered manually.t
 docs-sync:
-	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md OLD_URL='/Cluster-VictoriaMetrics.html' ORDER=2 TITLE='Cluster version' $(MAKE) copy-docs
+	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md ORDER=2 $(MAKE) copy-docs
+	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=3 $(MAKE) copy-docs
+	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=4 $(MAKE) copy-docs
+	SRC=app/vmauth/README.md DST=docs/vmauth.md ORDER=5 $(MAKE) copy-docs
+	SRC=app/vmbackup/README.md DST=docs/vmbackup.md ORDER=6 $(MAKE) copy-docs
+	SRC=app/vmrestore/README.md DST=docs/vmrestore.md ORDER=7 $(MAKE) copy-docs
+	SRC=app/vmctl/README.md DST=docs/vmctl.md ORDER=8 $(MAKE) copy-docs
+	SRC=app/vmgateway/README.md DST=docs/vmgateway.md ORDER=9 $(MAKE) copy-docs
+	SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md ORDER=10 $(MAKE) copy-docs

SECURITY.md

@@ -1,14 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-| Version | Supported          |
-|---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
-| v1.93.x LTS release | :white_check_mark: |
-| v1.87.x LTS release | :white_check_mark: |
-| other releases  | :x:                |
-
-## Reporting a Vulnerability
-
-Please report any security issues to security@victoriametrics.com

app/victoria-logs/Makefile

@@ -1,103 +0,0 @@
-# All these commands must run from repository root.
-
-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) app-local
-
-victoria-logs-race:
-	APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
-
-victoria-logs-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker
-
-victoria-logs-pure-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
-
-victoria-logs-linux-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
-
-victoria-logs-linux-arm-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
-
-victoria-logs-linux-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
-
-victoria-logs-linux-ppc64le-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
-
-victoria-logs-linux-386-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
-
-victoria-logs-darwin-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
-
-victoria-logs-darwin-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
-
-victoria-logs-freebsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
-
-victoria-logs-openbsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
-
-victoria-logs-windows-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
-
-package-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker
-
-package-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
-
-package-victoria-logs-amd64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
-
-package-victoria-logs-arm:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
-
-package-victoria-logs-arm64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
-
-package-victoria-logs-ppc64le:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
-
-package-victoria-logs-386:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-386
-
-publish-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) publish-via-docker
-
-victoria-logs-linux-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-ppc64le:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-s390x:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-386:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-freebsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-openbsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-windows-amd64:
-	GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
-
-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) app-local-pure

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image
-FROM $base_image
-
-EXPOSE 8428
-
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary
-COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -1,103 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-)
-
-var (
-	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	cgroup.SetGOGC(*gogc)
-	buildinfo.Init()
-	logger.Init()
-	pushmetrics.Init()
-
-	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
-	startTime := time.Now()
-
-	vlstorage.Init()
-	vlselect.Init()
-	vlinsert.Init()
-
-	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
-	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
-
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
-
-	vlinsert.Stop()
-	vlselect.Stop()
-	vlstorage.Stop()
-
-	fs.MustStopDirRemover()
-
-	logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
-}
-
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
-			return false
-		}
-		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/VictoriaLogs/'>https://docs.victoriametrics.com/VictoriaLogs/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints:</br>")
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"select/vmui", "Web UI for VictoriaLogs"},
-			{"metrics", "available service metrics"},
-			{"flags", "command-line flags"},
-		})
-		return true
-	}
-	if vlinsert.RequestHandler(w, r) {
-		return true
-	}
-	if vlselect.RequestHandler(w, r) {
-		return true
-	}
-	return false
-}
-
-func usage() {
-	const s = `
-victoria-logs is a log management and analytics service.
-
-See the docs at https://docs.victoriametrics.com/VictoriaLogs/
-`
-	flagutil.Usage(s)
-}

app/victoria-logs/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
-FROM $certs_image as certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 8428
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG TARGETARCH
-COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/vlinsert/elasticsearch/bulk_response.qtpl

@@ -1,20 +0,0 @@
-{% stripspace %}
-
-{% func BulkResponse(n int, tookMs int64) %}
-{
-	"took":{%dl tookMs %},
-	"errors":false,
-	"items":[
-		{% for i := 0; i < n; i++ %}
-		{
-			"create":{
-				"status":201
-			}
-		}
-		{% if i+1 < n %},{% endif %}
-		{% endfor %}
-	]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlinsert/elasticsearch/bulk_response.qtpl.go

@@ -1,69 +0,0 @@
-// Code generated by qtc from "bulk_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-package elasticsearch
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-func StreamBulkResponse(qw422016 *qt422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-	qw422016.N().S(`{"took":`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().DL(tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().S(`,"errors":false,"items":[`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-	for i := 0; i < n; i++ {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-		qw422016.N().S(`{"create":{"status":201}}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		if i+1 < n {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	qw422016.N().S(`]}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func WriteBulkResponse(qq422016 qtio422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	StreamBulkResponse(qw422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func BulkResponse(n int, tookMs int64) string {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	WriteBulkResponse(qb422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	return qs422016
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -1,281 +0,0 @@
-package elasticsearch
-
-import (
-	"bufio"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-)
-
-var (
-	elasticsearchVersion = flag.String("elasticsearch.version", "8.9.0", "Elasticsearch version to report to client")
-)
-
-// RequestHandler processes Elasticsearch insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	// This header is needed for Logstash
-	w.Header().Set("X-Elastic-Product", "Elasticsearch")
-
-	if strings.HasPrefix(path, "/_ilm/policy") {
-		// Return fake response for Elasticsearch ilm request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_index_template") {
-		// Return fake response for Elasticsearch index template request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_ingest") {
-		// Return fake response for Elasticsearch ingest pipeline request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_nodes") {
-		// Return fake response for Elasticsearch nodes discovery request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	switch path {
-	case "/":
-		switch r.Method {
-		case http.MethodGet:
-			// Return fake response for Elasticsearch ping request.
-			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
-			fmt.Fprintf(w, `{
-			"version": {
-				"number": %q
-			}
-		}`, *elasticsearchVersion)
-		case http.MethodHead:
-			// Return empty response for Logstash ping request.
-		}
-
-		return true
-	case "/_license":
-		// Return fake response for Elasticsearch license request.
-		fmt.Fprintf(w, `{
-			"license": {
-				"uid": "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
-				"type": "oss",
-				"status": "active",
-				"expiry_date_in_millis" : 4000000000000
-			}
-		}`)
-		return true
-	case "/_bulk":
-		startTime := time.Now()
-		bulkRequestsTotal.Inc()
-
-		cp, err := insertutils.GetCommonParams(r)
-		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		if err := vlstorage.CanWriteData(); err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-		processLogMessage := cp.GetProcessLogMessageFunc(lr)
-		isGzip := r.Header.Get("Content-Encoding") == "gzip"
-		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
-		vlstorage.MustAddRows(lr)
-		logstorage.PutLogRows(lr)
-		if err != nil {
-			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
-			return true
-		}
-
-		tookMs := time.Since(startTime).Milliseconds()
-		bw := bufferedwriter.Get(w)
-		defer bufferedwriter.Put(bw)
-		WriteBulkResponse(bw, n, tookMs)
-		_ = bw.Flush()
-
-		// update bulkRequestDuration only for successfully parsed requests
-		// There is no need in updating bulkRequestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		bulkRequestDuration.UpdateDuration(startTime)
-
-		return true
-	default:
-		return false
-	}
-}
-
-var (
-	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
-	rowsIngestedTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
-	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
-)
-
-func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (int, error) {
-	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
-
-	if isGzip {
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
-		}
-		defer common.PutGzipReader(zr)
-		r = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lb := lineBufferPool.Get()
-	defer lineBufferPool.Put(lb)
-
-	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
-	sc := bufio.NewScanner(wcr)
-	sc.Buffer(lb.B, len(lb.B))
-
-	n := 0
-	nCheckpoint := 0
-	for {
-		ok, err := readBulkLine(sc, timeField, msgField, processLogMessage)
-		wcr.DecConcurrency()
-		if err != nil || !ok {
-			rowsIngestedTotal.Add(n - nCheckpoint)
-			return n, err
-		}
-		n++
-		if batchSize := n - nCheckpoint; n >= 1000 {
-			rowsIngestedTotal.Add(batchSize)
-			nCheckpoint = n
-		}
-	}
-}
-
-var lineBufferPool bytesutil.ByteBufferPool
-
-func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (bool, error) {
-	var line []byte
-
-	// Read the command, must be "create" or "index"
-	for len(line) == 0 {
-		if !sc.Scan() {
-			if err := sc.Err(); err != nil {
-				if errors.Is(err, bufio.ErrTooLong) {
-					return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
-						insertutils.MaxLineSizeBytes.IntN())
-				}
-				return false, err
-			}
-			return false, nil
-		}
-		line = sc.Bytes()
-	}
-	lineStr := bytesutil.ToUnsafeString(line)
-	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
-		return false, fmt.Errorf(`unexpected command %q; expecting "create" or "index"`, line)
-	}
-
-	// Decode log message
-	if !sc.Scan() {
-		if err := sc.Err(); err != nil {
-			if errors.Is(err, bufio.ErrTooLong) {
-				return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
-			}
-			return false, err
-		}
-		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
-	}
-	line = sc.Bytes()
-	p := logjson.GetParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
-	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
-	logjson.PutParser(p)
-
-	return true, nil
-}
-
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		timestamp, err := parseElasticsearchTimestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
-	}
-	return 0, nil
-}
-
-func parseElasticsearchTimestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case - zero or empty timestamp must be substituted
-		// with the current time by the caller.
-		return 0, nil
-	}
-	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in milliseconds
-		n, err := strconv.ParseInt(s, 10, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
-		}
-		if n > int64(math.MaxInt64)/1e6 {
-			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
-		}
-		if n < int64(math.MinInt64)/1e6 {
-			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
-		}
-		n *= 1e6
-		return n, nil
-	}
-	if len(s) == len("YYYY-MM-DD") {
-		t, err := time.Parse("2006-01-02", s)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse date %q: %w", s, err)
-		}
-		return t.UnixNano(), nil
-	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
-	}
-	return t.UnixNano(), nil
-}

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -1,129 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestReadBulkRequestFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
-		}
-
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, "_time", "_msg", processLogMessage)
-		if err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if rows != 0 {
-			t.Fatalf("unexpected non-zero rows=%d", rows)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`{"create":{}}`)
-	f(`{"creat":{}}
-{}`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadBulkRequestSuccess(t *testing.T) {
-	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
-
-		// Read the request without compression
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, timeField, msgField, processLogMessage)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-
-		// Read the request with compression
-		timestamps = nil
-		result = ""
-		compressedData := compressData(data)
-		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest(r, true, timeField, msgField, processLogMessage)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Verify an empty data
-	f("", "_time", "_msg", 0, nil, "")
-	f("\n", "_time", "_msg", 0, nil, "")
-	f("\n\n", "_time", "_msg", 0, nil, "")
-
-	// Verify non-empty data
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	rowsExpected := 3
-	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
-	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
-}
-
-func compressData(s string) string {
-	var bb bytes.Buffer
-	zw := gzip.NewWriter(&bb)
-	if _, err := zw.Write([]byte(s)); err != nil {
-		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
-	}
-	if err := zw.Close(); err != nil {
-		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
-	}
-	return bb.String()
-}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -1,50 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("gzip:off", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, false)
-	})
-	b.Run("gzip:on", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, true)
-	})
-}
-
-func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	if isGzip {
-		data = compressData(data)
-	}
-	dataBytes := bytesutil.ToUnsafeBytes(data)
-
-	timeField := "@timestamp"
-	msgField := "message"
-	processLogMessage := func(timestmap int64, fields []logstorage.Field) {}
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		for pb.Next() {
-			r.Reset(dataBytes)
-			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}

app/vlinsert/insertutils/common_params.go

@@ -1,100 +0,0 @@
-package insertutils
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// CommonParams contains common HTTP parameters used by log ingestion APIs.
-//
-// See https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/#http-parameters
-type CommonParams struct {
-	TenantID     logstorage.TenantID
-	TimeField    string
-	MsgField     string
-	StreamFields []string
-	IgnoreFields []string
-
-	Debug           bool
-	DebugRequestURI string
-	DebugRemoteAddr string
-}
-
-// GetCommonParams returns CommonParams from r.
-func GetCommonParams(r *http.Request) (*CommonParams, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// Extract time field name from _time_field query arg
-	var timeField = "_time"
-	if tf := r.FormValue("_time_field"); tf != "" {
-		timeField = tf
-	}
-
-	// Extract message field name from _msg_field query arg
-	var msgField = ""
-	if msgf := r.FormValue("_msg_field"); msgf != "" {
-		msgField = msgf
-	}
-
-	streamFields := httputils.GetArray(r, "_stream_fields")
-	ignoreFields := httputils.GetArray(r, "ignore_fields")
-
-	debug := httputils.GetBool(r, "debug")
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-	if debug {
-		debugRequestURI = httpserver.GetRequestURI(r)
-		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
-	}
-
-	cp := &CommonParams{
-		TenantID:        tenantID,
-		TimeField:       timeField,
-		MsgField:        msgField,
-		StreamFields:    streamFields,
-		IgnoreFields:    ignoreFields,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
-	}
-	return cp, nil
-}
-
-// GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
-func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
-	return func(timestamp int64, fields []logstorage.Field) {
-		if len(fields) > *MaxFieldsPerLine {
-			rf := logstorage.RowFormatter(fields)
-			logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
-			rowsDroppedTotalTooManyFields.Inc()
-			return
-		}
-
-		lr.MustAdd(cp.TenantID, timestamp, fields)
-		if cp.Debug {
-			s := lr.GetRowString(0)
-			lr.ResetKeepSettings()
-			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
-			rowsDroppedTotalDebug.Inc()
-			return
-		}
-		if lr.NeedFlush() {
-			vlstorage.MustAddRows(lr)
-			lr.ResetKeepSettings()
-		}
-	}
-}
-
-var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/flags.go

@@ -1,15 +0,0 @@
-package insertutils
-
-import (
-	"flag"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-)
-
-var (
-	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
-
-	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
-)

app/vlinsert/jsonline/jsonline.go

@@ -1,161 +0,0 @@
-package jsonline
-
-import (
-	"bufio"
-	"errors"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// RequestHandler processes jsonline insert requests
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	startTime := time.Now()
-	w.Header().Add("Content-Type", "application/json")
-
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return true
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			logger.Errorf("cannot read gzipped _bulk request: %s", err)
-			return true
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lb := lineBufferPool.Get()
-	defer lineBufferPool.Put(lb)
-
-	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
-	sc := bufio.NewScanner(wcr)
-	sc.Buffer(lb.B, len(lb.B))
-
-	n := 0
-	for {
-		ok, err := readLine(sc, cp.TimeField, cp.MsgField, processLogMessage)
-		wcr.DecConcurrency()
-		if err != nil {
-			logger.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
-			break
-		}
-		if !ok {
-			break
-		}
-		n++
-		rowsIngestedTotal.Inc()
-	}
-
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-
-	// update jsonlineRequestDuration only for successfully parsed requests.
-	// There is no need in updating jsonlineRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	jsonlineRequestDuration.UpdateDuration(startTime)
-
-	return true
-}
-
-func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
-	var line []byte
-	for len(line) == 0 {
-		if !sc.Scan() {
-			if err := sc.Err(); err != nil {
-				if errors.Is(err, bufio.ErrTooLong) {
-					return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
-				}
-				return false, err
-			}
-			return false, nil
-		}
-		line = sc.Bytes()
-	}
-
-	p := logjson.GetParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
-	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
-	logjson.PutParser(p)
-
-	return true, nil
-}
-
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		timestamp, err := parseISO8601Timestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
-	}
-	return 0, nil
-}
-
-func parseISO8601Timestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case for returning the current timestamp.
-		// It must be automatically converted to the current timestamp by the caller.
-		return 0, nil
-	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
-	}
-	return t.UnixNano(), nil
-}
-
-var lineBufferPool bytesutil.ByteBufferPool
-
-var (
-	requestsTotal           = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	rowsIngestedTotal       = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
-	jsonlineRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
-)

app/vlinsert/jsonline/jsonline_test.go

@@ -1,70 +0,0 @@
-package jsonline
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"reflect"
-	"strings"
-	"testing"
-)
-
-func TestReadBulkRequestSuccess(t *testing.T) {
-	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
-
-		// Read the request without compression
-		r := bytes.NewBufferString(data)
-		sc := bufio.NewScanner(r)
-		rows := 0
-		for {
-			ok, err := readLine(sc, timeField, msgField, processLogMessage)
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-			if !ok {
-				break
-			}
-			rows++
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Verify non-empty data
-	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	rowsExpected := 3
-	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
-	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
-}

app/vlinsert/loki/loki.go

@@ -1,58 +0,0 @@
-package loki
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// RequestHandler processes Loki insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/api/v1/push":
-		return handleInsert(r, w)
-	case "/ready":
-		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("ready"))
-		return true
-	default:
-		return false
-	}
-}
-
-// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-func handleInsert(r *http.Request, w http.ResponseWriter) bool {
-	contentType := r.Header.Get("Content-Type")
-	switch contentType {
-	case "application/json":
-		return handleJSON(r, w)
-	default:
-		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		return handleProtobuf(r, w)
-	}
-}
-
-func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// If parsed tenant is (0,0) it is likely to be default tenant
-	// Try parsing tenant from Loki headers
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		org := r.Header.Get("X-Scope-OrgID")
-		if org != "" {
-			tenantID, err := logstorage.GetTenantIDFromString(org)
-			if err != nil {
-				return nil, err
-			}
-			cp.TenantID = tenantID
-		}
-
-	}
-
-	return cp, nil
-}

app/vlinsert/loki/loki_json.go

@@ -1,205 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-)
-
-var parserPool fastjson.ParserPool
-
-func handleJSON(r *http.Request, w http.ResponseWriter) bool {
-	startTime := time.Now()
-	lokiRequestsJSONTotal.Inc()
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return true
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
-	}
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseJSONRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
-		return true
-	}
-
-	rowsIngestedJSONTotal.Add(n)
-
-	// update lokiRequestJSONDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestJSONDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestJSONDuration.UpdateDuration(startTime)
-
-	return true
-}
-
-var (
-	lokiRequestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	rowsIngestedJSONTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
-	lokiRequestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
-)
-
-func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-
-	streamsV := v.Get("streams")
-	if streamsV == nil {
-		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
-	}
-	streams, err := streamsV.Array()
-	if err != nil {
-		return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
-	}
-
-	currentTimestamp := time.Now().UnixNano()
-	var commonFields []logstorage.Field
-	rowsIngested := 0
-	for _, stream := range streams {
-		// populate common labels from `stream` dict
-		commonFields = commonFields[:0]
-		labelsV := stream.Get("stream")
-		var labels *fastjson.Object
-		if labelsV != nil {
-			o, err := labelsV.Object()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
-			}
-			labels = o
-		}
-		labels.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
-			vStr, errLocal := v.StringBytes()
-			if errLocal != nil {
-				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-				return
-			}
-			commonFields = append(commonFields, logstorage.Field{
-				Name:  bytesutil.ToUnsafeString(k),
-				Value: bytesutil.ToUnsafeString(vStr),
-			})
-		})
-		if err != nil {
-			return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
-		}
-
-		// populate messages from `values` array
-		linesV := stream.Get("values")
-		if linesV == nil {
-			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
-		}
-		lines, err := linesV.Array()
-		if err != nil {
-			return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
-		}
-
-		fields := commonFields
-		for _, line := range lines {
-			lineA, err := line.Array()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
-			}
-			if len(lineA) != 2 {
-				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
-			}
-
-			// parse timestamp
-			timestamp, err := lineA[0].StringBytes()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
-			}
-			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
-			if err != nil {
-				return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
-			}
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			// parse log message
-			msg, err := lineA[1].StringBytes()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
-			}
-
-			fields = append(fields[:len(commonFields)], logstorage.Field{
-				Name:  "_msg",
-				Value: bytesutil.ToUnsafeString(msg),
-			})
-			processLogMessage(ts, fields)
-		}
-		rowsIngested += len(lines)
-	}
-
-	return rowsIngested, nil
-}
-
-func parseLokiTimestamp(s string) (int64, error) {
-	if s == "" {
-		// Special case - an empty timestamp must be substituted with the current time by the caller.
-		return 0, nil
-	}
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		// Fall back to parsing floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, err
-		}
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		n = int64(f)
-	}
-	if n < 0 {
-		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
-	}
-	return n, nil
-}

app/vlinsert/loki/loki_json_test.go

@@ -1,130 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestParseJSONRequestFailure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			t.Fatalf("unexpected call to parseJSONRequest callback!")
-		})
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if n != 0 {
-			t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
-		}
-	}
-	f(``)
-
-	// Invalid json
-	f(`{}`)
-	f(`[]`)
-	f(`"foo"`)
-	f(`123`)
-
-	// invalid type for `streams` item
-	f(`{"streams":123}`)
-
-	// Missing `values` item
-	f(`{"streams":[{}]}`)
-
-	// Invalid type for `values` item
-	f(`{"streams":[{"values":"foobar"}]}`)
-
-	// Invalid type for `stream` item
-	f(`{"streams":[{"stream":[],"values":[]}]}`)
-
-	// Invalid type for `values` individual item
-	f(`{"streams":[{"values":[123]}]}`)
-
-	// Invalid length of `values` individual item
-	f(`{"streams":[{"values":[[]]}]}`)
-	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
-
-	// Invalid type for timestamp inside `values` individual item
-	f(`{"streams":[{"values":[[123,"456"]}]}`)
-
-	// Invalid type for log message
-	f(`{"streams":[{"values":[["123",1234]]}]}`)
-}
-
-func TestParseJSONRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
-		t.Helper()
-		var lines []string
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
-}

app/vlinsert/loki/loki_json_timing_test.go

@@ -1,78 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func BenchmarkParseJSONRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseJSONRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		data := getJSONBody(streams, rows, labels)
-		for pb.Next() {
-			_, err := parseJSONRequest(data, func(timestamp int64, fields []logstorage.Field) {})
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getJSONBody(streams, rows, labels int) []byte {
-	body := append([]byte{}, `{"streams":[`...)
-	now := time.Now().UnixNano()
-	valuePrefix := fmt.Sprintf(`["%d","value_`, now)
-
-	for i := 0; i < streams; i++ {
-		body = append(body, `{"stream":{`...)
-
-		for j := 0; j < labels; j++ {
-			body = append(body, `"label_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `":"value_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, '"')
-			if j < labels-1 {
-				body = append(body, ',')
-			}
-
-		}
-		body = append(body, `}, "values":[`...)
-
-		for j := 0; j < rows; j++ {
-			body = append(body, valuePrefix...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `"]`...)
-			if j < rows-1 {
-				body = append(body, ',')
-			}
-		}
-
-		body = append(body, `]}`...)
-		if i < streams-1 {
-			body = append(body, ',')
-		}
-
-	}
-
-	body = append(body, `]}`...)
-
-	return body
-}

app/vlinsert/loki/loki_protobuf.go

@@ -1,189 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
-)
-
-var (
-	bytesBufPool bytesutil.ByteBufferPool
-	pushReqsPool sync.Pool
-)
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) bool {
-	startTime := time.Now()
-	lokiRequestsProtobufTotal.Inc()
-	wcr := writeconcurrencylimiter.GetReader(r.Body)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
-	}
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseProtobufRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
-		return true
-	}
-
-	rowsIngestedProtobufTotal.Add(n)
-
-	// update lokiRequestProtobufDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestProtobufDuration.UpdateDuration(startTime)
-
-	return true
-}
-
-var (
-	lokiRequestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	rowsIngestedProtobufTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
-	lokiRequestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
-)
-
-func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
-	bb := bytesBufPool.Get()
-	defer bytesBufPool.Put(bb)
-
-	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
-	if err != nil {
-		return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
-	}
-	bb.B = buf
-
-	req := getPushRequest()
-	defer putPushRequest(req)
-
-	err = req.Unmarshal(bb.B)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse request body: %w", err)
-	}
-
-	var commonFields []logstorage.Field
-	rowsIngested := 0
-	streams := req.Streams
-	currentTimestamp := time.Now().UnixNano()
-	for i := range streams {
-		stream := &streams[i]
-		// st.Labels contains labels for the stream.
-		// Labels are same for all entries in the stream.
-		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
-		if err != nil {
-			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
-		}
-		fields := commonFields
-
-		entries := stream.Entries
-		for j := range entries {
-			entry := &entries[j]
-			fields = append(fields[:len(commonFields)], logstorage.Field{
-				Name:  "_msg",
-				Value: entry.Line,
-			})
-			ts := entry.Timestamp.UnixNano()
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-			processLogMessage(ts, fields)
-		}
-		rowsIngested += len(stream.Entries)
-	}
-	return rowsIngested, nil
-}
-
-// parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
-//
-// See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
-func parsePromLabels(dst []logstorage.Field, s string) ([]logstorage.Field, error) {
-	// Make sure s is wrapped into `{...}`
-	s = strings.TrimSpace(s)
-	if len(s) < 2 {
-		return nil, fmt.Errorf("too short string to parse: %q", s)
-	}
-	if s[0] != '{' {
-		return nil, fmt.Errorf("missing `{` at the beginning of %q", s)
-	}
-	if s[len(s)-1] != '}' {
-		return nil, fmt.Errorf("missing `}` at the end of %q", s)
-	}
-	s = s[1 : len(s)-1]
-
-	for len(s) > 0 {
-		// Parse label name
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("cannot find `=` char for label value at %s", s)
-		}
-		name := s[:n]
-		s = s[n+1:]
-
-		// Parse label value
-		qs, err := strconv.QuotedPrefix(s)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse value for label %q at %s: %w", name, s, err)
-		}
-		s = s[len(qs):]
-		value, err := strconv.Unquote(qs)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unquote value %q for label %q: %w", qs, name, err)
-		}
-
-		// Append the found field to dst.
-		dst = append(dst, logstorage.Field{
-			Name:  name,
-			Value: value,
-		})
-
-		// Check whether there are other labels remaining
-		if len(s) == 0 {
-			break
-		}
-		if !strings.HasPrefix(s, ",") {
-			return nil, fmt.Errorf("missing `,` char at %s", s)
-		}
-		s = s[1:]
-		s = strings.TrimPrefix(s, " ")
-	}
-	return dst, nil
-}
-
-func getPushRequest() *PushRequest {
-	v := pushReqsPool.Get()
-	if v == nil {
-		return &PushRequest{}
-	}
-	return v.(*PushRequest)
-}
-
-func putPushRequest(req *PushRequest) {
-	req.Reset()
-	pushReqsPool.Put(req)
-}

app/vlinsert/loki/loki_protobuf_test.go

@@ -1,171 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/golang/snappy"
-)
-
-func TestParseProtobufRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
-		t.Helper()
-		var pr PushRequest
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			msg := ""
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					msg = f.Value
-				}
-			}
-			var a []string
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					continue
-				}
-				item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-				a = append(a, item)
-			}
-			labels := "{" + strings.Join(a, ", ") + "}"
-			pr.Streams = append(pr.Streams, Stream{
-				Labels: labels,
-				Entries: []Entry{
-					{
-						Timestamp: time.Unix(0, timestamp),
-						Line:      msg,
-					},
-				},
-			})
-		})
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if n != len(pr.Streams) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(pr.Streams), n)
-		}
-
-		data, err := pr.Marshal()
-		if err != nil {
-			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
-		}
-		encodedData := snappy.Encode(nil, data)
-
-		var lines []string
-		n, err = parseProtobufRequest(encodedData, func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
-}
-
-func TestParsePromLabelsSuccess(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		var a []string
-		for _, f := range fields {
-			a = append(a, fmt.Sprintf("%s=%q", f.Name, f.Value))
-		}
-		result := "{" + strings.Join(a, ", ") + "}"
-		if result != s {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, s)
-		}
-	}
-
-	f("{}")
-	f(`{foo="bar"}`)
-	f(`{foo="bar", baz="x", y="z"}`)
-	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
-}
-
-func TestParsePromLabelsFailure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(fields) > 0 {
-			t.Fatalf("unexpected non-empty fields: %s", fields)
-		}
-	}
-
-	f("")
-	f("{")
-	f(`{foo}`)
-	f(`{foo=bar}`)
-	f(`{foo="bar}`)
-	f(`{foo="ba\",r}`)
-	f(`{foo="bar" baz="aa"}`)
-	f(`foobar`)
-	f(`foo{bar="baz"}`)
-}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -1,66 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/golang/snappy"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			_, err := parseProtobufRequest(body, func(timestamp int64, fields []logstorage.Field) {})
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(streams, rows, labels int) []byte {
-	var pr PushRequest
-
-	for i := 0; i < streams; i++ {
-		var st Stream
-
-		st.Labels = `{`
-		for j := 0; j < labels; j++ {
-			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
-			if j < labels-1 {
-				st.Labels += `,`
-			}
-		}
-		st.Labels += `}`
-
-		for j := 0; j < rows; j++ {
-			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
-		}
-
-		pr.Streams = append(pr.Streams, st)
-	}
-
-	body, _ := pr.Marshal()
-	encodedBody := snappy.Encode(nil, body)
-
-	return encodedBody
-}

app/vlinsert/loki/push_request.proto

@@ -1,38 +0,0 @@
-syntax = "proto3";
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package logproto;
-
-import "gogoproto/gogo.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
-
-message PushRequest {
-  repeated StreamAdapter streams = 1 [
-    (gogoproto.jsontag) = "streams",
-    (gogoproto.customtype) = "Stream"
-  ];
-}
-
-message StreamAdapter {
-  string labels = 1 [(gogoproto.jsontag) = "labels"];
-  repeated EntryAdapter entries = 2 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "entries"
-  ];
-  // hash contains the original hash of the stream.
-  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
-}
-
-message EntryAdapter {
-  google.protobuf.Timestamp timestamp = 1 [
-    (gogoproto.stdtime) = true,
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "ts"
-  ];
-  string line = 2 [(gogoproto.jsontag) = "line"];
-}

app/vlinsert/loki/timestamp.go

@@ -1,110 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"errors"
-	"strconv"
-	"time"
-
-	"github.com/gogo/protobuf/types"
-)
-
-const (
-	// Seconds field of the earliest valid Timestamp.
-	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	minValidSeconds = -62135596800
-	// Seconds field just after the latest valid Timestamp.
-	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	maxValidSeconds = 253402300800
-)
-
-// validateTimestamp determines whether a Timestamp is valid.
-// A valid timestamp represents a time in the range
-// [0001-01-01, 10000-01-01) and has a Nanos field
-// in the range [0, 1e9).
-//
-// If the Timestamp is valid, validateTimestamp returns nil.
-// Otherwise, it returns an error that describes
-// the problem.
-//
-// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
-func validateTimestamp(ts *types.Timestamp) error {
-	if ts == nil {
-		return errors.New("timestamp: nil Timestamp")
-	}
-	if ts.Seconds < minValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
-	}
-	if ts.Seconds >= maxValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
-	}
-	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
-		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
-	}
-	return nil
-}
-
-// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
-// but avoids the escape incurred by using fmt.Sprintf, eliminating
-// unnecessary heap allocations.
-func formatTimestamp(ts *types.Timestamp) string {
-	if ts == nil {
-		return "nil"
-	}
-
-	seconds := strconv.FormatInt(ts.Seconds, 10)
-	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
-	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
-}
-
-func sizeOfStdTime(t time.Time) int {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0
-	}
-	return ts.Size()
-}
-
-func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0, err
-	}
-	return ts.MarshalTo(data)
-}
-
-func stdTimeUnmarshal(t *time.Time, data []byte) error {
-	ts := &types.Timestamp{}
-	if err := ts.Unmarshal(data); err != nil {
-		return err
-	}
-	tt, err := timestampFromProto(ts)
-	if err != nil {
-		return err
-	}
-	*t = tt
-	return nil
-}
-
-func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
-	// Don't return the zero value on error, because corresponds to a valid
-	// timestamp. Instead return whatever time.Unix gives us.
-	var t time.Time
-	if ts == nil {
-		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
-	} else {
-		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	}
-	return t, validateTimestamp(ts)
-}
-
-func timestampProto(t time.Time) (types.Timestamp, error) {
-	ts := types.Timestamp{
-		Seconds: t.Unix(),
-		Nanos:   int32(t.Nanosecond()),
-	}
-	return ts, validateTimestamp(&ts)
-}

app/vlinsert/loki/types.go

@@ -1,481 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"fmt"
-	"io"
-	"time"
-)
-
-// Stream contains a unique labels set as a string and a set of entries for it.
-// We are not using the proto generated version but this custom one so that we
-// can improve serialization see benchmark.
-type Stream struct {
-	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
-	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
-	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
-}
-
-// Entry is a log entry with a timestamp.
-type Entry struct {
-	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
-	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Stream) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Hash != 0 {
-		i = encodeVarintPush(dAtA, i, m.Hash)
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Entries) > 0 {
-		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintPush(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x12
-		}
-	}
-	if len(m.Labels) > 0 {
-		i -= len(m.Labels)
-		copy(dAtA[i:], m.Labels)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Entry) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if len(m.Line) > 0 {
-		i -= len(m.Line)
-		copy(dAtA[i:], m.Line)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
-		i--
-		dAtA[i] = 0x12
-	}
-	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
-	if err7 != nil {
-		return 0, err7
-	}
-	i -= n7
-	i = encodeVarintPush(dAtA, i, uint64(n7))
-	i--
-	dAtA[i] = 0xa
-	return len(dAtA) - i, nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Stream) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Labels = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Entries = append(m.Entries, Entry{})
-			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
-			}
-			m.Hash = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Hash |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Entry) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Line = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Size returns the size of the serialized Stream.
-func (m *Stream) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Labels)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	if len(m.Entries) > 0 {
-		for _, e := range m.Entries {
-			l = e.Size()
-			n += 1 + l + sovPush(uint64(l))
-		}
-	}
-	if m.Hash != 0 {
-		n += 1 + sovPush(m.Hash)
-	}
-	return n
-}
-
-// Size returns the size of the serialized Entry
-func (m *Entry) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = sizeOfStdTime(m.Timestamp)
-	n += 1 + l + sovPush(uint64(l))
-	l = len(m.Line)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	return n
-}
-
-// Equal returns true if the two Streams are equal.
-func (m *Stream) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Stream)
-	if !ok {
-		that2, ok := that.(Stream)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if m.Labels != that1.Labels {
-		return false
-	}
-	if len(m.Entries) != len(that1.Entries) {
-		return false
-	}
-	for i := range m.Entries {
-		if !m.Entries[i].Equal(that1.Entries[i]) {
-			return false
-		}
-	}
-	return m.Hash == that1.Hash
-}
-
-// Equal returns true if the two Entries are equal.
-func (m *Entry) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Entry)
-	if !ok {
-		that2, ok := that.(Entry)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if !m.Timestamp.Equal(that1.Timestamp) {
-		return false
-	}
-	if m.Line != that1.Line {
-		return false
-	}
-	return true
-}

app/vlinsert/main.go

@@ -1,43 +0,0 @@
-package vlinsert
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-)
-
-// Init initializes vlinsert
-func Init() {
-}
-
-// Stop stops vlinsert
-func Stop() {
-}
-
-// RequestHandler handles insert requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if !strings.HasPrefix(path, "/insert/") {
-		// Skip requests, which do not start with /insert/, since these aren't our requests.
-		return false
-	}
-	path = strings.TrimPrefix(path, "/insert")
-	path = strings.ReplaceAll(path, "//", "/")
-
-	if path == "/jsonline" {
-		return jsonline.RequestHandler(w, r)
-	}
-	switch {
-	case strings.HasPrefix(path, "/elasticsearch/"):
-		path = strings.TrimPrefix(path, "/elasticsearch")
-		return elasticsearch.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/loki/"):
-		path = strings.TrimPrefix(path, "/loki")
-		return loki.RequestHandler(path, w, r)
-	default:
-		return false
-	}
-}

app/vlselect/logsql/logsql.go

@@ -1,56 +0,0 @@
-package logsql
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	maxSortBufferSize = flagutil.NewBytes("select.maxSortBufferSize", 1024*1024, "Query results from /select/logsql/query are automatically sorted by _time "+
-		"if their summary size doesn't exceed this value; otherwise, query results are streamed in the response without sorting; "+
-		"too big value for this flag may result in high memory usage since the sorting is performed in memory")
-)
-
-// ProcessQueryRequest handles /select/logsql/query request
-func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse query [%s]: %s", qStr, err)
-		return
-	}
-	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
-
-	sw := getSortWriter()
-	sw.Init(w, maxSortBufferSize.IntN())
-	tenantIDs := []logstorage.TenantID{tenantID}
-	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
-		if len(columns) == 0 {
-			return
-		}
-		rowsCount := len(columns[0].Values)
-
-		bb := blockResultPool.Get()
-		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
-			WriteJSONRow(bb, columns, rowIdx)
-		}
-		sw.MustWrite(bb.B)
-		blockResultPool.Put(bb)
-	})
-	sw.FinalFlush()
-	putSortWriter(sw)
-}
-
-var blockResultPool bytesutil.ByteBufferPool

app/vlselect/logsql/query_response.qtpl

@@ -1,41 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// JSONRow creates JSON row from the given fields.
-{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-{
-	{% code c := &columns[0] %}
-	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% code columns = columns[1:] %}
-	{% for colIdx := range columns %}
-		{% code c := &columns[colIdx] %}
-		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% endfor %}
-}{% newline %}
-{% endfunc %}
-
-// JSONRows prints formatted rows
-{% func JSONRows(rows [][]logstorage.Field) %}
-	{% if len(rows) == 0 %}
-		{% return %}
-	{% endif %}
-	{% for _, fields := range rows %}
-	{
-		{% if len(fields) > 0 %}
-			{% code
-				f := fields[0]
-				fields = fields[1:]
-			%}
-			{%q= f.Name %}:{%q= f.Value %}
-			{% for _, f := range fields %}
-				,{%q= f.Name %}:{%q= f.Value %}
-			{% endfor %}
-		{% endif %}
-	}{% newline %}
-	{% endfor %}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/query_response.qtpl.go

@@ -1,166 +0,0 @@
-// Code generated by qtc from "query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/query_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/query_response.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// JSONRow creates JSON row from the given fields.
-
-//line app/vlselect/logsql/query_response.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:8
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:10
-	c := &columns[0]
-
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:12
-	columns = columns[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:13
-	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:14
-		c := &columns[colIdx]
-
-//line app/vlselect/logsql/query_response.qtpl:14
-		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:16
-	}
-//line app/vlselect/logsql/query_response.qtpl:16
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:17
-	qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-//line app/vlselect/logsql/query_response.qtpl:18
-func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-//line app/vlselect/logsql/query_response.qtpl:18
-func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:18
-	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-// JSONRows prints formatted rows
-
-//line app/vlselect/logsql/query_response.qtpl:21
-func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:22
-	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:23
-		return
-//line app/vlselect/logsql/query_response.qtpl:24
-	}
-//line app/vlselect/logsql/query_response.qtpl:25
-	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:25
-		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:27
-		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:29
-			f := fields[0]
-			fields = fields[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:33
-			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:33
-				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:35
-			}
-//line app/vlselect/logsql/query_response.qtpl:36
-		}
-//line app/vlselect/logsql/query_response.qtpl:36
-		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:37
-		qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:38
-	}
-//line app/vlselect/logsql/query_response.qtpl:39
-}
-
-//line app/vlselect/logsql/query_response.qtpl:39
-func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:39
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-}
-
-//line app/vlselect/logsql/query_response.qtpl:39
-func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:39
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:39
-	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:39
-}

app/vlselect/logsql/sort_writer.go

@@ -1,225 +0,0 @@
-package logsql
-
-import (
-	"bytes"
-	"io"
-	"sort"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func getSortWriter() *sortWriter {
-	v := sortWriterPool.Get()
-	if v == nil {
-		return &sortWriter{}
-	}
-	return v.(*sortWriter)
-}
-
-func putSortWriter(sw *sortWriter) {
-	sw.reset()
-	sortWriterPool.Put(sw)
-}
-
-var sortWriterPool sync.Pool
-
-// sortWriter expects JSON line stream to be written to it.
-//
-// It buffers the incoming data until its size reaches maxBufLen.
-// Then it streams the buffered data and all the incoming data to w.
-//
-// The FinalFlush() must be called when all the data is written.
-// If the buf isn't empty at FinalFlush() call, then the buffered data
-// is sorted by _time field.
-type sortWriter struct {
-	mu         sync.Mutex
-	w          io.Writer
-	maxBufLen  int
-	buf        []byte
-	bufFlushed bool
-
-	hasErr bool
-}
-
-func (sw *sortWriter) reset() {
-	sw.w = nil
-	sw.maxBufLen = 0
-	sw.buf = sw.buf[:0]
-	sw.bufFlushed = false
-	sw.hasErr = false
-}
-
-func (sw *sortWriter) Init(w io.Writer, maxBufLen int) {
-	sw.reset()
-
-	sw.w = w
-	sw.maxBufLen = maxBufLen
-}
-
-func (sw *sortWriter) MustWrite(p []byte) {
-	sw.mu.Lock()
-	defer sw.mu.Unlock()
-
-	if sw.hasErr {
-		return
-	}
-
-	if sw.bufFlushed {
-		if _, err := sw.w.Write(p); err != nil {
-			sw.hasErr = true
-		}
-		return
-	}
-	if len(sw.buf)+len(p) < sw.maxBufLen {
-		sw.buf = append(sw.buf, p...)
-		return
-	}
-	sw.bufFlushed = true
-	if len(sw.buf) > 0 {
-		if _, err := sw.w.Write(sw.buf); err != nil {
-			sw.hasErr = true
-			return
-		}
-		sw.buf = sw.buf[:0]
-	}
-	if _, err := sw.w.Write(p); err != nil {
-		sw.hasErr = true
-	}
-}
-
-func (sw *sortWriter) FinalFlush() {
-	if sw.hasErr || sw.bufFlushed {
-		return
-	}
-	rs := getRowsSorter()
-	rs.parseRows(sw.buf)
-	rs.sort()
-	WriteJSONRows(sw.w, rs.rows)
-	putRowsSorter(rs)
-}
-
-func getRowsSorter() *rowsSorter {
-	v := rowsSorterPool.Get()
-	if v == nil {
-		return &rowsSorter{}
-	}
-	return v.(*rowsSorter)
-}
-
-func putRowsSorter(rs *rowsSorter) {
-	rs.reset()
-	rowsSorterPool.Put(rs)
-}
-
-var rowsSorterPool sync.Pool
-
-type rowsSorter struct {
-	buf       []byte
-	fieldsBuf []logstorage.Field
-	rows      [][]logstorage.Field
-	times     []string
-}
-
-func (rs *rowsSorter) reset() {
-	rs.buf = rs.buf[:0]
-
-	fieldsBuf := rs.fieldsBuf
-	for i := range fieldsBuf {
-		fieldsBuf[i].Reset()
-	}
-	rs.fieldsBuf = fieldsBuf[:0]
-
-	rows := rs.rows
-	for i := range rows {
-		rows[i] = nil
-	}
-	rs.rows = rows[:0]
-
-	times := rs.times
-	for i := range times {
-		times[i] = ""
-	}
-	rs.times = times[:0]
-}
-
-func (rs *rowsSorter) parseRows(src []byte) {
-	rs.reset()
-
-	buf := rs.buf
-	fieldsBuf := rs.fieldsBuf
-	rows := rs.rows
-	times := rs.times
-
-	p := logjson.GetParser()
-	for len(src) > 0 {
-		var line []byte
-		n := bytes.IndexByte(src, '\n')
-		if n < 0 {
-			line = src
-			src = nil
-		} else {
-			line = src[:n]
-			src = src[n+1:]
-		}
-		if len(line) == 0 {
-			continue
-		}
-
-		if err := p.ParseLogMessage(line); err != nil {
-			logger.Panicf("BUG: unexpected invalid JSON line: %s", err)
-		}
-
-		timeValue := ""
-		fieldsBufLen := len(fieldsBuf)
-		for _, f := range p.Fields {
-			bufLen := len(buf)
-			buf = append(buf, f.Name...)
-			name := bytesutil.ToUnsafeString(buf[bufLen:])
-
-			bufLen = len(buf)
-			buf = append(buf, f.Value...)
-			value := bytesutil.ToUnsafeString(buf[bufLen:])
-
-			fieldsBuf = append(fieldsBuf, logstorage.Field{
-				Name:  name,
-				Value: value,
-			})
-
-			if name == "_time" {
-				timeValue = value
-			}
-		}
-		rows = append(rows, fieldsBuf[fieldsBufLen:])
-		times = append(times, timeValue)
-	}
-	logjson.PutParser(p)
-
-	rs.buf = buf
-	rs.fieldsBuf = fieldsBuf
-	rs.rows = rows
-	rs.times = times
-}
-
-func (rs *rowsSorter) Len() int {
-	return len(rs.rows)
-}
-
-func (rs *rowsSorter) Less(i, j int) bool {
-	times := rs.times
-	return times[i] < times[j]
-}
-
-func (rs *rowsSorter) Swap(i, j int) {
-	times := rs.times
-	rows := rs.rows
-	times[i], times[j] = times[j], times[i]
-	rows[i], rows[j] = rows[j], rows[i]
-}
-
-func (rs *rowsSorter) sort() {
-	sort.Sort(rs)
-}

app/vlselect/logsql/sort_writer_test.go

@@ -1,39 +0,0 @@
-package logsql
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-)
-
-func TestSortWriter(t *testing.T) {
-	f := func(maxBufLen int, data string, expectedResult string) {
-		t.Helper()
-
-		var bb bytes.Buffer
-		sw := getSortWriter()
-		sw.Init(&bb, maxBufLen)
-
-		for _, s := range strings.Split(data, "\n") {
-			sw.MustWrite([]byte(s + "\n"))
-		}
-		sw.FinalFlush()
-		putSortWriter(sw)
-
-		result := bb.String()
-		if result != expectedResult {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, expectedResult)
-		}
-	}
-
-	f(100, "", "")
-	f(100, "{}", "{}\n")
-
-	data := `{"_time":"def","_msg":"xxx"}
-{"_time":"abc","_msg":"foo"}`
-	resultExpected := `{"_time":"abc","_msg":"foo"}
-{"_time":"def","_msg":"xxx"}
-`
-	f(100, data, resultExpected)
-	f(10, data, data+"\n")
-}

app/vlselect/main.go

@@ -1,168 +0,0 @@
-package vlselect
-
-import (
-	"embed"
-	"flag"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration")
-	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
-		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
-)
-
-func getDefaultMaxConcurrentRequests() int {
-	n := cgroup.AvailableCPUs()
-	if n <= 4 {
-		n *= 2
-	}
-	if n > 16 {
-		// A single request can saturate all the CPU cores, so there is no sense
-		// in allowing higher number of concurrent requests - they will just contend
-		// for unavailable CPU time.
-		n = 16
-	}
-	return n
-}
-
-// Init initializes vlselect
-func Init() {
-	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
-}
-
-// Stop stops vlselect
-func Stop() {
-}
-
-var concurrencyLimitCh chan struct{}
-
-var (
-	concurrencyLimitReached = metrics.NewCounter(`vl_concurrent_select_limit_reached_total`)
-	concurrencyLimitTimeout = metrics.NewCounter(`vl_concurrent_select_limit_timeout_total`)
-
-	_ = metrics.NewGauge(`vl_concurrent_select_capacity`, func() float64 {
-		return float64(cap(concurrencyLimitCh))
-	})
-	_ = metrics.NewGauge(`vl_concurrent_select_current`, func() float64 {
-		return float64(len(concurrencyLimitCh))
-	})
-)
-
-//go:embed vmui
-var vmuiFiles embed.FS
-
-var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
-
-// RequestHandler handles select requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if !strings.HasPrefix(path, "/select/") {
-		// Skip requests, which do not start with /select/, since these aren't our requests.
-		return false
-	}
-	path = strings.TrimPrefix(path, "/select")
-	path = strings.ReplaceAll(path, "//", "/")
-
-	if path == "/vmui" {
-		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
-		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
-		// is hidden behind vmauth or similar proxy.
-		_ = r.ParseForm()
-		newURL := "vmui/?" + r.Form.Encode()
-		httpserver.Redirect(w, newURL)
-		return true
-	}
-	if strings.HasPrefix(path, "/vmui/") {
-		if strings.HasPrefix(path, "/vmui/static/") {
-			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
-			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
-			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
-			w.Header().Set("Cache-Control", "max-age=31536000")
-		}
-		r.URL.Path = path
-		vmuiFileServer.ServeHTTP(w, r)
-		return true
-	}
-
-	// Limit the number of concurrent queries, which can consume big amounts of CPU.
-	startTime := time.Now()
-	stopCh := r.Context().Done()
-	select {
-	case concurrencyLimitCh <- struct{}{}:
-		defer func() { <-concurrencyLimitCh }()
-	default:
-		// Sleep for a while until giving up. This should resolve short bursts in requests.
-		concurrencyLimitReached.Inc()
-		d := getMaxQueryDuration(r)
-		if d > *maxQueueDuration {
-			d = *maxQueueDuration
-		}
-		t := timerpool.Get(d)
-		select {
-		case concurrencyLimitCh <- struct{}{}:
-			timerpool.Put(t)
-			defer func() { <-concurrencyLimitCh }()
-		case <-stopCh:
-			timerpool.Put(t)
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-				time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			return true
-		case <-t.C:
-			timerpool.Put(t)
-			concurrencyLimitTimeout.Inc()
-			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
-					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusServiceUnavailable,
-			}
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-	}
-
-	switch {
-	case path == "/logsql/query":
-		logsqlQueryRequests.Inc()
-		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(w, r, stopCh)
-		return true
-	default:
-		return false
-	}
-}
-
-// getMaxQueryDuration returns the maximum duration for query from r.
-func getMaxQueryDuration(r *http.Request) time.Duration {
-	dms, err := httputils.GetDuration(r, "timeout", 0)
-	if err != nil {
-		dms = 0
-	}
-	d := time.Duration(dms) * time.Millisecond
-	if d <= 0 || d > *maxQueryDuration {
-		d = *maxQueryDuration
-	}
-	return d
-}
-
-var (
-	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-)

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +0,0 @@
-{
-  "files": {
-    "main.css": "./static/css/main.d1313636.css",
-    "main.js": "./static/js/main.1919fefe.js",
-    "static/js/522.da77e7b3.chunk.js": "./static/js/522.da77e7b3.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.8644fd7c964802dd34a9.md",
-    "index.html": "./index.html"
-  },
-  "entrypoints": [
-    "static/css/main.d1313636.css",
-    "static/js/main.1919fefe.js"
-  ]
-}

app/vlselect/vmui/index.html

@@ -1 +0,0 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.1919fefe.js"></script><link href="./static/css/main.d1313636.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/manifest.json

@@ -1,20 +0,0 @@
-{
-  "short_name": "Victoria Metrics UI",
-  "name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
-  "icons": [
-    {
-      "src": "favicon-32x32.png",
-      "sizes": "32x32",
-      "type": "image/png"
-    },
-    {
-      "src": "apple-touch-icon.png",
-      "type": "image/png",
-      "sizes": "192x192"
-    }
-  ],
-  "start_url": ".",
-  "display": "standalone",
-  "theme_color": "#000000",
-  "background_color": "#ffffff"
-}

app/vlselect/vmui/robots.txt

@@ -1,3 +0,0 @@
-# https://www.robotstxt.org/robotstxt.html
-User-agent: *
-Disallow:

app/vlselect/vmui/static/js/main.1919fefe.js.LICENSE.txt

@@ -1,40 +0,0 @@
-/*!
-	Copyright (c) 2018 Jed Watson.
-	Licensed under the MIT License (MIT), see
-	http://jedwatson.github.io/classnames
-*/
-
-/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
-
-/**
- * @remix-run/router v1.10.0
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router DOM v6.17.0
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router v6.17.0
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */

app/vlstorage/main.go

@@ -1,215 +0,0 @@
-package vlstorage
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
-		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
-	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory with the VictoriaLogs data; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#storage")
-	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
-		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
-	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows")
-	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/ ; see also -logNewStreams")
-	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
-		"the storage stops accepting new data")
-)
-
-// Init initializes vlstorage.
-//
-// Stop must be called when vlstorage is no longer needed
-func Init() {
-	if strg != nil {
-		logger.Panicf("BUG: Init() has been already called")
-	}
-
-	if retentionPeriod.Duration() < 24*time.Hour {
-		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
-	}
-	cfg := &logstorage.StorageConfig{
-		Retention:             retentionPeriod.Duration(),
-		FlushInterval:         *inmemoryDataFlushInterval,
-		FutureRetention:       futureRetention.Duration(),
-		LogNewStreams:         *logNewStreams,
-		LogIngestedRows:       *logIngestedRows,
-		MinFreeDiskSpaceBytes: minFreeDiskSpaceBytes.N,
-	}
-	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
-	startTime := time.Now()
-	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
-
-	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
-	logger.Infof("successfully opened storage in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
-		time.Since(startTime).Seconds(), ss.FileParts, ss.FileBlocks, ss.FileRowsCount, ss.CompressedFileSize)
-	storageMetrics = initStorageMetrics(strg)
-
-	metrics.RegisterSet(storageMetrics)
-}
-
-// Stop stops vlstorage.
-func Stop() {
-	metrics.UnregisterSet(storageMetrics)
-	storageMetrics = nil
-
-	strg.MustClose()
-	strg = nil
-}
-
-var strg *logstorage.Storage
-var storageMetrics *metrics.Set
-
-// CanWriteData returns non-nil error if it cannot write data to vlstorage.
-func CanWriteData() error {
-	if strg.IsReadOnly() {
-		return &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
-				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
-			StatusCode: http.StatusTooManyRequests,
-		}
-	}
-	return nil
-}
-
-// MustAddRows adds lr to vlstorage
-//
-// It is advised to call CanWriteData() before calling MustAddRows()
-func MustAddRows(lr *logstorage.LogRows) {
-	strg.MustAddRows(lr)
-}
-
-// RunQuery runs the given q and calls processBlock for the returned data blocks
-func RunQuery(tenantIDs []logstorage.TenantID, q *logstorage.Query, stopCh <-chan struct{}, processBlock func(columns []logstorage.BlockColumn)) {
-	strg.RunQuery(tenantIDs, q, stopCh, processBlock)
-}
-
-func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
-	ssCache := &logstorage.StorageStats{}
-	var ssCacheLock sync.Mutex
-	var lastUpdateTime time.Time
-
-	m := func() *logstorage.StorageStats {
-		ssCacheLock.Lock()
-		defer ssCacheLock.Unlock()
-		if time.Since(lastUpdateTime) < time.Second {
-			return ssCache
-		}
-		var ss logstorage.StorageStats
-		strg.UpdateStats(&ss)
-		ssCache = &ss
-		lastUpdateTime = time.Now()
-		return ssCache
-	}
-
-	ms := metrics.NewSet()
-
-	ms.NewGauge(fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), func() float64 {
-		return float64(fs.MustGetFreeSpace(*storageDataPath))
-	})
-	ms.NewGauge(fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), func() float64 {
-		if m().IsReadOnly {
-			return 1
-		}
-		return 0
-	})
-
-	ms.NewGauge(`vl_active_merges{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryActiveMerges)
-	})
-	ms.NewGauge(`vl_merges_total{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryMergesTotal)
-	})
-	ms.NewGauge(`vl_active_merges{type="file"}`, func() float64 {
-		return float64(m().FileActiveMerges)
-	})
-	ms.NewGauge(`vl_merges_total{type="file"}`, func() float64 {
-		return float64(m().FileMergesTotal)
-	})
-
-	ms.NewGauge(`vl_storage_rows{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryRowsCount)
-	})
-	ms.NewGauge(`vl_storage_rows{type="file"}`, func() float64 {
-		return float64(m().FileRowsCount)
-	})
-	ms.NewGauge(`vl_storage_parts{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryParts)
-	})
-	ms.NewGauge(`vl_storage_parts{type="file"}`, func() float64 {
-		return float64(m().FileParts)
-	})
-	ms.NewGauge(`vl_storage_blocks{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryBlocks)
-	})
-	ms.NewGauge(`vl_storage_blocks{type="file"}`, func() float64 {
-		return float64(m().FileBlocks)
-	})
-
-	ms.NewGauge(`vl_partitions`, func() float64 {
-		return float64(m().PartitionsCount)
-	})
-	ms.NewGauge(`vl_streams_created_total`, func() float64 {
-		return float64(m().StreamsCreatedTotal)
-	})
-
-	ms.NewGauge(`vl_indexdb_rows`, func() float64 {
-		return float64(m().IndexdbItemsCount)
-	})
-	ms.NewGauge(`vl_indexdb_parts`, func() float64 {
-		return float64(m().IndexdbPartsCount)
-	})
-	ms.NewGauge(`vl_indexdb_blocks`, func() float64 {
-		return float64(m().IndexdbBlocksCount)
-	})
-
-	ms.NewGauge(`vl_data_size_bytes{type="indexdb"}`, func() float64 {
-		return float64(m().IndexdbSizeBytes)
-	})
-	ms.NewGauge(`vl_data_size_bytes{type="storage"}`, func() float64 {
-		dm := m()
-		return float64(dm.CompressedInmemorySize + dm.CompressedFileSize)
-	})
-
-	ms.NewGauge(`vl_compressed_data_size_bytes{type="inmemory"}`, func() float64 {
-		return float64(m().CompressedInmemorySize)
-	})
-	ms.NewGauge(`vl_compressed_data_size_bytes{type="file"}`, func() float64 {
-		return float64(m().CompressedFileSize)
-	})
-	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="inmemory"}`, func() float64 {
-		return float64(m().UncompressedInmemorySize)
-	})
-	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="file"}`, func() float64 {
-		return float64(m().UncompressedFileSize)
-	})
-
-	ms.NewGauge(`vl_rows_dropped_total{reason="too_big_timestamp"}`, func() float64 {
-		return float64(m().RowsDroppedTooBigTimestamp)
-	})
-	ms.NewGauge(`vl_rows_dropped_total{reason="too_small_timestamp"}`, func() float64 {
-		return float64(m().RowsDroppedTooSmallTimestamp)
-	})
-
-	return ms
-}

app/vmagent/Makefile

@@ -85,9 +85,6 @@ vmagent-linux-arm64:
 vmagent-linux-ppc64le:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmagent-linux-s390x:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
 vmagent-linux-386:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmagent/csvimport/request_handler.go

@@ -9,8 +9,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -26,8 +26,10 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []parser.Row) error {
-		return insertRows(at, rows, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(at, rows, extraLabels)
+		})
 	})
 }
 
@@ -65,9 +67,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/datadog/request_handler.go

@@ -8,9 +8,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -28,13 +28,15 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(series []datadog.Series) error {
-		return insertRows(at, series, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		ce := req.Header.Get("Content-Encoding")
+		return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+			return insertRows(at, series, extraLabels)
+		})
 	})
 }
 
-func insertRows(at *auth.Token, series []datadog.Series, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -50,20 +52,12 @@ func insertRows(at *auth.Token, series []datadog.Series, extraLabels []prompbmar
 			Name:  "__name__",
 			Value: ss.Metric,
 		})
-		if ss.Host != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "host",
-				Value: ss.Host,
-			})
-		}
-		if ss.Device != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "device",
-				Value: ss.Device,
-			})
-		}
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "host",
+			Value: ss.Host,
+		})
 		for _, tag := range ss.Tags {
-			name, value := datadog.SplitTag(tag)
+			name, value := parser.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
@@ -88,9 +82,7 @@ func insertRows(at *auth.Token, series []datadog.Series, extraLabels []prompbmar
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/graphite/request_handler.go

@@ -5,10 +5,9 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -21,21 +20,12 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
-		return insertRows(nil, rows)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
 	})
 }
 
-// InsertHandlerForReader processes remote write for graphite plaintext protocol.
-//
-// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
-func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, isGzipped, func(rows []parser.Row) error {
-		return insertRows(at, rows)
-	})
-}
-
-func insertRows(at *auth.Token, rows []parser.Row) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -68,9 +58,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -15,14 +15,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metric name if InfluxDB line contains only a single field")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
 )
@@ -36,9 +36,11 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
-		return insertRows(at, db, rows, nil)
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+			return insertRows(nil, db, rows, nil)
+		})
 	})
 }
 
@@ -50,13 +52,15 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	q := req.URL.Query()
-	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-	db := q.Get("db")
-	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
-		return insertRows(at, db, rows, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+			return insertRows(at, db, rows, extraLabels)
+		})
 	})
 }
 
@@ -130,9 +134,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	if !remotewrite.TryPush(at, &ctx.ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.PushWithAuthToken(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/main.go

@@ -16,8 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
@@ -39,38 +37,24 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
-		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
-		"See also -influxListenAddr.useProxyProtocol")
-	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. "+
-		"See also -graphiteListenAddr.useProxyProtocol")
-	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
+		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
-		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
-	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
-		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
-	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
-		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
-		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
 
@@ -95,26 +79,22 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
 		return
 	}
 	if *dryRun {
-		if err := promscrape.CheckConfig(); err != nil {
-			logger.Fatalf("error when checking -promscrape.config: %s", err)
-		}
 		if err := remotewrite.CheckRelabelConfigs(); err != nil {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
-		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("all the configs are ok; exiting with 0 status code")
+		logger.Infof("all the configs are ok; exitting with 0 status code")
 		return
 	}
 
@@ -122,27 +102,26 @@ func main() {
 	startTime := time.Now()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
+	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, false)
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			return influx.InsertHandlerForReader(r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
-		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
 	}
 
-	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
+	promscrape.Init(remotewrite.Push)
 
 	if len(*httpListenAddr) > 0 {
-		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+		go httpserver.Serve(*httpListenAddr, requestHandler)
 	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
@@ -178,43 +157,9 @@ func main() {
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
 }
 
-func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
-	if !remotewrite.MultitenancyEnabled() {
-		return func(req *http.Request) error {
-			path := strings.Replace(req.URL.Path, "//", "/", -1)
-			if path != "/api/put" {
-				return fmt.Errorf("unsupported path requested: %q; expecting '/api/put'", path)
-			}
-			return opentsdbhttp.InsertHandler(nil, req)
-		}
-	}
-	return func(req *http.Request) error {
-		path := strings.Replace(req.URL.Path, "//", "/", -1)
-		at, err := getAuthTokenFromPath(path)
-		if err != nil {
-			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
-		}
-		return opentsdbhttp.InsertHandler(at, req)
-	}
-}
-
-func getAuthTokenFromPath(path string) (*auth.Token, error) {
-	p, err := httpserver.ParsePath(path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse multitenant path: %w", err)
-	}
-	if p.Prefix != "insert" {
-		return nil, fmt.Errorf(`unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
-	}
-	if p.Suffix != "opentsdb/api/put" {
-		return nil, fmt.Errorf("unsupported path requested: %q; expecting 'opentsdb/api/put'", p.Suffix)
-	}
-	return auth.NewTokenPossibleMultitenant(p.AuthToken)
-}
-
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
+		if r.Method != "GET" {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
@@ -224,7 +169,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
 			{"service-discovery", "labels before and after relabeling for discovered targets"},
-			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -235,33 +179,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
-	if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus") || strings.HasPrefix(path, "/api/v1/import/prometheus") {
-		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(nil, r); err != nil {
-			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		statusCode := http.StatusNoContent
-		if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus/metrics/job/") ||
-			strings.HasPrefix(path, "/api/v1/import/prometheus/metrics/job/") {
-			// Return 200 status code for pushgateway requests.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
-			statusCode = http.StatusOK
-		}
-		w.WriteHeader(statusCode)
-		return true
-	}
-	if strings.HasPrefix(path, "/datadog/") {
+	if strings.HasPrefix(path, "datadog/") {
 		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
 		path = strings.TrimSuffix(path, "/")
 	}
 	switch path {
-	case "/prometheus/api/v1/write", "/api/v1/write":
-		if common.HandleVMProtoServerHandshake(w, r) {
-			return true
-		}
+	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -270,7 +194,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import", "/api/v1/import":
+	case "/api/v1/import":
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(nil, r); err != nil {
 			vmimportErrors.Inc()
@@ -279,7 +203,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import/csv", "/api/v1/import/csv":
+	case "/api/v1/import/csv":
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(nil, r); err != nil {
 			csvimportErrors.Inc()
@@ -288,7 +212,16 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import/native", "/api/v1/import/native":
+	case "/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(nil, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/native":
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(nil, r); err != nil {
 			nativeimportErrors.Inc()
@@ -297,7 +230,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/influx/write", "/influx/api/v2/write", "/write", "/api/v2/write":
+	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
 			influxWriteErrors.Inc()
@@ -306,42 +239,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/influx/query", "/query":
+	case "/query":
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(nil, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusOK)
-		return true
-	case "/newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "/newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(nil, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	case "/datadog/api/v1/series":
 		datadogWriteRequests.Inc()
 		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
@@ -377,29 +278,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
-	case "/prometheus/targets", "/targets":
+	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
-	case "/prometheus/service-discovery", "/service-discovery":
+	case "/service-discovery":
 		promscrapeServiceDiscoveryRequests.Inc()
 		promscrape.WriteServiceDiscovery(w, r)
 		return true
-	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
-		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
-		return true
-	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
-		promscrapeTargetRelabelDebugRequests.Inc()
-		promscrape.WriteTargetRelabelDebug(w, r)
-		return true
-	case "/prometheus/api/v1/targets", "/api/v1/targets":
-		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		state := r.FormValue("state")
-		promscrape.WriteAPIV1Targets(w, state)
-		return true
-	case "/prometheus/target_response", "/target_response":
+	case "/target_response":
 		promscrapeTargetResponseRequests.Inc()
 		if err := promscrape.WriteTargetResponse(w, r); err != nil {
 			promscrapeTargetResponseErrors.Inc()
@@ -407,17 +294,27 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
-	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+	case "/config":
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeConfigRequests.Inc()
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
-	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
+	case "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -426,7 +323,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promscrape.WriteConfigData(&bb)
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
-	case "/prometheus/-/reload", "/-/reload":
+	case "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
+		return true
+	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
@@ -468,16 +371,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
 	}
-	if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus") {
-		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(at, r); err != nil {
-			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	}
 	if strings.HasPrefix(p.Suffix, "datadog/") {
 		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
@@ -511,6 +404,15 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "prometheus/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(at, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "prometheus/api/v1/import/native":
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(at, r); err != nil {
@@ -533,38 +435,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "opentelemetry/api/v1/push":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(at, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusOK)
-		return true
-	case "newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(at, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	case "datadog/api/v1/series":
 		datadogWriteRequests.Inc()
 		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
@@ -634,22 +504,9 @@ var (
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
-
-	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-
-	newrelicInventoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/inventory/deltas", protocol="newrelic"}`)
-	newrelicCheckRequest      = metrics.NewCounter(`vm_http_requests_total{path="/newrelic", protocol="newrelic"}`)
-
 	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
-
-	promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/metric-relabel-debug"}`)
-	promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target-relabel-debug"}`)
-
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)

app/vmagent/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
+RUN apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmagent/native/request_handler.go

@@ -10,8 +10,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -30,12 +31,14 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
-		return insertRows(at, block, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+			return insertRows(at, block, extraLabels)
+		})
 	})
 }
 
-func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -84,8 +87,6 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/newrelic/request_handler.go

@@ -1,88 +0,0 @@
-package newrelic
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="newrelic"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="newrelic"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="newrelic"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	isGzip := ce == "gzip"
-	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
-		return insertRows(at, rows, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	samplesCount := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		tags := r.Tags
-		srcSamples := r.Samples
-		for j := range srcSamples {
-			s := &srcSamples[j]
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: bytesutil.ToUnsafeString(s.Name),
-			})
-			for k := range tags {
-				t := &tags[k]
-				labels = append(labels, prompbmarshal.Label{
-					Name:  bytesutil.ToUnsafeString(t.Key),
-					Value: bytesutil.ToUnsafeString(t.Value),
-				})
-			}
-			samples = append(samples, prompbmarshal.Sample{
-				Value:     s.Value,
-				Timestamp: r.Timestamp,
-			})
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[len(samples)-1:],
-			})
-			labels = append(labels, extraLabels...)
-		}
-		samplesCount += len(srcSamples)
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(samplesCount)
-	}
-	rowsPerInsert.Update(float64(samplesCount))
-	return nil
-}

app/vmagent/opentelemetry/request_handler.go

@@ -1,71 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
-)
-
-// InsertHandler processes opentelemetry metrics.
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	if req.Header.Get("Content-Type") == "application/json" {
-		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-	}
-	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
-		return insertRows(at, tss, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range tss {
-		ts := &tss[i]
-		rowsTotal += len(ts.Samples)
-		labelsLen := len(labels)
-		labels = append(labels, ts.Labels...)
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		samples = append(samples, ts.Samples...)
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/opentsdb/request_handler.go

@@ -7,7 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,7 +20,9 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, insertRows)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
 }
 
 func insertRows(rows []parser.Row) error {
@@ -56,9 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(nil, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/opentsdbhttp/request_handler.go

@@ -5,11 +5,10 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,17 +19,19 @@ var (
 
 // InsertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
-func InsertHandler(at *auth.Token, req *http.Request) error {
+func InsertHandler(req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []parser.Row) error {
-		return insertRows(at, rows, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
+		})
 	})
 }
 
-func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -64,9 +65,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -1,17 +1,17 @@
 package prometheusimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -31,11 +31,20 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
-		return insertRows(at, rows, extraLabels)
-	}, func(s string) {
-		httpserver.LogError(req, s)
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+			return insertRows(at, rows, extraLabels)
+		}, nil)
+	})
+}
+
+// InsertHandlerForReader processes metrics from given reader with optional gzip format
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		}, nil)
 	})
 }
 
@@ -73,9 +82,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/prometheusimport/request_handler_test.go

@@ -1,60 +0,0 @@
-package prometheusimport
-
-import (
-	"bytes"
-	"flag"
-	"log"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-)
-
-var (
-	srv        *httptest.Server
-	testOutput *bytes.Buffer
-)
-
-func TestInsertHandler(t *testing.T) {
-	setUp()
-	defer tearDown()
-	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
-go_memstats_alloc_bytes_total 1`))
-	if err := InsertHandler(nil, req); err != nil {
-		t.Errorf("unxepected error %s", err)
-	}
-	expectedMsg := "cannot unmarshal Prometheus line"
-	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
-	}
-}
-
-func setUp() {
-	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(204)
-	}))
-	flag.Parse()
-	remoteWriteFlag := "remoteWrite.url"
-	if err := flag.Lookup(remoteWriteFlag).Value.Set(srv.URL); err != nil {
-		log.Fatalf("unable to set %q with value %q, err: %v", remoteWriteFlag, srv.URL, err)
-	}
-	logger.Init()
-	common.StartUnmarshalWorkers()
-	remotewrite.Init()
-	testOutput = &bytes.Buffer{}
-	logger.SetOutputForTests(testOutput)
-}
-
-func tearDown() {
-	common.StopUnmarshalWorkers()
-	srv.Close()
-	logger.ResetOutputForTest()
-	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()
-	fs.MustRemoveAll(tmpDataDir)
-
-}

app/vmagent/promremotewrite/request_handler.go

@@ -1,6 +1,7 @@
 package promremotewrite
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -10,8 +11,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -27,9 +29,19 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
-		return insertRows(at, tss, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, extraLabels)
+		})
+	})
+}
+
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
+			return insertRows(at, tss, nil)
+		})
 	})
 }
 
@@ -69,9 +81,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/remotewrite/client.go

@@ -2,9 +2,8 @@ package remotewrite
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -16,72 +15,60 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
-	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
-
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
-		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
+	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
+		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArray("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
-	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
-	tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
-	tlsCAFile  = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
-		"By default, system CA is used")
-	tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
-		"By default, the server name from -remoteWrite.url is used")
+	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
+	tlsCAFile  = flagutil.NewArray("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
+		"By default system CA is used")
+	tlsServerName = flagutil.NewArray("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used")
 
-	headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+	headers = flagutil.NewArray("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
 		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
 		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
-	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
-	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
-	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
+	basicAuthUsername     = flagutil.NewArray("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	basicAuthPassword     = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
+	basicAuthPasswordFile = flagutil.NewArray("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
 		"The file is re-read every second")
-	bearerToken     = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
-	bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
+	bearerToken     = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
+	bearerTokenFile = flagutil.NewArray("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
 		"The token is re-read from the file every second")
 
-	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
-	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
-	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
-	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
-	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+	oauth2ClientID         = flagutil.NewArray("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecret     = flagutil.NewArray("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecretFile = flagutil.NewArray("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
+	oauth2TokenURL         = flagutil.NewArray("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes           = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
 
 	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
 		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
-	awsEC2Endpoint = flagutil.NewArrayString("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsSTSEndpoint = flagutil.NewArrayString("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsRegion      = flagutil.NewArrayString("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsRoleARN     = flagutil.NewArrayString("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsAccessKey   = flagutil.NewArrayString("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsService     = flagutil.NewArrayString("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+	awsRegion    = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRoleARN   = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsService   = flagutil.NewArray("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
 		"Defaults to \"aps\"")
-	awsSecretKey = flagutil.NewArrayString("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
 )
 
 type client struct {
 	sanitizedURL   string
 	remoteWriteURL string
-
-	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
-	useVMProto bool
-
-	fq *persistentqueue.FastQueue
-	hc *http.Client
+	fq             *persistentqueue.FastQueue
+	hc             *http.Client
 
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
@@ -106,15 +93,12 @@ type client struct {
 func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
-		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
-	}
-	tlsCfg, err := authCfg.NewTLSConfig()
-	if err != nil {
-		logger.Fatalf("cannot initialize tls config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
+	tlsCfg := authCfg.NewTLSConfig()
 	awsCfg, err := getAWSAPIConfig(argIdx)
 	if err != nil {
-		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
 		DialContext:         statDial,
@@ -136,44 +120,24 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
-	hc := &http.Client{
-		Transport: tr,
-		Timeout:   sendTimeout.GetOptionalArg(argIdx),
-	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authCfg:        authCfg,
 		awsCfg:         awsCfg,
 		fq:             fq,
-		hc:             hc,
-		stopCh:         make(chan struct{}),
+		hc: &http.Client{
+			Transport: tr,
+			Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
+		},
+		stopCh: make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
-
-	useVMProto := forceVMProto.GetOptionalArg(argIdx)
-	usePromProto := forcePromProto.GetOptionalArg(argIdx)
-	if useVMProto && usePromProto {
-		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
-	}
-	if !useVMProto && !usePromProto {
-		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
-		doRequest := func(url string) (*http.Response, error) {
-			return c.doRequest(url, nil)
-		}
-		useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
-		if !useVMProto {
-			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
-				"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol", sanitizedURL)
-		}
-	}
-	c.useVMProto = useVMProto
-
 	return c
 }
 
 func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
-	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
+	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
@@ -182,7 +146,7 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
 	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(rateLimit.GetOptionalArg(argIdx))
+		return float64(rateLimit.GetOptionalArgOrDefault(argIdx, 0))
 	})
 	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
 	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
@@ -190,9 +154,6 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
-	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(*queues)
-	})
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -261,7 +222,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	}
 	authCfg, err := opts.NewConfig()
 	if err != nil {
-		return nil, fmt.Errorf("cannot populate auth config for remoteWrite idx: %d, err: %w", argIdx, err)
+		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
 	return authCfg, nil
 }
@@ -270,14 +231,12 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
 	if !awsUseSigv4.GetOptionalArg(argIdx) {
 		return nil, nil
 	}
-	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
-	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
 	region := awsRegion.GetOptionalArg(argIdx)
 	roleARN := awsRoleARN.GetOptionalArg(argIdx)
 	accessKey := awsAccessKey.GetOptionalArg(argIdx)
 	secretKey := awsSecretKey.GetOptionalArg(argIdx)
 	service := awsService.GetOptionalArg(argIdx)
-	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
+	cfg, err := awsapi.NewConfig(region, roleARN, accessKey, secretKey, service)
 	if err != nil {
 		return nil, err
 	}
@@ -305,7 +264,7 @@ func (c *client) runWorker() {
 				continue
 			}
 			// Return unsent block to the queue.
-			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			c.fq.MustWriteBlock(block)
 			return
 		case <-c.stopCh:
 			// c must be stopped. Wait for a while in the hope the block will be sent.
@@ -314,85 +273,49 @@ func (c *client) runWorker() {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
-					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+					c.fq.MustWriteBlock(block)
 				}
 			case <-time.After(graceDuration):
 				// Return unsent block to the queue.
-				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+				c.fq.MustWriteBlock(block)
 			}
 			return
 		}
 	}
 }
 
-func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
-	req, err := c.newRequest(url, body)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := c.hc.Do(req)
-	if err == nil {
-		return resp, nil
-	}
-	if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
-		return nil, err
-	}
-	// It is likely connection became stale or timed out during the first request.
-	// Make another attempt in hope request will succeed.
-	// If not, the error should be handled by the caller as usual.
-	// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
-	req, err = c.newRequest(url, body)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	resp, err = c.hc.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	return resp, nil
-}
-
-func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
-	reqBody := bytes.NewBuffer(body)
-	req, err := http.NewRequest(http.MethodPost, url, reqBody)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
-	}
-	err = c.authCfg.SetHeaders(req, true)
-	if err != nil {
-		return nil, err
-	}
-	h := req.Header
-	h.Set("User-Agent", "vmagent")
-	h.Set("Content-Type", "application/x-protobuf")
-	if c.useVMProto {
-		h.Set("Content-Encoding", "zstd")
-		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
-	} else {
-		h.Set("Content-Encoding", "snappy")
-		h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	}
-	if c.awsCfg != nil {
-		sigv4Hash := awsapi.HashHex(body)
-		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
-			return nil, fmt.Errorf("cannot sign remoteWrite request with AWS sigv4: %w", err)
-		}
-	}
-	return req, nil
-}
-
-// sendBlockHTTP sends the given block to c.remoteWriteURL.
-//
-// The function returns false only if c.stopCh is closed.
+// sendBlockHTTP returns false only if c.stopCh is closed.
 // Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.register(len(block), c.stopCh)
 	retryDuration := time.Second
 	retriesCount := 0
+	c.bytesSent.Add(len(block))
+	c.blocksSent.Inc()
+	sigv4Hash := ""
+	if c.awsCfg != nil {
+		sigv4Hash = awsapi.HashHex(block)
+	}
 
 again:
+	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
+	if err != nil {
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
+	}
+	c.authCfg.SetHeaders(req, true)
+	h := req.Header
+	h.Set("User-Agent", "vmagent")
+	h.Set("Content-Type", "application/x-protobuf")
+	h.Set("Content-Encoding", "snappy")
+	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.awsCfg != nil {
+		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
+			// there is no need in retry, request will be rejected by client.Do and retried by code below
+			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
+		}
+	}
 	startTime := time.Now()
-	resp, err := c.doRequest(c.remoteWriteURL, block)
+	resp, err := c.hc.Do(req)
 	c.requestDuration.UpdateDuration(startTime)
 	if err != nil {
 		c.errorsCount.Inc()
@@ -417,13 +340,11 @@ again:
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
 		c.requestsOKCount.Inc()
-		c.bytesSent.Add(len(block))
-		c.blocksSent.Inc()
 		return true
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
 	if statusCode == 409 || statusCode == 400 {
-		body, err := io.ReadAll(resp.Body)
+		body, err := ioutil.ReadAll(resp.Body)
 		_ = resp.Body.Close()
 		if err != nil {
 			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
@@ -447,7 +368,7 @@ again:
 	if retryDuration > time.Minute {
 		retryDuration = time.Minute
 	}
-	body, err := io.ReadAll(resp.Body)
+	body, err := ioutil.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
 		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)

app/vmagent/remotewrite/pendingseries.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -24,9 +23,6 @@ var (
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
 	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
-	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
-		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
-		"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
 )
 
 type pendingSeries struct {
@@ -37,10 +33,9 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(pushBlock func(block []byte), significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
-	ps.wr.fq = fq
-	ps.wr.isVMRemoteWrite = isVMRemoteWrite
+	ps.wr.pushBlock = pushBlock
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
@@ -57,11 +52,10 @@ func (ps *pendingSeries) MustStop() {
 	ps.periodicFlusherWG.Wait()
 }
 
-func (ps *pendingSeries) TryPush(tss []prompbmarshal.TimeSeries) bool {
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
 	ps.mu.Lock()
-	ok := ps.wr.tryPush(tss)
+	ps.wr.push(tss)
 	ps.mu.Unlock()
-	return ok
 }
 
 func (ps *pendingSeries) periodicFlusher() {
@@ -71,20 +65,18 @@ func (ps *pendingSeries) periodicFlusher() {
 	}
 	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
-	for {
+	mustStop := false
+	for !mustStop {
 		select {
 		case <-ps.stopCh:
-			ps.mu.Lock()
-			ps.wr.mustFlushOnStop()
-			ps.mu.Unlock()
-			return
+			mustStop = true
 		case <-ticker.C:
 			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
 				continue
 			}
 		}
 		ps.mu.Lock()
-		_ = ps.wr.tryFlush()
+		ps.wr.flush()
 		ps.mu.Unlock()
 	}
 }
@@ -93,16 +85,13 @@ type writeRequest struct {
 	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
 	lastFlushTime uint64
 
-	// The queue to send blocks to.
-	fq *persistentqueue.FastQueue
+	// pushBlock is called when whe write request is ready to be sent.
+	pushBlock func(block []byte)
 
-	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	isVMRemoteWrite bool
-
-	// How many significant figures must be left before sending the writeRequest to fq.
+	// How many significant figures must be left before sending the writeRequest to pushBlock.
 	significantFigures int
 
-	// How many decimal digits after point must be left before sending the writeRequest to fq.
+	// How many decimal digits after point must be left before sending the writeRequest to pushBlock.
 	roundDigits int
 
 	wr prompbmarshal.WriteRequest
@@ -115,7 +104,7 @@ type writeRequest struct {
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
+	// Do not reset pushBlock, significantFigures and roundDigits, since they are re-used.
 
 	wr.wr.Timeseries = nil
 
@@ -133,40 +122,23 @@ func (wr *writeRequest) reset() {
 	wr.buf = wr.buf[:0]
 }
 
-// mustFlushOnStop force pushes wr data into wr.fq
-//
-// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
-func (wr *writeRequest) mustFlushOnStop() {
-	wr.wr.Timeseries = wr.tss
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
-		logger.Panicf("BUG: final flush must always return true")
-	}
-	wr.reset()
-}
-
-func (wr *writeRequest) mustWriteBlock(block []byte) bool {
-	wr.fq.MustWriteBlockIgnoreDisabledPQ(block)
-	return true
-}
-
-func (wr *writeRequest) tryFlush() bool {
+func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
+	wr.adjustSampleValues()
 	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
-		return false
-	}
+	pushWriteRequest(&wr.wr, wr.pushBlock)
 	wr.reset()
-	return true
 }
 
-func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roundDigits int) {
-	if n := significantFigures; n > 0 {
+func (wr *writeRequest) adjustSampleValues() {
+	samples := wr.samples
+	if n := wr.significantFigures; n > 0 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToSignificantFigures(s.Value, n)
 		}
 	}
-	if n := roundDigits; n < 100 {
+	if n := wr.roundDigits; n < 100 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToDecimalDigits(s.Value, n)
@@ -174,27 +146,21 @@ func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roun
 	}
 }
 
-func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
 		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
-			if !wr.tryFlush() {
-				return false
-			}
+			wr.flush()
 			tssDst = wr.tss
 		}
-		tsSrc := &src[i]
-		adjustSampleValues(tsSrc.Samples, wr.significantFigures, wr.roundDigits)
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
-		wr.copyTimeSeries(&tssDst[len(tssDst)-1], tsSrc)
 	}
-
 	wr.tss = tssDst
-	return true
 }
 
 func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
@@ -222,29 +188,23 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
-func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
-		return true
+		return
 	}
 	bb := writeRequestBufPool.Get()
 	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
-	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
+	if len(bb.B) <= maxUnpackedBlockSize.N {
 		zb := snappyBufPool.Get()
-		if isVMRemoteWrite {
-			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
-		} else {
-			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
-		}
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		writeRequestBufPool.Put(bb)
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			if !tryPushBlock(zb.B) {
-				return false
-			}
+			pushBlock(zb.B)
 			blockSizeRows.Update(float64(len(wr.Timeseries)))
 			blockSizeBytes.Update(float64(len(zb.B)))
 			snappyBufPool.Put(zb)
-			return true
+			return
 		}
 		snappyBufPool.Put(zb)
 	} else {
@@ -257,36 +217,23 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 		samples := wr.Timeseries[0].Samples
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
-			return true
+			return
 		}
 		n := len(samples) / 2
 		wr.Timeseries[0].Samples = samples[:n]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			return false
-		}
+		pushWriteRequest(wr, pushBlock)
 		wr.Timeseries[0].Samples = samples[n:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			return false
-		}
+		pushWriteRequest(wr, pushBlock)
 		wr.Timeseries[0].Samples = samples
-		return true
+		return
 	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
+	pushWriteRequest(wr, pushBlock)
 	wr.Timeseries = timeseries[n:]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
+	pushWriteRequest(wr, pushBlock)
 	wr.Timeseries = timeseries
-	return true
 }
 
 var (

app/vmagent/remotewrite/pendingseries_test.go

@@ -1,73 +0,0 @@
-package remotewrite
-
-import (
-	"fmt"
-	"math"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-)
-
-func TestPushWriteRequest(t *testing.T) {
-	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
-	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
-	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
-	for i, rowsCount := range rowsCounts {
-		expectedBlockLenProm := expectedBlockLensProm[i]
-		expectedBlockLenVM := expectedBlockLensVM[i]
-		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
-			testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
-		})
-	}
-}
-
-func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
-	f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
-		t.Helper()
-		wr := newTestWriteRequest(rowsCount, 20)
-		pushBlockLen := 0
-		pushBlock := func(block []byte) bool {
-			if pushBlockLen > 0 {
-				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
-			}
-			pushBlockLen = len(block)
-			return true
-		}
-		if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
-			t.Fatalf("cannot push data to to remote storage")
-		}
-		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
-			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
-				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
-		}
-	}
-
-	// Check Prometheus remote write
-	f(false, expectedBlockLenProm, 0)
-
-	// Check VictoriaMetrics remote write
-	f(true, expectedBlockLenVM, 15)
-}
-
-func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {
-	var wr prompbmarshal.WriteRequest
-	for i := 0; i < seriesCount; i++ {
-		var labels []prompbmarshal.Label
-		for j := 0; j < labelsCount; j++ {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  fmt.Sprintf("label_%d_%d", i, j),
-				Value: fmt.Sprintf("value_%d_%d", i, j),
-			})
-		}
-		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
-			Labels: labels,
-			Samples: []prompbmarshal.Sample{
-				{
-					Value:     float64(i),
-					Timestamp: 1000 * int64(i),
-				},
-			},
-		})
-	}
-	return &wr
-}

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -1,36 +0,0 @@
-package remotewrite
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/golang/snappy"
-	"github.com/klauspost/compress/s2"
-)
-
-func BenchmarkCompressWriteRequestSnappy(b *testing.B) {
-	b.Run("snappy", func(b *testing.B) {
-		benchmarkCompressWriteRequest(b, snappy.Encode)
-	})
-	b.Run("s2", func(b *testing.B) {
-		benchmarkCompressWriteRequest(b, s2.EncodeSnappy)
-	})
-}
-
-func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []byte) []byte) {
-	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
-		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
-			wr := newTestWriteRequest(rowsCount, 10)
-			data := prompbmarshal.MarshalWriteRequest(nil, wr)
-			b.ReportAllocs()
-			b.SetBytes(int64(rowsCount))
-			b.RunParallel(func(pb *testing.PB) {
-				var zb []byte
-				for pb.Next() {
-					zb = compressFunc(zb[:cap(zb)], data)
-				}
-			})
-		})
-	}
-}

app/vmagent/remotewrite/relabel.go

@@ -3,7 +3,6 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -14,19 +13,18 @@ import (
 )
 
 var (
-	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
-		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
-		"The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
-	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
-		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
-
-	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
-		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
-		"See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
+		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
+		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
+	relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+
+		"If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs")
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
+		"The path can point either to local file or to http url")
+	relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+
+		"If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+
+		"This is useful for debugging the relabeling configs")
 )
 
 var labelsGlobal []prompbmarshal.Label
@@ -40,7 +38,7 @@ func CheckRelabelConfigs() error {
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
-		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal, *relabelDebugGlobal)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
 		}
@@ -56,7 +54,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 			// Skip empty relabel config.
 			continue
 		}
-		prc, err := promrelabel.LoadRelabelConfigs(path)
+		prc, err := promrelabel.LoadRelabelConfigs(path, relabelDebug.GetOptionalArg(i))
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
 		}
@@ -88,27 +86,32 @@ func initLabelsGlobal() {
 	}
 }
 
-func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
-	if pcs.Len() == 0 && !*usePromCompatibleNaming {
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && pcs.Len() == 0 {
 		// Nothing to change.
 		return tss
 	}
-	rctx.reset()
 	tssDst := tss[:0]
 	labels := rctx.labels[:0]
 	for i := range tss {
 		ts := &tss[i]
 		labelsLen := len(labels)
 		labels = append(labels, ts.Labels...)
-		labels = pcs.Apply(labels, labelsLen)
-		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		labels = pcs.Apply(labels, labelsLen, true)
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue
 		}
-		if *usePromCompatibleNaming {
-			fixPromCompatibleNaming(labels[labelsLen:])
-		}
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: ts.Samples,
@@ -118,58 +121,6 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *pro
 	return tssDst
 }
 
-func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) {
-	if len(extraLabels) == 0 {
-		return
-	}
-	rctx.reset()
-	labels := rctx.labels[:0]
-	for i := range tss {
-		ts := &tss[i]
-		labelsLen := len(labels)
-		labels = append(labels, ts.Labels...)
-		for j := range extraLabels {
-			extraLabel := extraLabels[j]
-			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
-			if tmp != nil {
-				tmp.Value = extraLabel.Value
-			} else {
-				labels = append(labels, extraLabel)
-			}
-		}
-		ts.Labels = labels[labelsLen:]
-	}
-	rctx.labels = labels
-}
-
-func (rctx *relabelCtx) tenantToLabels(tss []prompbmarshal.TimeSeries, accountID, projectID uint32) {
-	rctx.reset()
-	accountIDStr := strconv.FormatUint(uint64(accountID), 10)
-	projectIDStr := strconv.FormatUint(uint64(projectID), 10)
-	labels := rctx.labels[:0]
-	for i := range tss {
-		ts := &tss[i]
-		labelsLen := len(labels)
-		for _, label := range ts.Labels {
-			labelName := label.Name
-			if labelName == "vm_account_id" || labelName == "vm_project_id" {
-				continue
-			}
-			labels = append(labels, label)
-		}
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "vm_account_id",
-			Value: accountIDStr,
-		})
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "vm_project_id",
-			Value: projectIDStr,
-		})
-		ts.Labels = labels[labelsLen:]
-	}
-	rctx.labels = labels
-}
-
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
 	labels []prompbmarshal.Label
@@ -191,18 +142,6 @@ func getRelabelCtx() *relabelCtx {
 }
 
 func putRelabelCtx(rctx *relabelCtx) {
-	rctx.reset()
+	rctx.labels = rctx.labels[:0]
 	relabelCtxPool.Put(rctx)
 }
-
-func fixPromCompatibleNaming(labels []prompbmarshal.Label) {
-	// Replace unsupported Prometheus chars in label names and metric names with underscores.
-	for i := range labels {
-		label := &labels[i]
-		if label.Name == "__name__" {
-			label.Value = promrelabel.SanitizeMetricName(label.Value)
-		} else {
-			label.Name = promrelabel.SanitizeLabelName(label.Name)
-		}
-	}
-}

app/vmagent/remotewrite/relabel_test.go

@@ -1,69 +0,0 @@
-package remotewrite
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-func TestApplyRelabeling(t *testing.T) {
-	f := func(pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
-		rctx := &relabelCtx{}
-		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		gotTss := rctx.applyRelabeling(tss, pcs)
-		if !reflect.DeepEqual(gotTss, expTss) {
-			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
-		}
-	}
-
-	f(nil, "up", "up")
-
-	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
-- target_label: "foo"
-  replacement: "aaa"
-- action: labeldrop
-  regex: "env.*"
-`))
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	f(pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
-
-	oldVal := *usePromCompatibleNaming
-	*usePromCompatibleNaming = true
-	f(nil, `foo.bar`, `foo_bar`)
-	*usePromCompatibleNaming = oldVal
-}
-
-func TestAppendExtraLabels(t *testing.T) {
-	f := func(extraLabels []prompbmarshal.Label, sTss, sExpTss string) {
-		t.Helper()
-		rctx := &relabelCtx{}
-		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		rctx.appendExtraLabels(tss, extraLabels)
-		if !reflect.DeepEqual(tss, expTss) {
-			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, tss)
-		}
-	}
-
-	f(nil, "up", "up")
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, "up", `up{foo="bar"}`)
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, `up{foo="baz"}`, `up{foo="bar"}`)
-	f([]prompbmarshal.Label{{Name: "baz", Value: "qux"}}, `up{foo="baz"}`, `up{foo="baz",baz="qux"}`)
-
-	oldVal := *usePromCompatibleNaming
-	*usePromCompatibleNaming = true
-	f([]prompbmarshal.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo.bar="baz"}`)
-	*usePromCompatibleNaming = oldVal
-}
-
-func parseSeries(data string) []prompbmarshal.TimeSeries {
-	var tss []prompbmarshal.TimeSeries
-	tss = append(tss, prompbmarshal.TimeSeries{
-		Labels: promutils.MustNewLabelsFromString(data).GetLabels(),
-	})
-	return tss
-}

app/vmagent/remotewrite/remotewrite.go

@@ -3,73 +3,51 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"net/url"
-	"path/filepath"
 	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
 )
 
 var (
-	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
-		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
-		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
-		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set")
-	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
+	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
+		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
+	remoteWriteMultitenantURLs = flagutil.NewArray("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
-		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. "+
-		"This flag is deprecated in favor of -enableMultitenantHandlers . See https://docs.victoriametrics.com/vmagent.html#multitenancy")
-	enableMultitenantHandlers = flag.Bool("enableMultitenantHandlers", false, "Whether to process incoming data via multitenant insert handlers according to "+
-		"https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format . By default incoming data is processed via single-node insert handlers "+
-		"according to https://docs.victoriametrics.com/#how-to-import-time-series-data ."+
-		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details")
-	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
-		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages")
-	shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
-		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
-		"even distribution of series over the specified -remoteWrite.url systems")
-	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
-		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
-	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
-		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
+		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
+	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
+		"See also -remoteWrite.maxDiskUsagePerURL")
 	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
-	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+	maxPendingBytesPerURL = flagutil.NewBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
-		"Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. "+
+		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+
 		"Disk usage is unlimited if the value is set to 0")
-	significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", 0, "The number of significant figures to leave in metric values before writing them "+
+	significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", "The number of significant figures to leave in metric values before writing them "+
 		"to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
 		"This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits")
-	roundDigits = flagutil.NewArrayInt("remoteWrite.roundDigits", 100, "Round metric values to this number of decimal digits after the point before "+
-		"writing them to remote storage. "+
+	roundDigits = flagutil.NewArrayInt("remoteWrite.roundDigits", "Round metric values to this number of decimal digits after the point before writing them to remote storage. "+
 		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
-		"By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
+		"By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
 		"This option may be used for improving data compression for the stored metrics")
 	sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
 		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
@@ -79,23 +57,6 @@ var (
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
 		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
-
-	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
-		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
-		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
-	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation.html")
-	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before being aggregated. "+
-		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
-	disableOnDiskQueue = flag.Bool("remoteWrite.disableOnDiskQueue", false, "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence ."+
-		"See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence")
 )
 
 var (
@@ -108,30 +69,20 @@ var (
 
 	// Data without tenant id is written to defaultAuthToken if -remoteWrite.multitenantURL is specified.
 	defaultAuthToken = &auth.Token{}
-
-	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
-	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
-		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
-			"or remove -remoteWrite.disableOnDiskQueue from vmagent command-line flags, so it could save pending data to -remoteWrite.tmpDataPath; " +
-			"see https://docs.victoriametrics.com/vmagent.html#disabling-on-disk-persistence"),
-		StatusCode: http.StatusTooManyRequests,
-	}
 )
 
-// MultitenancyEnabled returns true if -enableMultitenantHandlers or -remoteWrite.multitenantURL is specified.
+// MultitenancyEnabled returns true if -remoteWrite.multitenantURL is specified.
 func MultitenancyEnabled() bool {
-	return *enableMultitenantHandlers || len(*remoteWriteMultitenantURLs) > 0
+	return len(*remoteWriteMultitenantURLs) > 0
 }
 
 // Contains the current relabelConfigs.
-var allRelabelConfigs atomic.Pointer[relabelConfigs]
+var allRelabelConfigs atomic.Value
 
 // maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
 // since it may lead to high memory usage due to big number of buffers.
 var maxQueues = cgroup.AvailableCPUs() * 16
 
-const persistentQueueDirname = "persistent-queue"
-
 // InitSecretFlags must be called after flag.Parse and before any logging.
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
@@ -140,8 +91,6 @@ func InitSecretFlags() {
 	}
 }
 
-var shardByURLLabelsMap map[string]struct{}
-
 // Init initializes remotewrite.
 //
 // It must be called after flag.Parse().
@@ -178,13 +127,6 @@ func Init() {
 	if *queues <= 0 {
 		*queues = 1
 	}
-	if len(*shardByURLLabels) > 0 {
-		m := make(map[string]struct{}, len(*shardByURLLabels))
-		for _, label := range *shardByURLLabels {
-			m[label] = struct{}{}
-		}
-		shardByURLLabelsMap = m
-	}
 	initLabelsGlobal()
 
 	// Register SIGHUP handler for config reload before loadRelabelConfigs.
@@ -197,13 +139,10 @@ func Init() {
 		logger.Fatalf("cannot load relabel configs: %s", err)
 	}
 	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
 	if len(*remoteWriteURLs) > 0 {
 		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 	}
-	dropDanglingQueues()
 
 	// Start config reloader.
 	configReloaderWG.Add(1)
@@ -212,92 +151,21 @@ func Init() {
 		for {
 			select {
 			case <-sighupCh:
-			case <-configReloaderStopCh:
+			case <-stopCh:
 				return
 			}
-			reloadRelabelConfigs()
-			reloadStreamAggrConfigs()
+			logger.Infof("SIGHUP received; reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+			rcs, err := loadRelabelConfigs()
+			if err != nil {
+				logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+				continue
+			}
+			allRelabelConfigs.Store(rcs)
+			logger.Infof("Successfully reloaded relabel configs")
 		}
 	}()
 }
 
-func dropDanglingQueues() {
-	if *keepDanglingQueues {
-		return
-	}
-	if len(*remoteWriteMultitenantURLs) > 0 {
-		// Do not drop dangling queues for *remoteWriteMultitenantURLs, since it is impossible to determine
-		// unused queues for multitenant urls - they are created on demand when new sample for the given
-		// tenant is pushed to remote storage.
-		return
-	}
-	// Remove dangling persistent queues, if any.
-	// This is required for the case when the number of queues has been changed or URL have been changed.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
-	//
-	existingQueues := make(map[string]struct{}, len(rwctxsDefault))
-	for _, rwctx := range rwctxsDefault {
-		existingQueues[rwctx.fq.Dirname()] = struct{}{}
-	}
-
-	queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
-	files := fs.MustReadDir(queuesDir)
-	removed := 0
-	for _, f := range files {
-		dirname := f.Name()
-		if _, ok := existingQueues[dirname]; !ok {
-			logger.Infof("removing dangling queue %q", dirname)
-			fullPath := filepath.Join(queuesDir, dirname)
-			fs.MustRemoveAll(fullPath)
-			removed++
-		}
-	}
-	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsDefault))
-	}
-}
-
-func reloadRelabelConfigs() {
-	relabelConfigReloads.Inc()
-	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		relabelConfigReloadErrors.Inc()
-		relabelConfigSuccess.Set(0)
-		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
-		return
-	}
-	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
-	logger.Infof("successfully reloaded relabel configs")
-}
-
-var (
-	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
-	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
-	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
-)
-
-func reloadStreamAggrConfigs() {
-	if len(*remoteWriteMultitenantURLs) > 0 {
-		rwctxsMapLock.Lock()
-		for _, rwctxs := range rwctxsMap {
-			reinitStreamAggr(rwctxs)
-		}
-		rwctxsMapLock.Unlock()
-	} else {
-		reinitStreamAggr(rwctxsDefault)
-	}
-}
-
-func reinitStreamAggr(rwctxs []*remoteWriteCtx) {
-	for _, rwctx := range rwctxs {
-		rwctx.reinitStreamAggr()
-	}
-}
-
 func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
@@ -328,19 +196,19 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
 	return rwctxs
 }
 
-var configReloaderStopCh = make(chan struct{})
+var stopCh = make(chan struct{})
 var configReloaderWG sync.WaitGroup
 
 // Stop stops remotewrite.
 //
-// It is expected that nobody calls TryPush during and after the call to this func.
+// It is expected that nobody calls Push during and after the call to this func.
 func Stop() {
-	close(configReloaderStopCh)
+	close(stopCh)
 	configReloaderWG.Wait()
 
 	for _, rwctx := range rwctxsDefault {
@@ -348,7 +216,7 @@ func Stop() {
 	}
 	rwctxsDefault = nil
 
-	// There is no need in locking rwctxsMapLock here, since nobody should call TryPush during the Stop call.
+	// There is no need in locking rwctxsMapLock here, since nobody should call Push during the Stop call.
 	for _, rwctxs := range rwctxsMap {
 		for _, rwctx := range rwctxs {
 			rwctx.MustStop()
@@ -364,47 +232,28 @@ func Stop() {
 	}
 }
 
-// PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
+// Push sends wr to remote storage systems set via `-remoteWrite.url`.
 //
-// If at is nil, then the data is pushed to the configured -remoteWrite.url.
-// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
-//
-// PushDropSamplesOnFailure can modify wr contents.
-func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
-	_ = tryPush(at, wr, true)
+// Note that wr may be modified by Push due to relabeling and rounding.
+func Push(wr *prompbmarshal.WriteRequest) {
+	PushWithAuthToken(nil, wr)
 }
 
-// TryPush tries sending wr to the configured remote storage systems set via -remoteWrite.url and -remoteWrite.multitenantURL
+// PushWithAuthToken sends wr to remote storage systems set via `-remoteWrite.multitenantURL`.
 //
-// If at is nil, then the data is pushed to the configured -remoteWrite.url.
-// If at isn't nil, the data is pushed to the configured -remoteWrite.multitenantURL.
-//
-// TryPush can modify wr contents, so the caller must re-initialize wr before calling TryPush() after unsuccessful attempt.
-// TryPush may send partial data from wr on unsuccessful attempt, so repeated call for the same wr may send the data multiple times.
-//
-// The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
-func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, *dropSamplesOnOverload)
-}
-
-func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailure bool) bool {
-	tss := wr.Timeseries
-
-	if at == nil && MultitenancyEnabled() {
-		// Write data to default tenant if at isn't set when multitenancy is enabled.
+// Note that wr may be modified by Push due to relabeling and rounding.
+func PushWithAuthToken(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
+		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
 		at = defaultAuthToken
 	}
-
-	var tenantRctx *relabelCtx
 	var rwctxs []*remoteWriteCtx
 	if at == nil {
 		rwctxs = rwctxsDefault
-	} else if len(*remoteWriteMultitenantURLs) == 0 {
-		// Convert at to (vm_account_id, vm_project_id) labels.
-		tenantRctx = getRelabelCtx()
-		defer putRelabelCtx(tenantRctx)
-		rwctxs = rwctxsDefault
 	} else {
+		if len(*remoteWriteMultitenantURLs) == 0 {
+			logger.Panicf("BUG: remoteWriteMultitenantURLs must be non-empty for non-nil at")
+		}
 		rwctxsMapLock.Lock()
 		tenantID := tenantmetrics.TenantID{
 			AccountID: at.AccountID,
@@ -418,37 +267,18 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 		rwctxsMapLock.Unlock()
 	}
 
-	rowsCount := getRowsCount(tss)
-
-	if *disableOnDiskQueue {
-		// Quick check whether writes to configured remote storage systems are blocked.
-		// This allows saving CPU time spent on relabeling and block compression
-		// if some of remote storage systems cannot keep up with the data ingestion rate.
-		for _, rwctx := range rwctxs {
-			if rwctx.fq.IsWriteBlocked() {
-				pushFailures.Inc()
-				if dropSamplesOnFailure {
-					// Just drop samples
-					samplesDropped.Add(rowsCount)
-					return true
-				}
-				return false
-			}
-		}
-	}
-
 	var rctx *relabelCtx
-	rcs := allRelabelConfigs.Load()
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 {
+	if pcsGlobal.Len() > 0 || len(labelsGlobal) > 0 {
 		rctx = getRelabelCtx()
-		defer putRelabelCtx(rctx)
 	}
+	tss := wr.Timeseries
+	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
-
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -456,7 +286,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 		i := 0
 		for i < len(tss) {
 			samplesCount += len(tss[i].Samples)
-			labelsCount += len(tss[i].Samples) * len(tss[i].Labels)
+			labelsCount += len(tss[i].Labels)
 			i++
 			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
 				break
@@ -469,107 +299,39 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, dropSamplesOnFailur
 		} else {
 			tss = nil
 		}
-		if tenantRctx != nil {
-			tenantRctx.tenantToLabels(tssBlock, at.AccountID, at.ProjectID)
-		}
 		if rctx != nil {
 			rowsCountBeforeRelabel := getRowsCount(tssBlock)
-			tssBlock = rctx.applyRelabeling(tssBlock, pcsGlobal)
+			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
 			rowsCountAfterRelabel := getRowsCount(tssBlock)
 			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock) {
-			if !*disableOnDiskQueue {
-				logger.Panicf("BUG: tryPushBlockToRemoteStorages must return true if -remoteWrite.disableOnDiskQueue isn't set")
-			}
-			pushFailures.Inc()
-			if dropSamplesOnFailure {
-				samplesDropped.Add(rowsCount)
-				return true
-			}
-			return false
+		pushBlockToRemoteStorages(rwctxs, tssBlock)
+		if rctx != nil {
+			rctx.reset()
 		}
 	}
-	return true
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
 }
 
-var (
-	samplesDropped = metrics.NewCounter(`vmagent_remotewrite_samples_dropped_total`)
-	pushFailures   = metrics.NewCounter(`vmagent_remotewrite_push_failures_total`)
-)
-
-func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) bool {
+func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
 	if len(tssBlock) == 0 {
 		// Nothing to push
-		return true
+		return
 	}
-
-	if len(rwctxs) == 1 {
-		// Fast path - just push data to the configured single remote storage
-		return rwctxs[0].TryPush(tssBlock)
-	}
-
-	// We need to push tssBlock to multiple remote storages.
-	// This is either sharding or replication depending on -remoteWrite.shardByURL command-line flag value.
-	if *shardByURL {
-		// Shard the data among rwctxs
-		tssByURL := make([][]prompbmarshal.TimeSeries, len(rwctxs))
-		tmpLabels := promutils.GetLabels()
-		for _, ts := range tssBlock {
-			hashLabels := ts.Labels
-			if len(shardByURLLabelsMap) > 0 {
-				hashLabels = tmpLabels.Labels[:0]
-				for _, label := range ts.Labels {
-					if _, ok := shardByURLLabelsMap[label.Name]; ok {
-						hashLabels = append(hashLabels, label)
-					}
-				}
-			}
-			h := getLabelsHash(hashLabels)
-			idx := h % uint64(len(tssByURL))
-			tssByURL[idx] = append(tssByURL[idx], ts)
-		}
-		promutils.PutLabels(tmpLabels)
-
-		// Push sharded data to remote storages in parallel in order to reduce
-		// the time needed for sending the data to multiple remote storage systems.
-		var wg sync.WaitGroup
-		wg.Add(len(rwctxs))
-		var anyPushFailed uint64
-		for i, rwctx := range rwctxs {
-			tssShard := tssByURL[i]
-			if len(tssShard) == 0 {
-				continue
-			}
-			go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
-				defer wg.Done()
-				if !rwctx.TryPush(tss) {
-					atomic.StoreUint64(&anyPushFailed, 1)
-				}
-			}(rwctx, tssShard)
-		}
-		wg.Wait()
-		return atomic.LoadUint64(&anyPushFailed) == 0
-	}
-
-	// Replicate data among rwctxs.
-	// Push block to remote storages in parallel in order to reduce
-	// the time needed for sending the data to multiple remote storage systems.
+	// Push block to remote storages in parallel in order to reduce the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
-	var anyPushFailed uint64
 	for _, rwctx := range rwctxs {
+		wg.Add(1)
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
-			if !rwctx.TryPush(tssBlock) {
-				atomic.StoreUint64(&anyPushFailed, 1)
-			}
+			rwctx.Push(tssBlock)
 		}(rwctx)
 	}
 	wg.Wait()
-	return atomic.LoadUint64(&anyPushFailed) == 0
 }
 
 // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
@@ -661,14 +423,9 @@ var (
 )
 
 type remoteWriteCtx struct {
-	idx int
-	fq  *persistentqueue.FastQueue
-	c   *client
-
-	sas                 atomic.Pointer[streamaggr.Aggregators]
-	streamAggrKeepInput bool
-	streamAggrDropInput bool
-
+	idx        int
+	fq         *persistentqueue.FastQueue
+	c          *client
 	pss        []*pendingSeries
 	pssNextIdx uint64
 
@@ -676,33 +433,20 @@ type remoteWriteCtx struct {
 	rowsDroppedByRelabel   *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
 	pqURL.Fragment = ""
 	h := xxhash.Sum64([]byte(pqURL.String()))
-	queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
-	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArg(argIdx)
-	if maxPendingBytes != 0 && maxPendingBytes < persistentqueue.DefaultChunkFileSize {
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195
-		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
-		maxPendingBytes = persistentqueue.DefaultChunkFileSize
-	}
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, *disableOnDiskQueue)
+	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytesPerURL.N)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
 	})
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetInmemoryQueueLen())
 	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queue_blocked{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
-		if fq.IsWriteBlocked() {
-			return 1
-		}
-		return 0
-	})
-
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
@@ -712,9 +456,8 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	c.init(argIdx, *queues, sanitizedURL)
 
-	// Initialize pss
-	sf := significantFigures.GetOptionalArg(argIdx)
-	rd := roundDigits.GetOptionalArg(argIdx)
+	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
+	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
 	pssLen := *queues
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
@@ -723,10 +466,9 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
+		pss[i] = newPendingSeries(fq.MustWriteBlock, sf, rd)
 	}
-
-	rwctx := &remoteWriteCtx{
+	return &remoteWriteCtx{
 		idx: argIdx,
 		fq:  fq,
 		c:   c,
@@ -735,31 +477,9 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
-
-	// Initialize sas
-	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
-	if sasFile != "" {
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
-		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
-		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
-		}
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
-	}
-
-	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
-	// sas must be stopped before rwctx is closed
-	// because sas can write pending series to rwctx.pss if there are any
-	sas := rwctx.sas.Swap(nil)
-	sas.MustStop()
-
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
 	}
@@ -768,7 +488,6 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.UnblockAllReaders()
 	rwctx.c.MustStop()
 	rwctx.c = nil
-
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
@@ -776,11 +495,10 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.rowsDroppedByRelabel = nil
 }
 
-func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
-	// Apply relabeling
+func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
-	rcs := allRelabelConfigs.Load()
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
 	pcs := rcs.perURL[rwctx.idx]
 	if pcs.Len() > 0 {
 		rctx = getRelabelCtx()
@@ -788,134 +506,26 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries) bool {
 		// from affecting time series for other remoteWrite.url configs.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
-		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
+		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
 		rowsCountBeforeRelabel := getRowsCount(tss)
-		tss = rctx.applyRelabeling(tss, pcs)
+		tss = rctx.applyRelabeling(tss, nil, pcs)
 		rowsCountAfterRelabel := getRowsCount(tss)
 		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 	}
-	rowsCount := getRowsCount(tss)
-	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
-
-	// Apply stream aggregation if any
-	sas := rwctx.sas.Load()
-	if sas != nil {
-		matchIdxs := matchIdxsPool.Get()
-		matchIdxs.B = sas.Push(tss, matchIdxs.B)
-		if !rwctx.streamAggrKeepInput {
-			if rctx == nil {
-				rctx = getRelabelCtx()
-				// Make a copy of tss before dropping aggregated series
-				v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
-				tss = append(*v, tss...)
-			}
-			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
-		}
-		matchIdxsPool.Put(matchIdxs)
-	}
-
-	// Try pushing the data to remote storage
-	ok := rwctx.tryPushInternal(tss)
-
-	// Return back relabeling contexts to the pool
-	if rctx != nil {
-		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
-	}
-
-	return ok
-}
-
-var matchIdxsPool bytesutil.ByteBufferPool
-
-func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, dropInput bool) []prompbmarshal.TimeSeries {
-	dst := src[:0]
-	if !dropInput {
-		for i, match := range matchIdxs {
-			if match == 1 {
-				continue
-			}
-			dst = append(dst, src[i])
-		}
-	}
-	tail := src[len(dst):]
-	_ = prompbmarshal.ResetTimeSeries(tail)
-	return dst
-}
-
-func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSeries) {
-	if rwctx.tryPushInternal(tss) {
-		return
-	}
-	if !*disableOnDiskQueue {
-		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
-	}
-	pushFailures.Inc()
-	if *dropSamplesOnOverload {
-		rowsCount := getRowsCount(tss)
-		samplesDropped.Add(rowsCount)
-	}
-}
-
-func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
-	var rctx *relabelCtx
-	var v *[]prompbmarshal.TimeSeries
-	if len(labelsGlobal) > 0 {
-		// Make a copy of tss before adding extra labels in order to prevent
-		// from affecting time series for other remoteWrite.url configs.
-		rctx = getRelabelCtx()
-		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
-		tss = append(*v, tss...)
-		rctx.appendExtraLabels(tss, labelsGlobal)
-	}
-
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
-
-	ok := pss[idx].TryPush(tss)
-
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
+	pss[idx].Push(tss)
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
+		tssRelabelPool.Put(v)
 		putRelabelCtx(rctx)
 	}
-
-	return ok
 }
 
-func (rwctx *remoteWriteCtx) reinitStreamAggr() {
-	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if sasFile == "" {
-		// There is no stream aggregation for rwctx
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(rwctx.idx)
-	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, dedupInterval)
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
-		logger.Errorf("cannot reload stream aggregation config from -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", sasFile, err)
-		return
-	}
-	sas := rwctx.sas.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := rwctx.sas.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded stream aggregation configs at -remoteWrite.streamAggr.config=%q", sasFile)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("the config at -remoteWrite.streamAggr.config=%q wasn't changed", sasFile)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
-}
-
-var tssPool = &sync.Pool{
+var tssRelabelPool = &sync.Pool{
 	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}
 		return &a
@@ -929,20 +539,3 @@ func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 	}
 	return rowsCount
 }
-
-// CheckStreamAggrConfigs checks configs pointed by -remoteWrite.streamAggr.config
-func CheckStreamAggrConfigs() error {
-	pushNoop := func(tss []prompbmarshal.TimeSeries) {}
-	for idx, sasFile := range *streamAggrConfig {
-		if sasFile == "" {
-			continue
-		}
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, dedupInterval)
-		if err != nil {
-			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)
-		}
-		sas.MustStop()
-	}
-	return nil
-}

app/vmagent/remotewrite/statconn.go

@@ -27,7 +27,7 @@ var (
 	stdDialerOnce sync.Once
 )
 
-func statDial(ctx context.Context, _, addr string) (conn net.Conn, err error) {
+func statDial(ctx context.Context, networkUnused, addr string) (conn net.Conn, err error) {
 	network := netutil.GetTCPNetwork()
 	d := getStdDialer()
 	conn, err = d.DialContext(ctx, network, addr)

app/vmagent/vmimport/request_handler.go

@@ -1,6 +1,7 @@
 package vmimport
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -11,8 +12,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -30,9 +31,20 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
-		return insertRows(at, rows, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+			return insertRows(at, rows, extraLabels)
+		})
+	})
+}
+
+// InsertHandlerForReader processes metrics from given reader
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
+			return insertRows(nil, rows, nil)
+		})
 	})
 }
 
@@ -76,9 +88,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.PushWithAuthToken(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmalert-tool/README.md

@@ -1,3 +0,0 @@
-See vmalert-tool docs [here](https://docs.victoriametrics.com/vmalert-tool.html).
-
-vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert-tool.md).

app/vmalert-tool/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
-FROM $certs_image as certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 8429
-ENTRYPOINT ["/vmalert-tool-prod"]
-ARG TARGETARCH
-COPY vmalert-tool-linux-${TARGETARCH}-prod ./vmalert-tool-prod

app/vmalert/Makefile

@@ -73,7 +73,6 @@ test-vmalert:
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
 	go test -v -race -cover ./app/vmalert/remotewrite
-	go test -v -race -cover ./app/vmalert/utils
 
 run-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \
@@ -115,9 +114,6 @@ vmalert-linux-arm64:
 vmalert-linux-ppc64le:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmalert-linux-s390x:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
 vmalert-linux-386:
 	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert/alerting.go

@@ -0,0 +1,625 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// AlertingRule is basic alert entity
+type AlertingRule struct {
+	Type         datasource.Type
+	RuleID       uint64
+	Name         string
+	Expr         string
+	For          time.Duration
+	Labels       map[string]string
+	Annotations  map[string]string
+	GroupID      uint64
+	GroupName    string
+	EvalInterval time.Duration
+
+	q datasource.Querier
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores list of active alerts
+	alerts map[uint64]*notifier.Alert
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores the duration of the last Exec call
+	lastExecDuration time.Duration
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+	// stores the number of samples returned during
+	// the last evaluation
+	lastExecSamples int
+
+	metrics *alertingRuleMetrics
+}
+
+type alertingRuleMetrics struct {
+	errors  *utils.Gauge
+	pending *utils.Gauge
+	active  *utils.Gauge
+	samples *utils.Gauge
+}
+
+func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
+	ar := &AlertingRule{
+		Type:         group.Type,
+		RuleID:       cfg.ID,
+		Name:         cfg.Alert,
+		Expr:         cfg.Expr,
+		For:          cfg.For.Duration(),
+		Labels:       cfg.Labels,
+		Annotations:  cfg.Annotations,
+		GroupID:      group.ID(),
+		GroupName:    group.Name,
+		EvalInterval: group.Interval,
+		q: qb.BuildWithParams(datasource.QuerierParams{
+			DataSourceType:     &group.Type,
+			EvaluationInterval: group.Interval,
+			QueryParams:        group.Params,
+		}),
+		alerts:  make(map[uint64]*notifier.Alert),
+		metrics: &alertingRuleMetrics{},
+	}
+
+	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
+	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+		func() float64 {
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StatePending {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+		func() float64 {
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StateFiring {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
+		func() float64 {
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
+			if ar.lastExecError == nil {
+				return 0
+			}
+			return 1
+		})
+	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			ar.mu.RLock()
+			defer ar.mu.RUnlock()
+			return float64(ar.lastExecSamples)
+		})
+	return ar
+}
+
+// Close unregisters rule metrics
+func (ar *AlertingRule) Close() {
+	ar.metrics.active.Unregister()
+	ar.metrics.pending.Unregister()
+	ar.metrics.errors.Unregister()
+	ar.metrics.samples.Unregister()
+}
+
+// String implements Stringer interface
+func (ar *AlertingRule) String() string {
+	return ar.Name
+}
+
+// ID returns unique Rule ID
+// within the parent Group.
+func (ar *AlertingRule) ID() uint64 {
+	return ar.RuleID
+}
+
+type labelSet struct {
+	// origin labels from series
+	// used for templating
+	origin map[string]string
+	// processed labels with additional data
+	// used as Alert labels
+	processed map[string]string
+}
+
+// toLabels converts labels from given Metric
+// to labelSet which contains original and processed labels.
+func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
+	ls := &labelSet{
+		origin:    make(map[string]string, len(m.Labels)),
+		processed: make(map[string]string),
+	}
+	for _, l := range m.Labels {
+		ls.origin[l.Name] = l.Value
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		ls.processed[l.Name] = l.Value
+	}
+
+	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
+		Labels: ls.origin,
+		Value:  m.Values[0],
+		Expr:   ar.Expr,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %s", err)
+	}
+	for k, v := range extraLabels {
+		ls.processed[k] = v
+	}
+
+	// set additional labels to identify group and rule name
+	if ar.Name != "" {
+		ls.processed[alertNameLabel] = ar.Name
+	}
+	if !*disableAlertGroupLabel && ar.GroupName != "" {
+		ls.processed[alertGroupNameLabel] = ar.GroupName
+	}
+	return ls, nil
+}
+
+// ExecRange executes alerting rule on the given time range similarly to Exec.
+// It doesn't update internal states of the Rule and meant to be used just
+// to get time series for backfilling.
+// It returns ALERT and ALERT_FOR_STATE time series as result.
+func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error) {
+	series, err := ar.q.QueryRange(ctx, ar.Expr, start, end)
+	if err != nil {
+		return nil, err
+	}
+	var result []prompbmarshal.TimeSeries
+	qFn := func(query string) ([]datasource.Metric, error) {
+		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
+	}
+	for _, s := range series {
+		a, err := ar.newAlert(s, nil, time.Time{}, qFn) // initial alert
+		if err != nil {
+			return nil, fmt.Errorf("failed to create alert: %s", err)
+		}
+		if ar.For == 0 { // if alert is instant
+			a.State = notifier.StateFiring
+			for i := range s.Values {
+				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+			}
+			continue
+		}
+
+		// if alert with For > 0
+		prevT := time.Time{}
+		for i := range s.Values {
+			at := time.Unix(s.Timestamps[i], 0)
+			if at.Sub(prevT) > ar.EvalInterval {
+				// reset to Pending if there are gaps > EvalInterval between DPs
+				a.State = notifier.StatePending
+				a.ActiveAt = at
+			} else if at.Sub(a.ActiveAt) >= ar.For {
+				a.State = notifier.StateFiring
+				a.Start = at
+			}
+			prevT = at
+			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+		}
+	}
+	return result, nil
+}
+
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequently repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
+// Exec executes AlertingRule expression via the given Querier.
+// Based on the Querier results AlertingRule maintains notifier.Alerts
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
+	qMetrics, err := ar.q.Query(ctx, ar.Expr, ts)
+	ar.mu.Lock()
+	defer ar.mu.Unlock()
+
+	ar.lastExecTime = start
+	ar.lastExecDuration = time.Since(start)
+	ar.lastExecError = err
+	ar.lastExecSamples = len(qMetrics)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
+	}
+
+	for h, a := range ar.alerts {
+		// cleanup inactive alerts from previous Exec
+		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
+			delete(ar.alerts, h)
+		}
+	}
+
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
+	updated := make(map[uint64]struct{})
+	// update list of active alerts
+	for _, m := range qMetrics {
+		ls, err := ar.toLabels(m, qFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+		h := hash(ls.processed)
+		if _, ok := updated[h]; ok {
+			// duplicate may be caused by extra labels
+			// conflicting with the metric labels
+			ar.lastExecError = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, ar.lastExecError
+		}
+		updated[h] = struct{}{}
+		if a, ok := ar.alerts[h]; ok {
+			if a.State == notifier.StateInactive {
+				// alert could be in inactive state for resolvedRetention
+				// so when we again receive metrics for it - we switch it
+				// back to notifier.StatePending
+				a.State = notifier.StatePending
+				a.ActiveAt = ts
+			}
+			if a.Value != m.Values[0] {
+				// update Value field with latest value
+				a.Value = m.Values[0]
+				// and re-exec template since Value can be used
+				// in annotations
+				a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
+				if err != nil {
+					return nil, err
+				}
+			}
+			continue
+		}
+		a, err := ar.newAlert(m, ls, ar.lastExecTime, qFn)
+		if err != nil {
+			ar.lastExecError = err
+			return nil, fmt.Errorf("failed to create alert: %w", err)
+		}
+		a.ID = h
+		a.State = notifier.StatePending
+		a.ActiveAt = ts
+		ar.alerts[h] = a
+	}
+	var numActivePending int
+	for h, a := range ar.alerts {
+		// if alert wasn't updated in this iteration
+		// means it is resolved already
+		if _, ok := updated[h]; !ok {
+			if a.State == notifier.StatePending {
+				// alert was in Pending state - it is not
+				// active anymore
+				delete(ar.alerts, h)
+				continue
+			}
+			if a.State == notifier.StateFiring {
+				a.State = notifier.StateInactive
+				a.ResolvedAt = ts
+			}
+			continue
+		}
+		numActivePending++
+		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
+			a.State = notifier.StateFiring
+			a.Start = ts
+			alertsFired.Inc()
+		}
+	}
+	if limit > 0 && numActivePending > limit {
+		ar.alerts = map[uint64]*notifier.Alert{}
+		return nil, fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
+	}
+	return ar.toTimeSeries(ts.Unix()), nil
+}
+
+func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		ts := ar.alertToTimeSeries(a, timestamp)
+		tss = append(tss, ts...)
+	}
+	return tss
+}
+
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (ar *AlertingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*AlertingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update alerting rule with wrong type %#v", r)
+	}
+	ar.Expr = nr.Expr
+	ar.For = nr.For
+	ar.Labels = nr.Labels
+	ar.Annotations = nr.Annotations
+	ar.EvalInterval = nr.EvalInterval
+	ar.q = nr.q
+	return nil
+}
+
+// TODO: consider hashing algorithm in VM
+func hash(labels map[string]string) uint64 {
+	hash := fnv.New64a()
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		// drop __name__ to be consistent with Prometheus alerting
+		if k == "__name__" {
+			continue
+		}
+		name, value := k, labels[k]
+		hash.Write([]byte(name))
+		hash.Write([]byte(value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn templates.QueryFn) (*notifier.Alert, error) {
+	var err error
+	if ls == nil {
+		ls, err = ar.toLabels(m, qFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+	}
+	a := &notifier.Alert{
+		GroupID:  ar.GroupID,
+		Name:     ar.Name,
+		Labels:   ls.processed,
+		Value:    m.Values[0],
+		ActiveAt: start,
+		Expr:     ar.Expr,
+	}
+	a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
+	return a, err
+}
+
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
+	ar.mu.RLock()
+	defer ar.mu.RUnlock()
+	a, ok := ar.alerts[id]
+	if !ok {
+		return nil
+	}
+	return ar.newAlertAPI(*a)
+}
+
+// ToAPI returns Rule representation in form
+// of APIRule
+func (ar *AlertingRule) ToAPI() APIRule {
+	r := APIRule{
+		Type:           "alerting",
+		DatasourceType: ar.Type.String(),
+		Name:           ar.Name,
+		Query:          ar.Expr,
+		Duration:       ar.For.Seconds(),
+		Labels:         ar.Labels,
+		Annotations:    ar.Annotations,
+		LastEvaluation: ar.lastExecTime,
+		EvaluationTime: ar.lastExecDuration.Seconds(),
+		Health:         "ok",
+		State:          "inactive",
+		Alerts:         ar.AlertsToAPI(),
+		LastSamples:    ar.lastExecSamples,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
+	if ar.lastExecError != nil {
+		r.LastError = ar.lastExecError.Error()
+		r.Health = "err"
+	}
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
+	}
+	return r
+}
+
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
+	var alerts []*APIAlert
+	ar.mu.RLock()
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		alerts = append(alerts, ar.newAlertAPI(*a))
+	}
+	ar.mu.RUnlock()
+	return alerts
+}
+
+func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
+	aa := &APIAlert{
+		// encode as strings to avoid rounding
+		ID:      fmt.Sprintf("%d", a.ID),
+		GroupID: fmt.Sprintf("%d", a.GroupID),
+		RuleID:  fmt.Sprintf("%d", ar.RuleID),
+
+		Name:        a.Name,
+		Expression:  ar.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.ActiveAt,
+		Restored:    a.Restored,
+		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
+	}
+	if alertURLGeneratorFn != nil {
+		aa.SourceLink = alertURLGeneratorFn(a)
+	}
+	return aa
+}
+
+const (
+	// alertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// alertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// alertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// alertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+
+	// alertGroupNameLabel defines the label name attached for generated time series.
+	// attaching this label may be disabled via `-disableAlertgroupLabel` flag.
+	alertGroupNameLabel = "alertgroup"
+)
+
+// alertToTimeSeries converts the given alert with the given timestamp to time series
+func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
+}
+
+// alertForToTimeSeries returns a timeseries that represents
+// state of active alerts, where value is time when alert become active
+func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
+}
+
+// Restore restores the state of active alerts basing on previously written time series.
+// Restore restores only ActiveAt field. Field State will be always Pending and supposed
+// to be updated on next Exec, as well as Value field.
+// Only rules with For > 0 will be restored.
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
+	if q == nil {
+		return fmt.Errorf("querier is nil")
+	}
+
+	ts := time.Now()
+	qFn := func(query string) ([]datasource.Metric, error) { return ar.q.Query(ctx, query, ts) }
+
+	// account for external labels in filter
+	var labelsFilter string
+	for k, v := range labels {
+		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
+	}
+
+	// Get the last data point in range via MetricsQL `last_over_time`.
+	// We don't use plain PromQL since Prometheus doesn't support
+	// remote write protocol which is used for state persistence in vmalert.
+	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
+		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
+	qMetrics, err := q.Query(ctx, expr, ts)
+	if err != nil {
+		return err
+	}
+
+	for _, m := range qMetrics {
+		ls := &labelSet{
+			origin:    make(map[string]string, len(m.Labels)),
+			processed: make(map[string]string, len(m.Labels)),
+		}
+		for _, l := range m.Labels {
+			if l.Name == "__name__" {
+				continue
+			}
+			ls.origin[l.Name] = l.Value
+			ls.processed[l.Name] = l.Value
+		}
+		a, err := ar.newAlert(m, ls, time.Unix(int64(m.Values[0]), 0), qFn)
+		if err != nil {
+			return fmt.Errorf("failed to create alert: %w", err)
+		}
+		a.ID = hash(ls.processed)
+		a.State = notifier.StatePending
+		a.Restored = true
+		ar.alerts[a.ID] = a
+		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
+	}
+	return nil
+}
+
+// alertsToSend walks through the current alerts of AlertingRule
+// and returns only those which should be sent to notifier.
+// Isn't concurrent safe.
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
+	needsSending := func(a *notifier.Alert) bool {
+		if a.State == notifier.StatePending {
+			return false
+		}
+		if a.ResolvedAt.After(a.LastSent) {
+			return true
+		}
+		return a.LastSent.Add(resendDelay).Before(ts)
+	}
+
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		if !needsSending(a) {
+			continue
+		}
+		a.End = ts.Add(resolveDuration)
+		if a.State == notifier.StateInactive {
+			a.End = a.ResolvedAt
+		}
+		a.LastSent = ts
+		alerts = append(alerts, *a)
+	}
+	return alerts
+}

app/vmalert/alerting_test.go

@@ -0,0 +1,919 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestAlertingRule_ToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule  *AlertingRule
+		alert *notifier.Alert
+		expTS []prompbmarshal.TimeSeries
+	}{
+		{
+			newTestAlertingRule("instant", 0),
+			&notifier.Alert{State: notifier.StateFiring},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+				}),
+			},
+		},
+		{
+			newTestAlertingRule("instant extra labels", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					"job":           "foo",
+					"instance":      "bar",
+				}),
+			},
+		},
+		{
+			newTestAlertingRule("instant labels override", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+				}),
+			},
+		},
+		{
+			newTestAlertingRule("for", time.Second),
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
+			},
+		},
+		{
+			newTestAlertingRule("for pending", 10*time.Second),
+			&notifier.Alert{State: notifier.StatePending, ActiveAt: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StatePending.String(),
+				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			tc.rule.alerts[tc.alert.ID] = tc.alert
+			tss := tc.rule.toTimeSeries(timestamp.Unix())
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Exec(t *testing.T) {
+	const defaultStep = 5 * time.Millisecond
+	type testAlert struct {
+		labels []string
+		alert  *notifier.Alert
+	}
+	testCases := []struct {
+		rule      *AlertingRule
+		steps     [][]datasource.Metric
+		expAlerts []testAlert
+	}{
+		{
+			newTestAlertingRule("empty", 0),
+			[][]datasource.Metric{},
+			nil,
+		},
+		{
+			newTestAlertingRule("empty labels", 0),
+			[][]datasource.Metric{
+				{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
+			},
+			[]testAlert{
+				{alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-firing", 0),
+			[][]datasource.Metric{
+				{
+					metricWithLabels(t, "name", "foo"),
+					metricWithLabels(t, "name", "foo1"),
+					metricWithLabels(t, "name", "foo2"),
+				},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-steps-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo1")},
+				{metricWithLabels(t, "name", "foo2")},
+			},
+			// 1: fire first alert
+			// 2: fire second alert, set first inactive
+			// 3: fire third alert, set second inactive
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending", time.Minute),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
+			},
+		},
+		{
+			newTestAlertingRule("for-fired", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>empty", time.Second),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset and delete pending alerts
+				{},
+			},
+			nil,
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.q = fq
+			tc.rule.GroupID = fakeGroup.ID()
+			for _, step := range tc.steps {
+				fq.reset()
+				fq.add(step...)
+				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
+					t.Fatalf("unexpected err: %s", err)
+				}
+				// artificial delay between applying steps
+				time.Sleep(defaultStep)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			expAlerts := make(map[uint64]*notifier.Alert)
+			for _, ta := range tc.expAlerts {
+				labels := make(map[string]string)
+				for i := 0; i < len(ta.labels); i += 2 {
+					k, v := ta.labels[i], ta.labels[i+1]
+					labels[k] = v
+				}
+				labels[alertNameLabel] = tc.rule.Name
+				h := hash(labels)
+				expAlerts[h] = ta.alert
+			}
+			for key, exp := range expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertingRule_ExecRange(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		data      []datasource.Metric
+		expAlerts []*notifier.Alert
+	}{
+		{
+			newTestAlertingRule("empty", 0),
+			[]datasource.Metric{},
+			nil,
+		},
+		{
+			newTestAlertingRule("empty labels", 0),
+			[]datasource.Metric{
+				{Values: []float64{1}, Timestamps: []int64{1}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing", 0),
+			[]datasource.Metric{
+				metricWithLabels(t, "name", "foo"),
+			},
+			[]*notifier.Alert{
+				{
+					Labels: map[string]string{"name": "foo"},
+					State:  notifier.StateFiring,
+				},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing-on-range", 0),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending", time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(3, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("for-firing", 3*time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1, 1, 1}, Timestamps: []int64{1, 2, 5, 6, 20}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(20, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("multi-series-for=>pending=>pending=>firing", 3*time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+				},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				//
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"foo": "bar",
+					}},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0),
+					Labels: map[string]string{
+						"foo": "bar",
+					}},
+			},
+		},
+		{
+			newTestRuleWithLabels("multi-series-firing", "source", "vm"),
+			[]datasource.Metric{
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+				},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				//
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_ExecRange"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.q = fq
+			tc.rule.GroupID = fakeGroup.ID()
+			fq.add(tc.data...)
+			gotTS, err := tc.rule.ExecRange(context.TODO(), time.Now(), time.Now())
+			if err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			var expTS []prompbmarshal.TimeSeries
+			var j int
+			for _, series := range tc.data {
+				for _, timestamp := range series.Timestamps {
+					a := tc.expAlerts[j]
+					if a.Labels == nil {
+						a.Labels = make(map[string]string)
+					}
+					a.Labels[alertNameLabel] = tc.rule.Name
+					expTS = append(expTS, tc.rule.alertToTimeSeries(a, timestamp)...)
+					j++
+				}
+			}
+			if len(gotTS) != len(expTS) {
+				t.Fatalf("expected %d time series; got %d", len(expTS), len(gotTS))
+			}
+			for i := range expTS {
+				got, exp := gotTS[i], expTS[i]
+				if !reflect.DeepEqual(got, exp) {
+					t.Fatalf("%d: expected \n%v but got \n%v", i, exp, got)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Restore(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		metrics   []datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestRuleWithLabels("no extra labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(nil): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("metric labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "metric labels",
+					alertGroupNameLabel, "groupID",
+					"foo", "bar",
+					"namespace", "baz",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{
+					alertNameLabel:      "metric labels",
+					alertGroupNameLabel: "groupID",
+					"foo":               "bar",
+					"namespace":         "baz",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("rule labels", "source", "vm"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"foo", "bar",
+					"namespace", "baz",
+					// extra labels set by rule
+					"source", "vm",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{
+					"foo":       "bar",
+					"namespace": "baz",
+					"source":    "vm",
+				}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("multiple alerts"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-1",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(2*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-2",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(3*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-3",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{"host": "localhost-1"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(time.Hour)},
+				hash(map[string]string{"host": "localhost-2"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(2 * time.Hour)},
+				hash(map[string]string{"host": "localhost-3"}): {State: notifier.StatePending,
+					ActiveAt: time.Now().Truncate(3 * time.Hour)},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.GroupID = fakeGroup.ID()
+			tc.rule.q = fq
+			fq.add(tc.metrics...)
+			if err := tc.rule.Restore(context.TODO(), fq, time.Hour, nil); err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+				if got.ActiveAt != exp.ActiveAt {
+					t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Exec_Negative(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+
+	// successful attempt
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	_, err := ar.Exec(context.TODO(), time.Now(), 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// label `job` will collide with rule extra label and will make both time series equal
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
+	if !errors.Is(err, errDuplicate) {
+		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
+	}
+
+	fq.reset()
+
+	expErr := "connection reset by peer"
+	fq.setErr(errors.New(expErr))
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), expErr) {
+		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+	}
+}
+
+func TestAlertingRuleLimit(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+	ar.For = time.Minute
+	testCases := []struct {
+		limit  int
+		err    string
+		tssNum int
+	}{
+		{
+			limit:  0,
+			tssNum: 4,
+		},
+		{
+			limit:  -1,
+			tssNum: 4,
+		},
+		{
+			limit:  1,
+			err:    "exec exceeded limit of 1 with 2 alerts",
+			tssNum: 0,
+		},
+		{
+			limit:  4,
+			tssNum: 4,
+		},
+	}
+	var (
+		err       error
+		timestamp = time.Now()
+	)
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
+	for _, testCase := range testCases {
+		_, err = ar.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+	fq.reset()
+}
+
+func TestAlertingRule_Template(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		metrics   []datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestRuleWithLabels("common", "region", "east"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "instance", "foo"),
+				metricWithValueAndLabels(t, 1, "instance", "bar"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
+					Annotations: map[string]string{},
+					Labels: map[string]string{
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "foo",
+					},
+				},
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
+					Annotations: map[string]string{},
+					Labels: map[string]string{
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "bar",
+					},
+				},
+			},
+		},
+		{
+			&AlertingRule{
+				Name: "override label",
+				Labels: map[string]string{
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
+					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
+					Labels: map[string]string{
+						alertNameLabel: "override label",
+						"instance":     "foo",
+					},
+					Annotations: map[string]string{
+						"summary":     `first: Too high connection number for "foo"`,
+						"description": `override: It is 2 connections for "foo"`,
+					},
+				},
+				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
+					Labels: map[string]string{
+						alertNameLabel: "override label",
+						"instance":     "bar",
+					},
+					Annotations: map[string]string{
+						"summary":     `second: Too high connection number for "bar"`,
+						"description": `override: It is 10 connections for "bar"`,
+					},
+				},
+			},
+		},
+		{
+			&AlertingRule{
+				Name:      "OriginLabels",
+				GroupName: "Testing",
+				Labels: map[string]string{
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1,
+					alertNameLabel, "originAlertname",
+					alertGroupNameLabel, "originGroupname",
+					"instance", "foo"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{
+					alertNameLabel:      "OriginLabels",
+					alertGroupNameLabel: "Testing",
+					"instance":          "foo"}): {
+					Labels: map[string]string{
+						alertNameLabel:      "OriginLabels",
+						alertGroupNameLabel: "Testing",
+						"instance":          "foo",
+					},
+					Annotations: map[string]string{
+						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
+					},
+				},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.GroupID = fakeGroup.ID()
+			tc.rule.q = fq
+			fq.add(tc.metrics...)
+			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			for hash, expAlert := range tc.expAlerts {
+				gotAlert := tc.rule.alerts[hash]
+				if gotAlert == nil {
+					t.Fatalf("alert %d is missing; labels: %v; annotations: %v",
+						hash, expAlert.Labels, expAlert.Annotations)
+				}
+				if !reflect.DeepEqual(expAlert.Annotations, gotAlert.Annotations) {
+					t.Fatalf("expected to have annotations %#v; got %#v", expAlert.Annotations, gotAlert.Annotations)
+				}
+				if !reflect.DeepEqual(expAlert.Labels, gotAlert.Labels) {
+					t.Fatalf("expected to have labels %#v; got %#v", expAlert.Labels, gotAlert.Labels)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertsToSend(t *testing.T) {
+	ts := time.Now()
+	f := func(alerts, expAlerts []*notifier.Alert, resolveDuration, resendDelay time.Duration) {
+		t.Helper()
+		ar := &AlertingRule{alerts: make(map[uint64]*notifier.Alert)}
+		for i, a := range alerts {
+			ar.alerts[uint64(i)] = a
+		}
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
+		if gotAlerts == nil && expAlerts == nil {
+			return
+		}
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected to get %d alerts; got %d instead",
+				len(expAlerts), len(gotAlerts))
+		}
+		sort.Slice(expAlerts, func(i, j int) bool {
+			return expAlerts[i].Name < expAlerts[j].Name
+		})
+		sort.Slice(gotAlerts, func(i, j int) bool {
+			return gotAlerts[i].Name < gotAlerts[j].Name
+		})
+		for i, exp := range expAlerts {
+			got := gotAlerts[i]
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
+			}
+		}
+	}
+
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
+		5*time.Minute, time.Minute,
+	)
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+	f( // inactive alert which has been sent already
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // inactive alert which has been resolved after last send
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+}
+
+func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
+	r := newTestAlertingRule(name, 0)
+	r.Labels = make(map[string]string)
+	for i := 0; i < len(labels); i += 2 {
+		r.Labels[labels[i]] = labels[i+1]
+	}
+	return r
+}
+
+func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
+	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor, EvalInterval: waitFor}
+}

app/vmalert/config/config.go

@@ -4,29 +4,29 @@ import (
 	"crypto/md5"
 	"fmt"
 	"hash/fnv"
+	"io/ioutil"
 	"net/url"
+	"path/filepath"
 	"sort"
 	"strings"
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 // Group contains list of Rules grouped into
 // entity with one name and evaluation interval
 type Group struct {
-	Type       Type `yaml:"type,omitempty"`
-	File       string
-	Name       string              `yaml:"name"`
-	Interval   *promutils.Duration `yaml:"interval,omitempty"`
-	EvalOffset *promutils.Duration `yaml:"eval_offset,omitempty"`
-	// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155
-	EvalDelay   *promutils.Duration `yaml:"eval_delay,omitempty"`
+	Type        datasource.Type `yaml:"type,omitempty"`
+	File        string
+	Name        string              `yaml:"name"`
+	Interval    *promutils.Duration `yaml:"interval,omitempty"`
 	Limit       int                 `yaml:"limit,omitempty"`
 	Rules       []Rule              `yaml:"rules"`
 	Concurrency int                 `yaml:"concurrency"`
@@ -38,12 +38,7 @@ type Group struct {
 	Checksum string
 	// Optional HTTP URL parameters added to each rule request
 	Params url.Values `yaml:"params"`
-	// Headers contains optional HTTP headers added to each rule request
-	Headers []Header `yaml:"headers,omitempty"`
-	// NotifierHeaders contains optional HTTP headers sent to notifiers for generated notifications
-	NotifierHeaders []Header `yaml:"notifier_headers,omitempty"`
-	// EvalAlignment will make the timestamp of group query requests be aligned with interval
-	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
+
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
 }
@@ -60,7 +55,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 	// change default value to prometheus datasource.
 	if g.Type.Get() == "" {
-		g.Type.Set(NewPrometheusType())
+		g.Type.Set(datasource.NewPrometheusType())
 	}
 
 	h := md5.New()
@@ -69,27 +64,11 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-// Validate checks configuration errors for group and internal rules
-func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool) error {
+// Validate check for internal Group or Rule configuration errors
+func (g *Group) Validate(validateAnnotations, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
-	if g.Interval.Duration() < 0 {
-		return fmt.Errorf("interval shouldn't be lower than 0")
-	}
-	if g.EvalOffset.Duration() < 0 {
-		return fmt.Errorf("eval_offset shouldn't be lower than 0")
-	}
-	// if `eval_offset` is set, interval won't use global evaluationInterval flag and must bigger than offset.
-	if g.EvalOffset.Duration() > g.Interval.Duration() {
-		return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
-	}
-	if g.Limit < 0 {
-		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", g.Limit)
-	}
-	if g.Concurrency < 0 {
-		return fmt.Errorf("invalid concurrency %d, shouldn't be less than 0", g.Concurrency)
-	}
 
 	uniqueRules := map[uint64]struct{}{}
 	for _, r := range g.Rules {
@@ -98,26 +77,26 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 			ruleName = r.Alert
 		}
 		if _, ok := uniqueRules[r.ID]; ok {
-			return fmt.Errorf("%q is a duplicate in group", r.String())
+			return fmt.Errorf("rule %q duplicate", ruleName)
 		}
 		uniqueRules[r.ID] = struct{}{}
 		if err := r.Validate(); err != nil {
-			return fmt.Errorf("invalid rule %q: %w", ruleName, err)
+			return fmt.Errorf("invalid rule %q.%q: %w", g.Name, ruleName, err)
 		}
 		if validateExpressions {
 			// its needed only for tests.
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
 			if err := exprValidator(r.Expr); err != nil {
-				return fmt.Errorf("invalid expression for rule  %q: %w", ruleName, err)
+				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
-		if validateTplFn != nil {
-			if err := validateTplFn(r.Annotations); err != nil {
-				return fmt.Errorf("invalid annotations for rule  %q: %w", ruleName, err)
+		if validateAnnotations {
+			if err := notifier.ValidateTemplates(r.Annotations); err != nil {
+				return fmt.Errorf("invalid annotations for rule %q.%q: %w", g.Name, ruleName, err)
 			}
-			if err := validateTplFn(r.Labels); err != nil {
-				return fmt.Errorf("invalid labels for rule  %q: %w", ruleName, err)
+			if err := notifier.ValidateTemplates(r.Labels); err != nil {
+				return fmt.Errorf("invalid labels for rule %q.%q: %w", g.Name, ruleName, err)
 			}
 		}
 	}
@@ -127,19 +106,13 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 // Rule describes entity that represent either
 // recording rule or alerting rule.
 type Rule struct {
-	ID     uint64
-	Record string              `yaml:"record,omitempty"`
-	Alert  string              `yaml:"alert,omitempty"`
-	Expr   string              `yaml:"expr"`
-	For    *promutils.Duration `yaml:"for,omitempty"`
-	// Alert will continue firing for this long even when the alerting expression no longer has results.
-	KeepFiringFor *promutils.Duration `yaml:"keep_firing_for,omitempty"`
-	Labels        map[string]string   `yaml:"labels,omitempty"`
-	Annotations   map[string]string   `yaml:"annotations,omitempty"`
-	Debug         bool                `yaml:"debug,omitempty"`
-	// UpdateEntriesLimit defines max number of rule's state updates stored in memory.
-	// Overrides `-rule.updateEntriesLimit`.
-	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
+	ID          uint64
+	Record      string              `yaml:"record,omitempty"`
+	Alert       string              `yaml:"alert,omitempty"`
+	Expr        string              `yaml:"expr"`
+	For         *promutils.Duration `yaml:"for,omitempty"`
+	Labels      map[string]string   `yaml:"labels,omitempty"`
+	Annotations map[string]string   `yaml:"annotations,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -163,32 +136,6 @@ func (r *Rule) Name() string {
 	return r.Alert
 }
 
-// String implements Stringer interface
-func (r *Rule) String() string {
-	ruleType := "recording"
-	if r.Alert != "" {
-		ruleType = "alerting"
-	}
-	b := strings.Builder{}
-	b.WriteString(fmt.Sprintf("%s rule %q", ruleType, r.Name()))
-	b.WriteString(fmt.Sprintf("; expr: %q", r.Expr))
-
-	kv := sortMap(r.Labels)
-	for i := range kv {
-		if i == 0 {
-			b.WriteString("; labels:")
-		}
-		b.WriteString(" ")
-		b.WriteString(kv[i].key)
-		b.WriteString("=")
-		b.WriteString(kv[i].value)
-		if i < len(kv)-1 {
-			b.WriteString(",")
-		}
-	}
-	return b.String()
-}
-
 // HashRule hashes significant Rule fields into
 // unique hash that supposed to define Rule uniqueness
 func HashRule(r Rule) uint64 {
@@ -221,54 +168,27 @@ func (r *Rule) Validate() error {
 	return checkOverflow(r.XXX, "rule")
 }
 
-// ValidateTplFn must validate the given annotations
-type ValidateTplFn func(annotations map[string]string) error
-
-// cLogger is a logger with support of logs suppressing.
-// it is used when logs emitted by config package needs
-// to be suppressed.
-var cLogger = &log.Logger{}
-
-// ParseSilent parses rule configs from given file patterns without emitting logs
-func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	cLogger.Suppress(true)
-	defer cLogger.Suppress(false)
-
-	files, err := readFromFS(pathPatterns)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read from the config: %w", err)
-	}
-	return parse(files, validateTplFn, validateExpressions)
-}
-
 // Parse parses rule configs from given file patterns
-func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read from the config: %w", err)
+func Parse(pathPatterns []string, validateAnnotations, validateExpressions bool) ([]Group, error) {
+	var fp []string
+	for _, pattern := range pathPatterns {
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file pattern %s: %w", pattern, err)
+		}
+		fp = append(fp, matches...)
 	}
-	groups, err := parse(files, validateTplFn, validateExpressions)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse %s: %w", pathPatterns, err)
-	}
-	if len(groups) < 1 {
-		cLogger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
-	}
-	return groups, nil
-}
-
-func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
 	errGroup := new(utils.ErrGroup)
 	var groups []Group
-	for file, data := range files {
+	for _, file := range fp {
 		uniqueGroups := map[string]struct{}{}
-		gr, err := parseConfig(data)
+		gr, err := parseFile(file)
 		if err != nil {
 			errGroup.Add(fmt.Errorf("failed to parse file %q: %w", file, err))
 			continue
 		}
 		for _, g := range gr {
-			if err := g.Validate(validateTplFn, validateExpressions); err != nil {
+			if err := g.Validate(validateAnnotations, validateExpressions); err != nil {
 				errGroup.Add(fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err))
 				continue
 			}
@@ -284,20 +204,18 @@ func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpress
 	if err := errGroup.Err(); err != nil {
 		return nil, err
 	}
-	sort.SliceStable(groups, func(i, j int) bool {
-		if groups[i].File != groups[j].File {
-			return groups[i].File < groups[j].File
-		}
-		return groups[i].Name < groups[j].Name
-	})
+	if len(groups) < 1 {
+		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
 	return groups, nil
 }
 
-func parseConfig(data []byte) ([]Group, error) {
-	data, err := envtemplate.ReplaceBytes(data)
+func parseFile(path string) ([]Group, error) {
+	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
+		return nil, fmt.Errorf("error reading alert rule file: %w", err)
 	}
+	data = envtemplate.Replace(data)
 	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.

app/vmalert/config/config_test.go

@@ -1,8 +1,6 @@
 package config
 
 import (
-	"net/http"
-	"net/http/httptest"
 	"net/url"
 	"os"
 	"strings"
@@ -11,7 +9,7 @@ import (
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -24,54 +22,16 @@ func TestMain(m *testing.M) {
 }
 
 func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, true, true); err != nil {
 		t.Errorf("error parsing files %s", err)
 	}
 }
 
-func TestParseFromURL(t *testing.T) {
-	mux := http.NewServeMux()
-	mux.HandleFunc("/bad", func(w http.ResponseWriter, _ *http.Request) {
-		w.Write([]byte("foo bar"))
-	})
-	mux.HandleFunc("/good-alert", func(w http.ResponseWriter, _ *http.Request) {
-		w.Write([]byte(`
-groups:
-  - name: TestGroup
-    rules:
-      - alert: Conns
-        expr: vm_tcplistener_conns > 0`))
-	})
-	mux.HandleFunc("/good-rr", func(w http.ResponseWriter, _ *http.Request) {
-		w.Write([]byte(`
-groups:
-  - name: TestGroup
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)`))
-	})
-
-	srv := httptest.NewServer(mux)
-	defer srv.Close()
-
-	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
-		t.Errorf("error parsing URLs %s", err)
-	}
-
-	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
-		t.Errorf("expected parsing error: %s", err)
-	}
-}
-
 func TestParseBad(t *testing.T) {
 	testCases := []struct {
 		path   []string
 		expErr string
 	}{
-		{
-			[]string{"testdata/rules/rules_interval_bad.rules"},
-			"eval_offset should be smaller than interval",
-		},
 		{
 			[]string{"testdata/rules/rules0-bad.rules"},
 			"unexpected token",
@@ -100,17 +60,9 @@ func TestParseBad(t *testing.T) {
 			[]string{"testdata/rules/rules1-bad.rules"},
 			"bad graphite expr",
 		},
-		{
-			[]string{"testdata/dir/rules6-bad.rules"},
-			"missing ':' in header",
-		},
-		{
-			[]string{"http://unreachable-url"},
-			"failed to",
-		},
 	}
 	for _, tc := range testCases {
-		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
+		_, err := Parse(tc.path, true, true)
 		if err == nil {
 			t.Errorf("expected to get error")
 			return
@@ -146,37 +98,7 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "group name must be set",
 		},
 		{
-			group: &Group{
-				Name:     "negative interval",
-				Interval: promutils.NewDuration(-1),
-			},
-			expErr: "interval shouldn't be lower than 0",
-		},
-		{
-			group: &Group{
-				Name:       "wrong eval_offset",
-				Interval:   promutils.NewDuration(time.Minute),
-				EvalOffset: promutils.NewDuration(2 * time.Minute),
-			},
-			expErr: "eval_offset should be smaller than interval",
-		},
-		{
-			group: &Group{
-				Name:  "wrong limit",
-				Limit: -1,
-			},
-			expErr: "invalid limit",
-		},
-		{
-			group: &Group{
-				Name:        "wrong concurrency",
-				Concurrency: -1,
-			},
-			expErr: "invalid concurrency",
-		},
-		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{
 						Record: "record",
@@ -187,8 +109,7 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{
 						Record: "record",
@@ -200,8 +121,7 @@ func TestGroup_Validate(t *testing.T) {
 			validateExpressions: true,
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{
 						Alert: "alert",
@@ -215,8 +135,7 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{
 						Alert: "alert",
@@ -233,8 +152,7 @@ func TestGroup_Validate(t *testing.T) {
 			validateAnnotations: true,
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{
 						Alert: "alert",
@@ -249,8 +167,7 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "duplicate",
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -263,8 +180,7 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "duplicate",
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{Record: "record", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -277,8 +193,7 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "duplicate",
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -291,8 +206,7 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{
-				Name: "test",
+			group: &Group{Name: "test",
 				Rules: []Rule{
 					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
 						"summary": "{{ value|query }}",
@@ -305,9 +219,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "",
 		},
 		{
-			group: &Group{
-				Name: "test thanos",
-				Type: NewRawType("thanos"),
+			group: &Group{Name: "test thanos",
+				Type: datasource.NewRawType("thanos"),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -318,9 +231,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr:              "unknown datasource type",
 		},
 		{
-			group: &Group{
-				Name: "test graphite",
-				Type: NewGraphiteType(),
+			group: &Group{Name: "test graphite",
+				Type: datasource.NewGraphiteType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "some-description",
@@ -331,9 +243,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr:              "",
 		},
 		{
-			group: &Group{
-				Name: "test prometheus",
-				Type: NewPrometheusType(),
+			group: &Group{Name: "test prometheus",
+				Type: datasource.NewPrometheusType(),
 				Rules: []Rule{
 					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
 						"description": "{{ value|query }}",
@@ -346,7 +257,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite inherit",
-				Type: NewGraphiteType(),
+				Type: datasource.NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sumSeries(time('foo.bar',10))",
@@ -361,7 +272,7 @@ func TestGroup_Validate(t *testing.T) {
 		{
 			group: &Group{
 				Name: "test graphite prometheus bad expr",
-				Type: NewGraphiteType(),
+				Type: datasource.NewGraphiteType(),
 				Rules: []Rule{
 					{
 						Expr: "sum(up == 0 ) by (host)",
@@ -375,13 +286,8 @@ func TestGroup_Validate(t *testing.T) {
 			expErr: "invalid rule",
 		},
 	}
-
 	for _, tc := range testCases {
-		var validateTplFn ValidateTplFn
-		if tc.validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
-		}
-		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
+		err := tc.group.Validate(tc.validateAnnotations, tc.validateExpressions)
 		if err == nil {
 			if tc.expErr != "" {
 				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
@@ -437,7 +343,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},
@@ -602,71 +508,6 @@ limit: 10
 rules:
   - alert: foo
     expr: sum by(job) (up == 1)
-`)
-	})
-
-	t.Run("`headers` change", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-headers:
-  - "TenantID: foo"
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-`, `
-name: TestGroup
-headers:
-  - "TenantID: bar"
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-`)
-	})
-
-	t.Run("`notifier_headers` change", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-notifier_headers:
-  - "TenantID: foo"
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-`, `
-name: TestGroup
-notifier_headers:
-  - "TenantID: bar"
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-`)
-	})
-
-	t.Run("`debug` change", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-`, `
-name: TestGroup
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-    debug: true
-`)
-	})
-	t.Run("`update_entries_limit` change", func(t *testing.T) {
-		f(t, `
-name: TestGroup
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-`, `
-name: TestGroup
-rules:
-  - alert: foo
-    expr: sum by(job) (up == 1)
-    update_entries_limit: 33
 `)
 	})
 }

app/vmalert/config/fs.go

@@ -1,111 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fsurl"
-)
-
-// FS represent a file system abstract for reading files.
-type FS interface {
-	// Init initializes FS.
-	Init() error
-
-	// String must return human-readable representation of FS.
-	String() string
-
-	// List returns the list of file names which will be read via Read fn
-	List() ([]string, error)
-
-	// Read returns a list of read files in form of a map
-	// where key is a file name and value is a content of read file.
-	// Read must be called only after the successful Init call.
-	Read(files []string) (map[string][]byte, error)
-}
-
-var (
-	fsRegistryMu sync.Mutex
-	fsRegistry   = make(map[string]FS)
-)
-
-// readFromFS parses the given path list and inits FS for each item.
-// Once initialed, readFromFS will try to read and return files from each FS.
-// readFromFS returns an error if at least one FS failed to init.
-// The function can be called multiple times but each unique path
-// will be initialed only once.
-//
-// It is allowed to mix different FS types in path list.
-func readFromFS(paths []string) (map[string][]byte, error) {
-	var err error
-	result := make(map[string][]byte)
-	for _, path := range paths {
-		fsRegistryMu.Lock()
-		fs, ok := fsRegistry[path]
-		if !ok {
-			fs, err = newFS(path)
-			if err != nil {
-				fsRegistryMu.Unlock()
-				return nil, fmt.Errorf("error while parsing path %q: %w", path, err)
-			}
-			if err := fs.Init(); err != nil {
-				fsRegistryMu.Unlock()
-				return nil, fmt.Errorf("error while initializing path %q: %w", path, err)
-			}
-			fsRegistry[path] = fs
-		}
-		fsRegistryMu.Unlock()
-
-		list, err := fs.List()
-		if err != nil {
-			return nil, fmt.Errorf("failed to list files from %q", fs)
-		}
-
-		cLogger.Infof("found %d files to read from %q", len(list), fs)
-
-		if len(list) < 1 {
-			continue
-		}
-
-		ts := time.Now()
-		files, err := fs.Read(list)
-		if err != nil {
-			return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
-		}
-		cLogger.Infof("finished reading %d files in %v from %q", len(list), time.Since(ts), fs)
-
-		for k, v := range files {
-			if _, ok := result[k]; ok {
-				return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k)
-			}
-			result[k] = v
-		}
-	}
-	return result, nil
-}
-
-// newFS creates FS based on the give path.
-// Supported file systems are: fs
-func newFS(originPath string) (FS, error) {
-	scheme := "fs"
-	path := originPath
-	n := strings.Index(path, "://")
-	if n >= 0 {
-		scheme = path[:n]
-		path = path[n+len("://"):]
-	}
-	if len(path) == 0 {
-		return nil, fmt.Errorf("path cannot be empty")
-	}
-	switch scheme {
-	case "fs":
-		return &fslocal.FS{Pattern: path}, nil
-	case "http", "https":
-		return &fsurl.FS{Path: originPath}, nil
-	default:
-		return nil, fmt.Errorf("unsupported scheme %q", scheme)
-	}
-}

app/vmalert/config/fs_test.go

@@ -1,39 +0,0 @@
-package config
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestNewFS(t *testing.T) {
-	f := func(path, expStr string) {
-		t.Helper()
-		fs, err := newFS(path)
-		if err != nil {
-			t.Fatalf("unexpected err: %s", err)
-		}
-		if fs.String() != expStr {
-			t.Fatalf("expected FS %q; got %q", expStr, fs.String())
-		}
-	}
-
-	f("/foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
-	f("fs:///foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
-}
-
-func TestNewFSNegative(t *testing.T) {
-	f := func(path, expErr string) {
-		t.Helper()
-		_, err := newFS(path)
-		if err == nil {
-			t.Fatalf("expected to have err: %s", expErr)
-		}
-		if !strings.Contains(err.Error(), expErr) {
-			t.Fatalf("expected to have err %q; got %q instead", expErr, err)
-		}
-	}
-
-	f("", "path cannot be empty")
-	f("fs://", "path cannot be empty")
-	f("foobar://baz", `unsupported scheme "foobar"`)
-}

app/vmalert/config/fslocal/fslocal.go

@@ -1,50 +0,0 @@
-package fslocal
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/bmatcuk/doublestar/v4"
-)
-
-// FS represents a local file system
-type FS struct {
-	// Pattern is used for matching one or multiple files.
-	// The pattern may describe hierarchical names such as
-	// /usr/*/bin/ed (assuming the Separator is '/').
-	Pattern string
-}
-
-// Init verifies that configured Pattern is correct
-func (fs *FS) Init() error {
-	_, err := doublestar.FilepathGlob(fs.Pattern)
-	return err
-}
-
-// String implements Stringer interface
-func (fs *FS) String() string {
-	return fmt.Sprintf("Local FS{MatchPattern: %q}", fs.Pattern)
-}
-
-// List returns the list of file names which will be read via Read fn
-func (fs *FS) List() ([]string, error) {
-	matches, err := doublestar.FilepathGlob(fs.Pattern)
-	if err != nil {
-		return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err)
-	}
-	return matches, nil
-}
-
-// Read returns a map of read files where
-// key is the file name and value is file's content.
-func (fs *FS) Read(files []string) (map[string][]byte, error) {
-	result := make(map[string][]byte)
-	for _, path := range files {
-		data, err := os.ReadFile(path)
-		if err != nil {
-			return nil, fmt.Errorf("error while reading file %q: %w", path, err)
-		}
-		result[path] = data
-	}
-	return result, nil
-}

app/vmalert/config/fsurl/url.go

@@ -1,57 +0,0 @@
-package fsurl
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-)
-
-// FS represents a struct which can read content from URL Path
-type FS struct {
-	// Path defines the URL to read the data from
-	Path string
-}
-
-// Init verifies that configured Path is correct
-func (fs *FS) Init() error {
-	_, err := url.Parse(fs.Path)
-	return err
-}
-
-// String implements Stringer interface
-func (fs *FS) String() string {
-	return fmt.Sprintf("URL {Path: %q}", fs.Path)
-}
-
-// List returns the list of file names which will be read via Read fn
-// List isn't supported by FS and reads from Path only
-func (fs *FS) List() ([]string, error) {
-	return []string{fs.Path}, nil
-}
-
-// Read returns a map of read files where
-// key is the file name and value is file's content.
-func (fs *FS) Read(files []string) (map[string][]byte, error) {
-	result := make(map[string][]byte)
-	for _, path := range files {
-		resp, err := http.Get(path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read from %q: %w", path, err)
-		}
-		data, err := io.ReadAll(resp.Body)
-		_ = resp.Body.Close()
-		if resp.StatusCode != http.StatusOK {
-			if len(data) > 4*1024 {
-				data = data[:4*1024]
-			}
-			return nil, fmt.Errorf("unexpected status code when fetching %q: %d, expecting %d; response: %q",
-				path, resp.StatusCode, http.StatusOK, data)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("cannot read %q: %w", path, err)
-		}
-		result[path] = data
-	}
-	return result, nil
-}

app/vmalert/config/log/logger.go

@@ -1,59 +0,0 @@
-package log
-
-import (
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-)
-
-// Logger is using lib/logger for logging
-// but can be suppressed via Suppress method
-type Logger struct {
-	mu       sync.RWMutex
-	disabled bool
-}
-
-// Suppress whether to ignore message logging.
-// Once suppressed, logging continues to be ignored
-// until logger is un-suppressed.
-func (l *Logger) Suppress(v bool) {
-	l.mu.Lock()
-	l.disabled = v
-	l.mu.Unlock()
-}
-
-func (l *Logger) isDisabled() bool {
-	l.mu.RLock()
-	defer l.mu.RUnlock()
-	return l.disabled
-}
-
-// Errorf logs error message.
-func (l *Logger) Errorf(format string, args ...interface{}) {
-	if l.isDisabled() {
-		return
-	}
-	logger.Errorf(format, args...)
-}
-
-// Warnf logs warning message.
-func (l *Logger) Warnf(format string, args ...interface{}) {
-	if l.isDisabled() {
-		return
-	}
-	logger.Warnf(format, args...)
-}
-
-// Infof logs info message.
-func (l *Logger) Infof(format string, args ...interface{}) {
-	if l.isDisabled() {
-		return
-	}
-	logger.Infof(format, args...)
-}
-
-// Panicf logs panic message and panics.
-// Panicf can't be suppressed
-func (l *Logger) Panicf(format string, args ...interface{}) {
-	logger.Panicf(format, args...)
-}

app/vmalert/config/log/logger_test.go

@@ -1,54 +0,0 @@
-package log
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-)
-
-func TestOutput(t *testing.T) {
-	testOutput := &bytes.Buffer{}
-	logger.SetOutputForTests(testOutput)
-	defer logger.ResetOutputForTest()
-
-	log := &Logger{}
-
-	mustMatch := func(exp string) {
-		t.Helper()
-		if exp == "" {
-			if testOutput.String() != "" {
-				t.Errorf("expected output to be empty; got %q", testOutput.String())
-				return
-			}
-		}
-		if !strings.Contains(testOutput.String(), exp) {
-			t.Errorf("output %q should contain %q", testOutput.String(), exp)
-		}
-		fmt.Println(testOutput.String())
-		testOutput.Reset()
-	}
-
-	log.Warnf("foo")
-	mustMatch("foo")
-
-	log.Infof("info %d", 2)
-	mustMatch("info 2")
-
-	log.Errorf("error %s %d", "baz", 5)
-	mustMatch("error baz 5")
-
-	log.Suppress(true)
-
-	log.Warnf("foo")
-	mustMatch("")
-
-	log.Infof("info %d", 2)
-	mustMatch("")
-
-	log.Errorf("error %q %d", "baz", 5)
-	mustMatch("")
-
-}