lib/promscrape: add -promscrape.disableKeepAlive command-line flag for disabling http keep-alive connections when scraping targets

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/580

.dockerignore

@@ -4,4 +4,3 @@ gocache-for-docker
 victoria-metrics-data
 vmstorage-data
 vmselect-cache
-.vscode

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version**
+The line returned when passing `--version` command line flag to binary. For example:
+```
+$ ./victoria-metrics-prod --version
+victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
+```
+
+**Used command-line flags**
+Command-line flags are listed as `flag{name="httpListenAddr", value=":443"} 1` lines at `/metrics` page.
+See the following docs for details:
+
+* [monitoring for single-node VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#monitoring)
+* [montioring for VictoriaMetrics cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#monitoring)
+
+**Additional context**
+Add any other context about the problem here such as error logs from VictoriaMetrics and Prometheus,
+`/metrics` output, screenshots from the official Grafana dashboards for VictoriaMetrics:
+
+* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
+* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,86 +0,0 @@
-name: Bug report
-description: Create a report to help us improve
-labels: [bug]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade) 
-        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-        and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) first.
-  - type: textarea
-    id: describe-the-bug
-    attributes:
-      label: Describe the bug
-      description: |
-        A clear and concise description of what the bug is.
-      placeholder: |
-        When I do `A` VictoriaMetrics does `B`. I expect it to do `C`.
-    validations:
-      required: true
-  - type: textarea
-    id: to-reproduce
-    attributes:
-      label: To Reproduce
-      description: |
-        Steps to reproduce the behavior.
-        If reproducing an issue requires some specific configuration file, please paste it here.
-      placeholder: |
-        Steps to reproduce the behavior.
-    validations:
-      required: true
-  - type: textarea
-    id: version
-    attributes:
-      label: Version
-      description: |
-        The line returned when passing `--version` command line flag to the binary. For example:
-        ```
-        $ ./victoria-metrics-prod --version
-        victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
-        ```
-    validations:
-      required: true
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs
-      description: |
-        Check if any warnings or errors were logged by VictoriaMetrics components
-        or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
-    validations:
-      required: false
-  - type: textarea
-    id: screenshots
-    attributes:
-      label: Screenshots
-      description: |
-        If applicable, add screenshots to help explain your problem.
-  
-        For VictoriaMetrics health-state issues please provide full-length screenshots
-        of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
-        
-        See how to setup monitoring here:
-          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
-    validations:
-      required: false
-  - type: textarea
-    id: flags
-    attributes:
-      label: Used command-line flags
-      description: |
-        Please provide the command-line flags used for running VictoriaMetrics and its components.
-    validations:
-      required: false
-  - type: textarea
-    id: additional-info
-    attributes:
-      label: Additional information
-      placeholder: |
-        Additional information that doesn't fit elsewhere
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/configuration.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Ask on Slack
-    url: https://slack.victoriametrics.com/
-    about: You can ask for help here!

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,43 +0,0 @@
-name: Feature request
-description: Suggest an idea for this project
-labels: [enhancement]
-body:
-  - type: textarea
-    id: describe-the-problem
-    attributes:
-      label: Is your feature request related to a problem? Please describe
-      description: |
-        A clear and concise description of what the problem is.
-      placeholder: |
-        Ex. I'm always frustrated when [...]
-    validations:
-      required: false
-  - type: textarea
-    id: describe-the-solution
-    attributes:
-      label: Describe the solution you'd like
-      description: |
-        A clear and concise description of what you want to happen.
-    validations:
-      required: true
-  - type: textarea
-    id: alternative-solutions
-    attributes:
-      label: Describe alternatives you've considered
-      description: |
-        A clear and concise description of any alternative solutions or features you've considered.
-      placeholder: |
-        I have tried to do `A`, but that doesn't solve a problem completely.
-        I have tried to do `A` and `B`, but implementing this would be better.
-    validations:
-      required: false
-  - type: textarea
-    id: feature-additional-info
-    attributes:
-      label: Additional information
-      description: |
-        Additional information which you consider helpful for implementing this feature.
-      placeholder: |
-        Add any other context or screenshots about the feature request here.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/question.yml

@@ -1,32 +0,0 @@
-name: Question
-description: Ask a question regarding VictoriaMetrics or its components
-labels: [question]
-body:
-  - type: textarea
-    id: describe-the-component
-    attributes:
-      label: Is your question request related to a specific component?
-      placeholder: |
-        VictoriaMetrics, vmagent, vmalert, vmui, etc...
-    validations:
-      required: false
-  - type: textarea
-    id: describe-the-question
-    attributes:
-      label: Describe the question in detail
-      description: |
-        A clear and concise description of the issue and the question.
-    validations:
-      required: true
-  - type: checkboxes
-    id: troubleshooting
-    attributes:
-      label: Troubleshooting docs
-      description: I am familiar with the following troubleshooting docs
-      options:
-        - label: General - https://docs.victoriametrics.com/victoriametrics/troubleshooting/
-          required: false
-        - label: vmagent - https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
-          required: false
-        - label: vmalert - https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting
-          required: false

.github/copilot-instructions.md

@@ -1,23 +0,0 @@
-# Project Overview
-
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
-
-## Folder Structure
-
-- `/app`: Contains the compilable binaries.
-- `/lib`: Contains the golang reusable libraries
-- `/docs/victoriametrics`: Contains documentation for the project.
-- `/apptest/tests`: Contains integration tests.
-
-## Libraries and Frameworks
-
-- Backend: Golang, no framework. Use third-party libraries sparingly.
-- Frontend: React.
-
-## Code review guidelines
-
-Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
-Verify the entry is under the ## tip section and matches the structure and style of existing entries.
-Chore-only changes may be omitted from the changelog.
-
-

.github/dependabot.yml

@@ -1,30 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 0
-  - package-ecosystem: "bundler"
-    directory: "/docs"
-    schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 0
-  - package-ecosystem: "gomod"
-    directory: "/app/vmui/packages/vmui/web"
-    schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 0
-  - package-ecosystem: "docker"
-    directory: "/"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "npm"
-    directory: "/app/vmui/packages/vmui"
-    schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 0

.github/pull_request_template.md

@@ -1,10 +0,0 @@
-### Describe Your Changes
-
-Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications.
-
-### Checklist
-
-The following checks are **mandatory**:
-
-- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
-- [ ] My change adheres to [VictoriaMetrics development goals](https://docs.victoriametrics.com/victoriametrics/goals/).

.github/workflows/build.yml

@@ -1,69 +0,0 @@
-name: build
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile'
-      - '**/Makefile'
-      - '!app/vmui/**'
-      - '.github/workflows/build.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile'
-      - '**/Makefile'
-      - '!app/vmui/**'
-      - '.github/workflows/build.yml'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  build:
-    name: ${{ matrix.os }}-${{ matrix.arch }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-          - os: linux
-            arch: arm
-          - os: linux
-            arch: ppc64le
-          - os: linux
-            arch: 386
-          - os: freebsd
-            arch: amd64
-          - os: openbsd
-            arch: amd64
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Build vmcluster for ${{ matrix.os }}-${{ matrix.arch }}
-        run: make vmcluster-${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/check-licenses.yml

@@ -1,38 +0,0 @@
-name: license-check
-on:
-  push:
-    paths:
-      - 'vendor'
-  pull_request:
-    paths:
-      - 'vendor'
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@master
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
-
-      - name: Check License
-        run: make check-licenses

.github/workflows/codeql-analysis-go.yml

@@ -1,62 +0,0 @@
-name: 'CodeQL Go'
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v5
-
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: go
-
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: 'language:go'

.github/workflows/docs.yaml

@@ -1,57 +0,0 @@
-name: publish-docs
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'docs/**'
-      - '.github/workflows/docs.yaml'
-  workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-        with:
-          path: __vm
-
-      - name: Checkout private code
-        uses: actions/checkout@v5
-        with:
-          repository: VictoriaMetrics/vmdocs
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: __vm-docs
-
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v6
-        id: import-gpg
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-          git_config_global: true
-
-      - name: Copy docs
-        id: update
-        run: |
-          find docs -type d -maxdepth 1 -mindepth 1 -exec \
-            sh -c 'rsync -zarvh --delete {}/ ../__vm-docs/content/$(basename {})/' \;
-          echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
-        working-directory: __vm
-
-      - name: Push to vmdocs
-        run: |
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          if [[ -n $(git status --porcelain) ]]; then
-            git add .
-            git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.update.outputs.SHORT_SHA }}"
-            git push
-          fi
-        working-directory: __vm-docs

.github/workflows/github-pages.yml

@@ -0,0 +1,30 @@
+name: github-pages
+on:
+  push:
+    paths:
+      - 'docs/*'
+      - 'README.md'
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git gpages
+          cp docs/* gpages
+          cp README.md gpages
+          cd gpages
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add .
+          git commit -m "update github pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf gpages

.github/workflows/main.yml

@@ -0,0 +1,52 @@
+name: main
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@master
+        with:
+          go-version: 1.14
+        id: go
+      - name: Dependencies
+        env:
+          GO111MODULE: on
+        run: |
+          go get -u golang.org/x/lint/golint
+          go get -u github.com/kisielk/errcheck
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0
+      - name: Code checkout
+        uses: actions/checkout@master
+      - name: Build
+        env:
+          GO111MODULE: on
+        run: |
+          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+          make check-all
+          git diff --exit-code
+          make test-full
+          make test-pure
+          make test-full-386
+          make victoria-metrics
+          make victoria-metrics-pure
+          make victoria-metrics-arm
+          make victoria-metrics-arm64
+          make vmutils
+          GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=darwin go build -mod=vendor ./app/victoria-metrics
+      - name: Publish coverage
+        uses: codecov/codecov-action@v1.0.6
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}
+          file: ./coverage.txt
+

.github/workflows/test.yml

@@ -1,113 +0,0 @@
-name: test
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-
-jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-
-      - name: Cache golangci-lint
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/golangci-lint
-            ~/go/bin
-          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
-
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
-
-  unit:
-    name: unit
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        scenario:
-          - 'test-full'
-          - 'test-full-386'
-          - 'test-pure'
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v5
-        with:
-          files: ./coverage.txt
-
-  integration:
-    name: integration
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Run integration tests
-        run: make integration-test

.github/workflows/vmui.yml

@@ -1,82 +0,0 @@
-name: vmui
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - 'app/vmui/packages/vmui/**'
-      - '.github/workflows/vmui.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - 'app/vmui/packages/vmui/**'
-      - '.github/workflows/vmui.yml'
-
-permissions:
-  contents: read
-  packages: read
-  pull-requests: read
-  checks: write
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  vmui-checks:
-    name: VMUI Checks (lint, test, typecheck)
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: '24.x'
-
-      - name: Cache node-modules
-        uses: actions/cache@v4
-        with:
-          path: |
-            app/vmui/packages/vmui/node_modules
-          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: vmui-artifacts-${{ runner.os }}-
-
-      - name: Run lint
-        id: lint
-        run: make vmui-lint
-        continue-on-error: true
-
-      - name: Run tests
-        id: test
-        run: make vmui-test
-        continue-on-error: true
-
-      - name: Run typecheck
-        id: typecheck
-        run: make vmui-typecheck
-        continue-on-error: true
-
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@v3
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
-      - name: Check overall status
-        run: |
-          echo "Lint status: ${{ steps.lint.outcome }}"
-          echo "Test status: ${{ steps.test.outcome }}"
-          echo "Typecheck status: ${{ steps.typecheck.outcome }}"
-
-          if [[ "${{ steps.lint.outcome }}" == "failure" || "${{ steps.test.outcome }}" == "failure" || "${{ steps.typecheck.outcome }}" == "failure" ]]; then
-            echo "One or more checks failed"
-            exit 1
-          else
-            echo "All checks passed"
-          fi

.github/workflows/wiki.yml

@@ -0,0 +1,28 @@
+name: wiki
+on:
+  push:
+    paths:
+      - 'docs/*'
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
+          cp docs/* wiki
+          cd wiki
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add .
+          git commit -m "update wiki pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf wiki

.gitignore

@@ -4,28 +4,14 @@
 *.pprof
 /bin
 .idea
-.vscode
 *.test
 *.swp
-/vmdocs
 /gocache-for-docker
-/victoria-logs-data
 /victoria-metrics-data
 /vmagent-remotewrite-data
-/vlagent-remotewritewrite
 /vmstorage-data
 /vmselect-cache
 /package/temp-deb-*
 /package/temp-rpm-*
 /package/*.deb
 /package/*.rpm
-.DS_store
-Gemfile.lock
-/_site
-_site
-*.tmp
-/docs/.jekyll-metadata
-coverage.txt
-cspell.json
-*~
-deployment/docker/provisioning/plugins/

.golangci.yml

@@ -1,29 +0,0 @@
-version: "2"
-linters:
-  settings:
-    errcheck:
-      exclude-functions:
-        - fmt.Fprintf
-        - fmt.Fprint
-        - (net/http.ResponseWriter).Write
-  exclusions:
-    generated: lax
-    presets:
-      - common-false-positives
-      - legacy
-      - std-error-handling
-    rules:
-      - linters:
-          - staticcheck
-        text: 'SA(4003|1019|5011):'
-    paths:
-      - third_party$
-      - builtin$
-      - examples$
-formatters:
-  exclusions:
-    generated: lax
-    paths:
-      - third_party$
-      - builtin$
-      - examples$

.wwhrd.yml

@@ -1,7 +0,0 @@
-allowlist:
-  - Apache-2.0
-  - MIT
-  - BSD-3-Clause
-  - BSD-2-Clause
-  - ISC
-  - MPL-2.0

CODE_OF_CONDUCT.md

@@ -7,7 +7,7 @@ contributors and maintainers pledge to making participation in our project and
 our community a harassment-free experience for everyone, regardless of age, body
 size, disability, ethnicity, sex characteristics, gender identity and expression,
 level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion or sexual identity and orientation.
+appearance, race, religion, or sexual identity and orientation.
 
 ## Our Standards
 
@@ -24,9 +24,9 @@ Examples of unacceptable behavior by participants include:
 
 * The use of sexualized language or imagery and unwelcome sexual attention or
  advances
-* Trolling, insulting/derogatory comments and personal or political attacks
+* Trolling, insulting/derogatory comments, and personal or political attacks
 * Public or private harassment
-* Publishing others' private information, such as physical or electronic
+* Publishing others' private information, such as a physical or electronic
  address, without explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
  professional setting
@@ -38,26 +38,26 @@ behavior and are expected to take appropriate and fair corrective action in
 response to any instances of unacceptable behavior.
 
 Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues and other contributions
-that are not aligned to this Code of Conduct or to ban temporarily or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
 permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive or harmful.
+threatening, offensive, or harmful.
 
 ## Scope
 
 This Code of Conduct applies both within project spaces and in public spaces
 when an individual is representing the project or its community. Examples of
 representing a project or community include using an official project e-mail
-address, posting via an official social media account or acting as an appointed
+address, posting via an official social media account, or acting as an appointed
 representative at an online or offline event. Representation of a project may be
 further defined and clarified by project maintainers.
 
 ## Enforcement
 
-Instances of abusive, harassing or otherwise unacceptable behavior may be
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported by contacting the project team at info@victoriametrics.com. All
 complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate for the circumstances. The project team is
+is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.
 Further details of specific enforcement policies may be posted separately.
 
@@ -68,9 +68,9 @@ members of the project's leadership.
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at <https://www.contributor-covenant.org/version/1/4/code-of-conduct.html>
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
 
 [homepage]: https://www.contributor-covenant.org
 
 For answers to common questions about this code of conduct, see
-<https://www.contributor-covenant.org/faq>
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -1 +1,16 @@
-The document has been moved [here](https://docs.victoriametrics.com/victoriametrics/contributing/).
+If you like VictoriaMetrics and want to contribute, then we need the following:
+
+- Filing issues and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+- Spreading a word about VictoriaMetrics: conference talks, articles, comments, experience sharing with colleagues.
+- Updating documentation.
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2020 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,330 +1,155 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
-MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
-DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
 PKG_TAG := $(BUILDINFO_TAG)
 endif
 
-EXTRA_DOCKER_TAG_SUFFIX ?=
-EXTRA_GO_BUILD_TAGS ?=
-
-GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
-TAR_OWNERSHIP ?= --owner=1000 --group=1000
-
-GOLANGCI_LINT_VERSION := 2.4.0
-
-.PHONY: $(MAKECMDGOALS)
-
-include app/*/Makefile
-include codespell/Makefile
-include docs/Makefile
-include deployment/*/Makefile
-include dashboards/Makefile
-include package/release/Makefile
-include benchmarks/Makefile
+GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date -u +'%Y%m%d-%H%M%S')-$(BUILDINFO_TAG)'
 
 all: \
-	vminsert \
-	vmselect \
-	vmstorage
+	victoria-metrics-prod \
+	vmagent-prod \
+	vmalert-prod \
+	vmauth-prod \
+	vmbackup-prod \
+	vmrestore-prod
 
-all-pure: \
-	vminsert-pure \
-	vmselect-pure \
-	vmstorage-pure
+include app/*/Makefile
+include deployment/*/Makefile
 
 clean:
 	rm -rf bin/*
 
-vmcluster-linux-amd64: \
-	vminsert-linux-amd64 \
-	vmselect-linux-amd64 \
-	vmstorage-linux-amd64
-
-vmcluster-linux-arm64: \
-	vminsert-linux-arm64 \
-	vmselect-linux-arm64 \
-	vmstorage-linux-arm64
-
-vmcluster-linux-arm: \
-	vminsert-linux-arm \
-	vmselect-linux-arm \
-	vmstorage-linux-arm
-
-vmcluster-linux-ppc64le: \
-	vminsert-linux-ppc64le \
-	vmselect-linux-ppc64le \
-	vmstorage-linux-ppc64le
-
-vmcluster-linux-386: \
-	vminsert-linux-386 \
-	vmselect-linux-386 \
-	vmstorage-linux-386
-
-vmcluster-freebsd-amd64: \
-	vminsert-freebsd-amd64 \
-	vmselect-freebsd-amd64 \
-	vmstorage-freebsd-amd64
-
-vmcluster-openbsd-amd64: \
-	vminsert-openbsd-amd64 \
-	vmselect-openbsd-amd64 \
-	vmstorage-openbsd-amd64
-
-vmcluster-windows-amd64: \
-	vminsert-windows-amd64 \
-	vmselect-windows-amd64 \
-	vmstorage-windows-amd64
-
-vmcluster-darwin-amd64: \
-	vminsert-darwin-amd64 \
-	vmselect-darwin-amd64 \
-	vmstorage-darwin-amd64
-
-vmcluster-darwin-arm64: \
-	vminsert-darwin-arm64 \
-	vmselect-darwin-arm64 \
-	vmstorage-darwin-arm64
-
-# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
-crossbuild: vmcluster-crossbuild
-
-# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
-vmcluster-crossbuild:
-	$(MAKE_PARALLEL) vmcluster-linux-amd64 \
-		vmcluster-linux-arm64 \
-		vmcluster-linux-arm \
-		vmcluster-linux-ppc64le \
-		vmcluster-linux-386 \
-		vmcluster-freebsd-amd64 \
-		vmcluster-openbsd-amd64
-
 publish: \
-	publish-vminsert \
-	publish-vmselect \
-	publish-vmstorage
+	publish-victoria-metrics \
+	publish-vmagent \
+	publish-vmalert \
+	publish-vmauth \
+	publish-vmbackup \
+	publish-vmrestore
 
 package: \
-	package-vminsert \
-	package-vmselect \
-	package-vmstorage
+	package-victoria-metrics \
+	package-vmagent \
+	package-vmalert \
+	package-vmauth \
+	package-vmbackup \
+	package-vmrestore
 
-publish-final-images:
-	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=victoria-metrics $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmagent $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmalert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmauth $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackup $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmrestore $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) $(MAKE) publish-latest
+vmutils: \
+	vmagent \
+	vmalert \
+	vmauth \
+	vmbackup \
+	vmrestore
 
-publish-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
+release: \
+	release-victoria-metrics \
+	release-vmutils
 
-publish-release:
-	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && $(MAKE) publish
+release-victoria-metrics: victoria-metrics-prod
+	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod && \
+		sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt
 
-release:
-	$(MAKE_PARALLEL) release-vmcluster
-
-release-vmcluster: \
-	release-vmcluster-linux-amd64 \
-	release-vmcluster-linux-arm64 \
-	release-vmcluster-freebsd-amd64 \
-	release-vmcluster-openbsd-amd64 \
-	release-vmcluster-windows-amd64 \
-	release-vmcluster-darwin-amd64 \
-	release-vmcluster-darwin-arm64
-
-release-vmcluster-linux-amd64:
-	GOOS=linux GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-linux-arm64:
-	GOOS=linux GOARCH=arm64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-freebsd-amd64:
-	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-openbsd-amd64:
-	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmcluster-windows-goarch
-
-release-vmcluster-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-darwin-arm64:
-	 GOOS=darwin GOARCH=arm64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-goos-goarch: \
-	vminsert-$(GOOS)-$(GOARCH)-prod \
-	vmselect-$(GOOS)-$(GOARCH)-prod \
-	vmstorage-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vminsert-$(GOOS)-$(GOARCH)-prod \
-			vmselect-$(GOOS)-$(GOARCH)-prod \
-			vmstorage-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vminsert-$(GOOS)-$(GOARCH)-prod \
-			vmselect-$(GOOS)-$(GOARCH)-prod \
-			vmstorage-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vminsert-$(GOOS)-$(GOARCH)-prod \
-		vmselect-$(GOOS)-$(GOARCH)-prod \
-		vmstorage-$(GOOS)-$(GOARCH)-prod
-
-release-vmcluster-windows-goarch: \
-	vminsert-windows-$(GOARCH)-prod \
-	vmselect-windows-$(GOARCH)-prod \
-	vmstorage-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vminsert-windows-$(GOARCH)-prod.exe \
-			vmselect-windows-$(GOARCH)-prod.exe \
-			vmstorage-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vminsert-windows-$(GOARCH)-prod.exe \
-			vmselect-windows-$(GOARCH)-prod.exe \
-			vmstorage-windows-$(GOARCH)-prod.exe \
-		> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vminsert-windows-$(GOARCH)-prod.exe \
-		vmselect-windows-$(GOARCH)-prod.exe \
-		vmstorage-windows-$(GOARCH)-prod.exe
+release-vmutils: \
+	vmagent-prod \
+	vmalert-prod \
+	vmauth-prod \
+	vmbackup-prod \
+	vmrestore-prod
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmalert-prod vmauth-prod vmbackup-prod vmrestore-prod && \
+		sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
 
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
-	gofmt -l -w -s ./lib
-	gofmt -l -w -s ./app
-	gofmt -l -w -s ./apptest
+	GO111MODULE=on gofmt -l -w -s ./lib
+	GO111MODULE=on gofmt -l -w -s ./app
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
-	go vet ./app/...
-	go vet ./apptest/...
+	GO111MODULE=on go vet -mod=vendor ./lib/...
+	GO111MODULE=on go vet -mod=vendor ./app/...
 
-check-all: fmt vet golangci-lint govulncheck
+lint: install-golint
+	golint lib/...
+	golint app/...
 
-clean-checkers: remove-golangci-lint remove-govulncheck
+install-golint:
+	which golint || GO111MODULE=off go get -u golang.org/x/lint/golint
+
+errcheck: install-errcheck
+	errcheck -exclude=errcheck_excludes.txt ./lib/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
+
+install-errcheck:
+	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
+
+check-all: fmt vet lint errcheck golangci-lint
 
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
-
-integration-test:
-	$(MAKE) apptest
-
-apptest:
-	$(MAKE) all vmctl vmbackup vmrestore
-	go test ./apptest/... -skip="^TestSingle.*"
+	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
-	go test -bench=. ./app/...
+	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
-	CGO_ENABLED=0 go test -bench=. ./app/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
 
 vendor-update:
-	go get -u ./lib/...
-	go get -u ./app/...
-	go mod tidy -compat=1.24
-	go mod vendor
+	GO111MODULE=on go get -u ./lib/...
+	GO111MODULE=on go get -u ./app/...
+	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
 
 app-local:
-	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
-
-app-local-goos-goarch:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
-
-app-local-windows-goarch:
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
-	which qtc || go install github.com/valyala/quicktemplate/qtc@latest
+	which qtc || GO111MODULE=off go get -u github.com/valyala/quicktemplate/qtc
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)
+	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
 
-remove-golangci-lint:
-	rm -rf `which golangci-lint`
-
-govulncheck: install-govulncheck
-	govulncheck ./...
-
-install-govulncheck:
-	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
-
-remove-govulncheck:
-	rm -rf `which govulncheck`
-
-install-wwhrd:
-	which wwhrd || go install github.com/frapposelli/wwhrd@latest
-
-check-licenses: install-wwhrd
-	wwhrd check -f .wwhrd.yml
+docs-sync:
+	cp app/vmagent/README.md docs/vmagent.md
+	cp app/vmalert/README.md docs/vmalert.md
+	cp app/vmauth/README.md docs/vmauth.md
+	cp app/vmbackup/README.md docs/vmbackup.md
+	cp app/vmrestore/README.md docs/vmrestore.md
+	cp README.md docs/Single-server-VictoriaMetrics.md

SECURITY.md

@@ -1,18 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-The following versions of VictoriaMetrics receive regular security fixes:
-
-| Version | Supported          |
-|---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| other releases  | :x:                |
-
-See [this page](https://victoriametrics.com/security/) for more details.
-
-## Reporting a Vulnerability
-
-Please report any security issues to <security@victoriametrics.com>

app/victoria-logs/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/victoria-metrics/Makefile

@@ -0,0 +1,110 @@
+# All these commands must run from repository root.
+
+victoria-metrics:
+	APP_NAME=victoria-metrics $(MAKE) app-local
+
+victoria-metrics-race:
+	APP_NAME=victoria-metrics RACE=-race $(MAKE) app-local
+
+victoria-metrics-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker
+
+victoria-metrics-pure-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-pure
+
+victoria-metrics-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-amd64
+
+victoria-metrics-arm-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm
+
+victoria-metrics-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm64
+
+victoria-metrics-ppc64le-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-ppc64le
+
+victoria-metrics-386-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
+
+package-victoria-metrics:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker
+
+package-victoria-metrics-pure:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-pure
+
+package-victoria-metrics-amd64:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-amd64
+
+package-victoria-metrics-arm:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-arm
+
+package-victoria-metrics-arm64:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-arm64
+
+package-victoria-metrics-ppc64le:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-ppc64le
+
+package-victoria-metrics-386:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-386
+
+publish-victoria-metrics:
+	APP_NAME=victoria-metrics $(MAKE) publish-via-docker
+
+run-victoria-metrics:
+	mkdir -p victoria-metrics-data
+	DOCKER_OPTS='-v $(shell pwd)/victoria-metrics-data:/victoria-metrics-data' \
+	APP_NAME=victoria-metrics \
+	ARGS='-graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -retentionPeriod=12 -search.maxUniqueTimeseries=1000000 -search.maxQueryDuration=10m' \
+	$(MAKE) run-via-docker
+
+victoria-metrics-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-amd64 ./app/victoria-metrics
+
+victoria-metrics-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics
+
+victoria-metrics-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm64 ./app/victoria-metrics
+
+victoria-metrics-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-ppc64le ./app/victoria-metrics
+
+victoria-metrics-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-386 ./app/victoria-metrics
+
+victoria-metrics-pure:
+	APP_NAME=victoria-metrics $(MAKE) app-local-pure
+
+### Packaging as DEB - amd64
+victoria-metrics-package-deb: victoria-metrics-prod
+	./package/package_deb.sh amd64
+
+### Packaging as DEB - arm64
+victoria-metrics-package-deb-arm64: victoria-metrics-arm64-prod
+	./package/package_deb.sh arm64
+
+### Packaging as DEB - all
+victoria-metrics-package-deb-all: \
+        victoria-metrics-package-deb \
+        victoria-metrics-package-deb-arm64
+
+### Packaging as RPM - amd64
+victoria-metrics-package-rpm: victoria-metrics-prod
+	./package/package_rpm.sh amd64
+
+### Packaging as RPM - arm64
+victoria-metrics-package-rpm-arm64: victoria-metrics-arm64-prod
+	./package/package_rpm.sh arm64
+
+### Packaging as RPM - all
+victoria-metrics-package-rpm-all: \
+        victoria-metrics-package-rpm \
+        victoria-metrics-package-rpm-arm64
+
+### Packaging as both DEB and RPM - all
+victoria-metrics-package-deb-rpm-all: \
+        victoria-metrics-package-deb \
+        victoria-metrics-package-deb-arm64 \
+        victoria-metrics-package-rpm \
+        victoria-metrics-package-rpm-arm64

app/victoria-metrics/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8428
+
+ENTRYPOINT ["/victoria-metrics-prod"]
+ARG src_binary
+COPY $src_binary ./victoria-metrics-prod

app/victoria-metrics/main.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+var (
+	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
+		"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
+		"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
+	startTime := time.Now()
+	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
+	vmstorage.Init()
+	vmselect.Init()
+	vminsert.Init()
+	startSelfScraper()
+
+	go httpserver.Serve(*httpListenAddr, requestHandler)
+	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	stopSelfScraper()
+
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	startTime = time.Now()
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	vminsert.Stop()
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+
+	vmstorage.Stop()
+	vmselect.Stop()
+
+	fs.MustStopDirRemover()
+
+	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if vminsert.RequestHandler(w, r) {
+		return true
+	}
+	if vmselect.RequestHandler(w, r) {
+		return true
+	}
+	if vmstorage.RequestHandler(w, r) {
+		return true
+	}
+	return false
+}

app/victoria-metrics/main_test.go

@@ -0,0 +1,496 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+const (
+	testFixturesDir            = "testdata"
+	testStorageSuffix          = "vm-test-storage"
+	testHTTPListenAddr         = ":7654"
+	testStatsDListenAddr       = ":2003"
+	testOpenTSDBListenAddr     = ":4242"
+	testOpenTSDBHTTPListenAddr = ":4243"
+	testLogLevel               = "INFO"
+)
+
+const (
+	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+)
+
+const (
+	testStorageInitTimeout = 10 * time.Second
+)
+
+var (
+	storagePath   string
+	insertionTime = time.Now().UTC()
+)
+
+type test struct {
+	Name             string     `json:"name"`
+	Data             []string   `json:"data"`
+	Query            []string   `json:"query"`
+	ResultMetrics    []Metric   `json:"result_metrics"`
+	ResultSeries     Series     `json:"result_series"`
+	ResultQuery      Query      `json:"result_query"`
+	ResultQueryRange QueryRange `json:"result_query_range"`
+	Issue            string     `json:"issue"`
+}
+
+type Metric struct {
+	Metric     map[string]string `json:"metric"`
+	Values     []float64         `json:"values"`
+	Timestamps []int64           `json:"timestamps"`
+}
+
+func (r *Metric) UnmarshalJSON(b []byte) error {
+	type plain Metric
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+type Series struct {
+	Status string              `json:"status"`
+	Data   []map[string]string `json:"data"`
+}
+type Query struct {
+	Status string    `json:"status"`
+	Data   QueryData `json:"data"`
+}
+type QueryData struct {
+	ResultType string            `json:"resultType"`
+	Result     []QueryDataResult `json:"result"`
+}
+
+type QueryDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Value  []interface{}     `json:"value"`
+}
+
+func (r *QueryDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+type QueryRange struct {
+	Status string         `json:"status"`
+	Data   QueryRangeData `json:"data"`
+}
+type QueryRangeData struct {
+	ResultType string                 `json:"resultType"`
+	Result     []QueryRangeDataResult `json:"result"`
+}
+
+type QueryRangeDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Values [][]interface{}   `json:"values"`
+}
+
+func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryRangeDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+func TestMain(m *testing.M) {
+	setUp()
+	code := m.Run()
+	tearDown()
+	os.Exit(code)
+}
+
+func setUp() {
+	storagePath = filepath.Join(os.TempDir(), testStorageSuffix)
+	processFlags()
+	logger.Init()
+	vmstorage.InitWithoutMetrics()
+	vmselect.Init()
+	vminsert.Init()
+	go httpserver.Serve(*httpListenAddr, requestHandler)
+	readyStorageCheckFunc := func() bool {
+		resp, err := http.Get(testHealthHTTPPath)
+		if err != nil {
+			return false
+		}
+		resp.Body.Close()
+		return resp.StatusCode == 200
+	}
+	if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil {
+		log.Fatalf("http server can't start for %s seconds, err %s", testStorageInitTimeout, err)
+	}
+}
+
+func processFlags() {
+	envflag.Parse()
+	for _, fv := range []struct {
+		flag  string
+		value string
+	}{
+		{flag: "storageDataPath", value: storagePath},
+		{flag: "httpListenAddr", value: testHTTPListenAddr},
+		{flag: "graphiteListenAddr", value: testStatsDListenAddr},
+		{flag: "opentsdbListenAddr", value: testOpenTSDBListenAddr},
+		{flag: "loggerLevel", value: testLogLevel},
+		{flag: "opentsdbHTTPListenAddr", value: testOpenTSDBHTTPListenAddr},
+	} {
+		// panics if flag doesn't exist
+		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
+			log.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
+		}
+	}
+}
+
+func waitFor(timeout time.Duration, f func() bool) error {
+	fraction := timeout / 10
+	for i := fraction; i < timeout; i += fraction {
+		if f() {
+			return nil
+		}
+		time.Sleep(fraction)
+	}
+	return fmt.Errorf("timeout")
+}
+
+func tearDown() {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		log.Printf("cannot stop the webservice: %s", err)
+	}
+	vminsert.Stop()
+	vmstorage.Stop()
+	vmselect.Stop()
+	fs.MustRemoveAll(storagePath)
+}
+
+func TestWriteRead(t *testing.T) {
+	t.Run("write", testWrite)
+	time.Sleep(1 * time.Second)
+	vmstorage.Stop()
+	// open storage after stop in write
+	vmstorage.InitWithoutMetrics()
+	t.Run("read", testRead)
+}
+
+func testWrite(t *testing.T) {
+	t.Run("prometheus", func(t *testing.T) {
+		for _, test := range readIn("prometheus", t, insertionTime) {
+			s := newSuite(t)
+			r := testutil.WriteRequest{}
+			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
+			data, err := testutil.Compress(r)
+			s.greaterThan(len(r.Timeseries), 0)
+			if err != nil {
+				t.Errorf("error compressing %v %s", r, err)
+				t.Fail()
+			}
+			httpWrite(t, testPromWriteHTTPPath, bytes.NewBuffer(data))
+		}
+	})
+
+	t.Run("influxdb", func(t *testing.T) {
+		for _, x := range readIn("influxdb", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				httpWrite(t, testWriteHTTPPath, bytes.NewBufferString(strings.Join(test.Data, "\n")))
+			})
+		}
+	})
+	t.Run("graphite", func(t *testing.T) {
+		for _, x := range readIn("graphite", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				tcpWrite(t, "127.0.0.1"+testStatsDListenAddr, strings.Join(test.Data, "\n"))
+			})
+		}
+	})
+	t.Run("opentsdb", func(t *testing.T) {
+		for _, x := range readIn("opentsdb", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				tcpWrite(t, "127.0.0.1"+testOpenTSDBListenAddr, strings.Join(test.Data, "\n"))
+			})
+		}
+	})
+	t.Run("opentsdbhttp", func(t *testing.T) {
+		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				logger.Infof("writing %s", test.Data)
+				httpWrite(t, testOpenTSDBWriteHTTPPath, bytes.NewBufferString(strings.Join(test.Data, "\n")))
+			})
+		}
+	})
+}
+
+func testRead(t *testing.T) {
+	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+		t.Run(engine, func(t *testing.T) {
+			for _, x := range readIn(engine, t, insertionTime) {
+				test := x
+				t.Run(test.Name, func(t *testing.T) {
+					t.Parallel()
+					for _, q := range test.Query {
+						q = testutil.PopulateTimeTplString(q, insertionTime)
+						if test.Issue != "" {
+							test.Issue = "Regression in " + test.Issue
+						}
+						switch true {
+						case strings.HasPrefix(q, "/api/v1/export"):
+							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
+								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/series"):
+							s := Series{}
+							httpReadStruct(t, testReadHTTPPath, q, &s)
+							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
+								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/query_range"):
+							queryResult := QueryRange{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil {
+								t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/query"):
+							queryResult := Query{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
+								t.Fatalf("Query. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						default:
+							t.Fatalf("unsupported read query %s", q)
+						}
+					}
+				})
+			}
+		})
+	}
+}
+
+func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
+	t.Helper()
+	s := newSuite(t)
+	var tt []test
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Ext(path) != ".json" {
+			return nil
+		}
+		b, err := ioutil.ReadFile(path)
+		s.noError(err)
+		item := test{}
+		s.noError(json.Unmarshal(b, &item))
+		for i := range item.Data {
+			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
+		}
+		tt = append(tt, item)
+		return nil
+	}))
+	if len(tt) == 0 {
+		t.Fatalf("no test found in %s", filepath.Join(testFixturesDir, readFor))
+	}
+	return tt
+}
+
+func httpWrite(t *testing.T, address string, r io.Reader) {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Post(address, "", r)
+	s.noError(err)
+	s.noError(resp.Body.Close())
+	s.equalInt(resp.StatusCode, 204)
+}
+
+func tcpWrite(t *testing.T, address string, data string) {
+	t.Helper()
+	s := newSuite(t)
+	conn, err := net.Dial("tcp", address)
+	s.noError(err)
+	defer conn.Close()
+	n, err := conn.Write([]byte(data))
+	s.noError(err)
+	s.equalInt(n, len(data))
+}
+
+func httpReadMetrics(t *testing.T, address, query string) []Metric {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
+	defer resp.Body.Close()
+	s.equalInt(resp.StatusCode, 200)
+	var rows []Metric
+	for dec := json.NewDecoder(resp.Body); dec.More(); {
+		var row Metric
+		s.noError(dec.Decode(&row))
+		rows = append(rows, row)
+	}
+	return rows
+}
+func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
+	defer resp.Body.Close()
+	s.equalInt(resp.StatusCode, 200)
+	s.noError(json.NewDecoder(resp.Body).Decode(dst))
+}
+
+func checkMetricsResult(got, want []Metric) error {
+	for _, r := range append([]Metric(nil), got...) {
+		want = removeIfFoundMetrics(r, want)
+	}
+	if len(want) > 0 {
+		return fmt.Errorf("exptected metrics %+v not found in %+v", want, got)
+	}
+	return nil
+}
+
+func removeIfFoundMetrics(r Metric, contains []Metric) []Metric {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) &&
+			reflect.DeepEqual(r.Timestamps, item.Timestamps) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkSeriesResult(got, want Series) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	wantData := append([]map[string]string(nil), want.Data...)
+	for _, r := range got.Data {
+		wantData = removeIfFoundSeries(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected seria(s) %+v not found in %+v", wantData, got.Data)
+	}
+	return nil
+}
+
+func removeIfFoundSeries(r map[string]string, contains []map[string]string) []map[string]string {
+	for i, item := range contains {
+		if reflect.DeepEqual(r, item) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryResult(got, want Query) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Value[0], item.Value[0]) && reflect.DeepEqual(r.Value[1], item.Value[1]) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryRangeResult(got, want QueryRange) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryRangeDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryRangeData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+type suite struct{ t *testing.T }
+
+func newSuite(t *testing.T) *suite { return &suite{t: t} }
+
+func (s *suite) noError(err error) {
+	s.t.Helper()
+	if err != nil {
+		s.t.Errorf("unexpected error %v", err)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) equalInt(a, b int) {
+	s.t.Helper()
+	if a != b {
+		s.t.Errorf("%d not equal %d", a, b)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) greaterThan(a, b int) {
+	s.t.Helper()
+	if a <= b {
+		s.t.Errorf("%d less or equal then %d", a, b)
+		s.t.FailNow()
+	}
+}

app/victoria-metrics/self_scraper.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+var (
+	selfScrapeInterval = flag.Duration("selfScrapeInterval", 0, "Interval for self-scraping own metrics at /metrics page")
+	selfScrapeInstance = flag.String("selfScrapeInstance", "self", "Value for 'instance' label, which is added to self-scraped metrics")
+	selfScrapeJob      = flag.String("selfScrapeJob", "victoria-metrics", "Value for 'job' label, which is added to self-scraped metrics")
+)
+
+var selfScraperStopCh chan struct{}
+var selfScraperWG sync.WaitGroup
+
+func startSelfScraper() {
+	selfScraperStopCh = make(chan struct{})
+	selfScraperWG.Add(1)
+	go func() {
+		defer selfScraperWG.Done()
+		selfScraper(*selfScrapeInterval)
+	}()
+}
+
+func stopSelfScraper() {
+	close(selfScraperStopCh)
+	selfScraperWG.Wait()
+}
+
+func selfScraper(scrapeInterval time.Duration) {
+	if scrapeInterval <= 0 {
+		// Self-scrape is disabled.
+		return
+	}
+	logger.Infof("started self-scraping `/metrics` page with interval %.3f seconds", scrapeInterval.Seconds())
+
+	var bb bytesutil.ByteBuffer
+	var rows prometheus.Rows
+	var mrs []storage.MetricRow
+	var labels []prompb.Label
+	t := time.NewTicker(scrapeInterval)
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
+		bb.Reset()
+		httpserver.WritePrometheusMetrics(&bb)
+		s := bytesutil.ToUnsafeString(bb.B)
+		rows.Reset()
+		rows.Unmarshal(s)
+		mrs = mrs[:0]
+		for i := range rows.Rows {
+			r := &rows.Rows[i]
+			labels = labels[:0]
+			labels = addLabel(labels, "", r.Metric)
+			labels = addLabel(labels, "job", *selfScrapeJob)
+			labels = addLabel(labels, "instance", *selfScrapeInstance)
+			for j := range r.Tags {
+				t := &r.Tags[j]
+				labels = addLabel(labels, t.Key, t.Value)
+			}
+			if len(mrs) < cap(mrs) {
+				mrs = mrs[:len(mrs)+1]
+			} else {
+				mrs = append(mrs, storage.MetricRow{})
+			}
+			mr := &mrs[len(mrs)-1]
+			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
+			mr.Timestamp = currentTimestamp
+			mr.Value = r.Value
+		}
+		logger.Infof("writing %d rows at timestamp %d", len(mrs), currentTimestamp)
+		vmstorage.AddRows(mrs)
+	}
+}
+
+func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
+	if len(dst) < cap(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, prompb.Label{})
+	}
+	lb := &dst[len(dst)-1]
+	lb.Name = bytesutil.ToUnsafeBytes(key)
+	lb.Value = bytesutil.ToUnsafeBytes(value)
+	return dst
+}

app/victoria-metrics/test/parser.go

@@ -0,0 +1,52 @@
+package test
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var (
+	parseTimeExpRegex = regexp.MustCompile(`"?{TIME[^}]*}"?`)
+	extractRegex      = regexp.MustCompile(`"?{([^}]*)}"?`)
+)
+
+// PopulateTimeTplString substitutes {TIME_*} with t in s and returns the result.
+func PopulateTimeTplString(s string, t time.Time) string {
+	return string(PopulateTimeTpl([]byte(s), t))
+}
+
+// PopulateTimeTpl substitutes {TIME_*} with tGlobal in b and returns the result.
+func PopulateTimeTpl(b []byte, tGlobal time.Time) []byte {
+	return parseTimeExpRegex.ReplaceAllFunc(b, func(repl []byte) []byte {
+		t := tGlobal
+		repl = extractRegex.FindSubmatch(repl)[1]
+		parts := strings.SplitN(string(repl), "-", 2)
+		if len(parts) == 2 {
+			duration, err := time.ParseDuration(strings.TrimSpace(parts[1]))
+			if err != nil {
+				log.Fatalf("error %s parsing duration %s in %s", err, parts[1], repl)
+			}
+			t = t.Add(-duration)
+		}
+		switch strings.TrimSpace(parts[0]) {
+		case `TIME_S`:
+			return []byte(fmt.Sprintf("%d", t.Unix()))
+		case `TIME_MSZ`:
+			return []byte(fmt.Sprintf("%d", t.Unix()*1e3))
+		case `TIME_MS`:
+			return []byte(fmt.Sprintf("%d", timeToMillis(t)))
+		case `TIME_NS`:
+			return []byte(fmt.Sprintf("%d", t.UnixNano()))
+		default:
+			log.Fatalf("unknown time pattern %s in %s", parts[0], repl)
+		}
+		return repl
+	})
+}
+
+func timeToMillis(t time.Time) int64 {
+	return t.UnixNano() / 1e6
+}

app/victoria-metrics/test/parser_test.go

@@ -0,0 +1,24 @@
+package test
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPopulateTimeTplString(t *testing.T) {
+	now, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+	if err != nil {
+		t.Fatalf("unexpected error when parsing time: %s", err)
+	}
+	f := func(s, resultExpected string) {
+		t.Helper()
+		result := PopulateTimeTplString(s, now)
+		if result != resultExpected {
+			t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
+		}
+	}
+	f("", "")
+	f("{TIME_S}", "1136214245")
+	f("now: {TIME_S}, past 30s: {TIME_MS-30s}, now: {TIME_S}", "now: 1136214245, past 30s: 1136214215000, now: 1136214245")
+	f("now: {TIME_MS}, past 30m: {TIME_MSZ-30m}, past 2h: {TIME_NS-2h}", "now: 1136214245000, past 30m: 1136212445000, past 2h: 1136207045000000000")
+}

app/victoria-metrics/test/prom_types.go

@@ -0,0 +1,364 @@
+package test
+
+// Source https://github.com/prometheus/prometheus/blob/master/prompb/remote.pb.go . Code is copy pasted and cleaned up
+import (
+	"encoding/binary"
+	"math"
+	"math/bits"
+)
+
+// WriteRequest is write request
+type WriteRequest struct {
+	Timeseries []TimeSeries `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"`
+}
+
+// Size returns m size in bytes after marshaling.
+func (m *WriteRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Timeseries) > 0 {
+		for _, e := range m.Timeseries {
+			l = e.Size()
+			n += 1 + l + sovRemote(uint64(l))
+		}
+	}
+	return n
+}
+func sovRemote(x uint64) (n int) {
+	return (bits.Len64(x|1) + 6) / 7
+}
+
+// Marshal marshals m.
+func (m *WriteRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA
+func (m *WriteRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *WriteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Timeseries) > 0 {
+		for iNdEx := len(m.Timeseries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Timeseries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintRemote(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintRemote(dAtA []byte, offset int, v uint64) int {
+	offset -= sovRemote(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+
+// Sample is time series sample.
+type Sample struct {
+	Value     float64 `protobuf:"fixed64,1,opt,name=value,proto3" json:"value,omitempty"`
+	Timestamp int64   `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+}
+
+// Reset resets m.
+func (m *Sample) Reset() { *m = Sample{} }
+
+// TimeSeries represents samples and labels for a single time series.
+type TimeSeries struct {
+	Labels  []Label  `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
+	Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"`
+}
+
+// Reset resets m.
+func (m *TimeSeries) Reset() { *m = TimeSeries{} }
+
+// Label is time series label.
+type Label struct {
+	Name  string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+// Reset resets m.
+func (m *Label) Reset() { *m = Label{} }
+
+// Labels is a set of labels.
+type Labels struct {
+	Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
+}
+
+// Reset resets m.
+func (m *Labels) Reset() { *m = Labels{} }
+
+// Marshal marshals m.
+func (m *Sample) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *Sample) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if m.Timestamp != 0 {
+		i = encodeVarintTypes(dAtA, i, uint64(m.Timestamp))
+		i--
+		dAtA[i] = 0x10
+	}
+	if m.Value != 0 {
+		i -= 8
+		binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.Value))))
+		i--
+		dAtA[i] = 0x9
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal marshals m.
+func (m *TimeSeries) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *TimeSeries) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Samples) > 0 {
+		for iNdEx := len(m.Samples) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Samples[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Labels[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal marshals m.
+func (m *Label) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *Label) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Value) > 0 {
+		i -= len(m.Value)
+		copy(dAtA[i:], m.Value)
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Value)))
+		i--
+		dAtA[i] = 0x12
+	}
+	if len(m.Name) > 0 {
+		i -= len(m.Name)
+		copy(dAtA[i:], m.Name)
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Name)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal marshals m.
+func (m *Labels) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *Labels) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *Labels) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Labels) > 0 {
+		for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Labels[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
+	offset -= sovTypes(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+
+// Size returns the size of marshaled m.
+func (m *Sample) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	if m.Value != 0 {
+		n += 9
+	}
+	if m.Timestamp != 0 {
+		n += 1 + sovTypes(uint64(m.Timestamp))
+	}
+	return n
+}
+
+// Size returns the size of marshaled m.
+func (m *TimeSeries) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Labels) > 0 {
+		for _, e := range m.Labels {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	if len(m.Samples) > 0 {
+		for _, e := range m.Samples {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	return n
+}
+
+// Size returns the size of marshaled m.
+func (m *Label) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Name)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	l = len(m.Value)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	return n
+}
+
+// Size returns the size of marshaled m.
+func (m *Labels) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Labels) > 0 {
+		for _, e := range m.Labels {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	return n
+}
+
+func sovTypes(x uint64) (n int) {
+	return (bits.Len64(x|1) + 6) / 7
+}

app/victoria-metrics/test/prom_writter.go

@@ -0,0 +1,12 @@
+package test
+
+import "github.com/golang/snappy"
+
+// Compress marshals and compresses wr.
+func Compress(wr WriteRequest) ([]byte, error) {
+	data, err := wr.Marshal()
+	if err != nil {
+		return nil, err
+	}
+	return snappy.Encode(nil, data), nil
+}

app/victoria-metrics/testdata/graphite/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["graphite.foo.bar.baz;tag1=value1;tag2=value2 123 {TIME_S}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"graphite.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -0,0 +1,16 @@
+{
+  "name": "comparison-not-inf-not-nan",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150",
+  "data": [
+    "not_nan_not_inf;item=x 1 {TIME_S-1m}",
+    "not_nan_not_inf;item=x 1 {TIME_S-2m}",
+    "not_nan_not_inf;item=y 3 {TIME_S-1m}",
+    "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
+  "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"],["{TIME_S}","0.5"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -0,0 +1,16 @@
+{
+  "name": "empty-label-match",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/395",
+  "data": [
+    "empty_label_match 1 {TIME_S-1m}",
+    "empty_label_match;foo=bar 2 {TIME_S-1m}",
+    "empty_label_match;foo=baz 3 {TIME_S-1m}"],
+  "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"__name__":"empty_label_match"},"values":[["{TIME_S}","1"]]},
+	      {"metric":{"__name__":"empty_label_match","foo":"bar"},"values":[["{TIME_S}","2"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -0,0 +1,21 @@
+{
+  "name": "max_lookback_set",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_set 1 {TIME_S-30s}",
+    "max_lookback_set 2 {TIME_S-60s}",
+    "max_lookback_set 3 {TIME_S-120s}",
+    "max_lookback_set 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_set"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -0,0 +1,30 @@
+{
+  "name": "max_lookback_unset",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_unset 1 {TIME_S-30s}",
+    "max_lookback_unset 2 {TIME_S-60s}",
+    "max_lookback_unset 3 {TIME_S-120s}",
+    "max_lookback_unset 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-140s}","4"],
+	      ["{TIME_S-130s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-110s}","3"],
+	      ["{TIME_S-100s}","3"],
+	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-50s}","2"],
+	      ["{TIME_S-40s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"],
+	      ["{TIME_S-10s}","1"],
+	      ["{TIME_S-0s}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -0,0 +1,18 @@
+{
+  "name": "not-nan-as-missing-data",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153",
+  "data": [
+    "not_nan_as_missing_data;item=x 2 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=x 1 {TIME_S-1m}",
+    "not_nan_as_missing_data;item=y 4 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
+  ],
+  "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"x"},"values":[["{TIME_S-2m}","2"]]},
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"],["{TIME_S}","3"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -0,0 +1,14 @@
+{
+  "name": "subquery-aggregation",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/184",
+  "data": [
+    "forms_daily_count;item=x 1 {TIME_S-1m}",
+    "forms_daily_count;item=x 2 {TIME_S-2m}",
+    "forms_daily_count;item=y 3 {TIME_S-1m}",
+    "forms_daily_count;item=y 4 {TIME_S-2m}"],
+  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","1"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","3"]}]}
+  }
+}

app/victoria-metrics/testdata/influxdb/basic.json

@@ -0,0 +1,9 @@
+{
+  "name": "basic_insertion",
+  "data": ["measurement,tag1=value1,tag2=value2 field1=1.23,field2=123 {TIME_NS}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[1.23], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdb/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["put openstdb.foo.bar.baz {TIME_S} 123 tag1=value1 tag2=value2"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"openstdb.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["{\"metric\": \"opentsdbhttp.foo\", \"value\": 1001, \"timestamp\": {TIME_S}, \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.foo","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/multi_line.json

@@ -0,0 +1,9 @@
+{
+  "name": "multiline",
+  "data": ["[{\"metric\": \"opentsdbhttp.multiline1\", \"value\": 1001, \"timestamp\": \"{TIME_S}\", \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}, {\"metric\": \"opentsdbhttp.multiline2\", \"value\": 1002, \"timestamp\": {TIME_S}}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.multiline1","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]},
+    {"metric":{"__name__":"opentsdbhttp.multiline2"},"values":[1002], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.bar\"},{\"name\":\"baz\",\"value\":\"qux\"}],\"samples\":[{\"value\":100000,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.bar","baz":"qux"},"values":[100000], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/case-sensitive-regex.json

@@ -0,0 +1,10 @@
+{
+  "name": "case-sensitive-regex",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/161",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"sensitiveRegex\"}],\"samples\":[{\"value\":2,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"SensitiveRegex\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={label=~'(?i)sensitiveregex'}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"sensitiveRegex"},"values":[2], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"SensitiveRegex"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/duplicate-label.json

@@ -0,0 +1,9 @@
+{
+  "name": "duplicate_label",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/172",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.duplicate_label\"},{\"name\":\"duplicate\",\"value\":\"label\"},{\"name\":\"duplicate\",\"value\":\"label\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.duplicate_label","duplicate":"label"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/match-series.json

@@ -0,0 +1,15 @@
+{
+  "name": "match_series",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/155",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"1\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"2\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"3\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"4\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/series?match[]={__name__='MatchSeries'}", "/api/v1/series?match[]={__name__=~'MatchSeries.*'}"],
+  "result_series": {
+    "status": "success",
+    "data": [
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"1","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"2","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"3","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"4","TurbineType":"V112"}
+    ]
+  }
+}

app/vlagent/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlinsert/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlogscli/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlogsgenerator/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlselect/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlstorage/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vmagent/Makefile

@@ -12,35 +12,20 @@ vmagent-prod:
 vmagent-pure-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-pure
 
-vmagent-linux-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-linux-amd64
+vmagent-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
 
-vmagent-linux-arm-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm
+vmagent-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm
 
-vmagent-linux-arm64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm64
+vmagent-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
 
-vmagent-linux-ppc64le-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-linux-ppc64le
+vmagent-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
 
-vmagent-linux-386-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-linux-386
-
-vmagent-darwin-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
-
-vmagent-darwin-arm64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64
-
-vmagent-freebsd-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-freebsd-amd64
-
-vmagent-openbsd-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-openbsd-amd64
-
-vmagent-windows-amd64-prod:
-	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
+vmagent-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-386
 
 package-vmagent:
 	APP_NAME=vmagent $(MAKE) package-via-docker
@@ -73,41 +58,20 @@ run-vmagent:
 	APP_NAME=vmagent \
 	$(MAKE) run-via-docker
 
-vmagent-linux-amd64:
-	APP_NAME=vmagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+vmagent-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-amd64 ./app/vmagent
 
-vmagent-linux-arm:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+vmagent-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm ./app/vmagent
 
-vmagent-linux-arm64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+vmagent-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm64 ./app/vmagent
 
-vmagent-linux-ppc64le:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+vmagent-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-ppc64le ./app/vmagent
 
-vmagent-linux-s390x:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vmagent-linux-loong64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vmagent-linux-386:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vmagent-darwin-amd64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmagent-darwin-arm64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vmagent-freebsd-amd64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmagent-openbsd-amd64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmagent-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-goarch
+vmagent-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-386 ./app/vmagent
 
 vmagent-pure:
 	APP_NAME=vmagent $(MAKE) app-local-pure

app/vmagent/README.md

@@ -1,3 +1,273 @@
-See vmagent docs [here](https://docs.victoriametrics.com/victoriametrics/vmagent/).
+## vmagent
 
-vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmagent.md).
+`vmagent` is a tiny but brave agent, which helps you collect metrics from various sources
+and stores them in [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
+or any other Prometheus-compatible storage system that supports the `remote_write` protocol.
+
+<img alt="vmagent" src="vmagent.png">
+
+
+### Motivation
+
+While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
+and RAM friendly to scrape metrics from Prometheus-compatible exporters to VictoriaMetrics.
+Also, we found that users’ infrastructure are snowflakes - no two are alike, and we decided to add more flexibility
+to `vmagent` (like the ability to push metrics instead of pulling them). We did our best and plan to do even more.
+
+
+### Features
+
+* Can be used as drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
+  See [Quick Start](#quick-start) for details.
+* Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
+* Accepts data via all the ingestion protocols supported by VictoriaMetrics:
+  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
+  * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
+  * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
+* Can replicate collected metrics simultaneously to multiple remote storage systems.
+* Works in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
+  are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as connection
+  to remote storage is recovered. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+* Uses lower amounts of RAM, CPU, disk IO and network bandwidth compared to Prometheus.
+
+
+### Quick Start
+
+Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
+and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
+
+* `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. The `-remoteWrite.url` argument can be specified multiple times in order to replicate data concurrently to an arbitrary amount of remote storage systems.
+
+Example command line:
+
+```
+/path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+If you only need to collect Influx data, then the following is sufficient:
+
+```
+/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+
+`vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/tags).
+
+Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
+
+
+### Use cases
+
+
+#### IoT and Edge monitoring
+
+`vmagent` can run and collect metrics in IoT and industrial networks with unreliable or scheduled connections to the remote storage.
+It buffers the collected data in local files until the connection to remote storage becomes available and then sends the buffered
+data to the remote storage. It re-tries sending the data to remote storage on any errors.
+The maximum buffer size can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+
+`vmagent` works on various architectures from IoT world - 32-bit arm, 64-bit arm, ppc64, 386, amd64.
+See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/Makefile) for details.
+
+
+#### Drop-in replacement for Prometheus
+
+If you use Prometheus only for scraping metrics from various targets and forwarding these metrics to remote storage,
+then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such a setup.
+See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
+
+
+#### Replication and high availability
+
+`vmagent` replicates the collected metrics among multiple remote storage instances configured via `-remoteWrite.url` args.
+If a single remote storage instance temporarily is out of service, then the collected data remains available in another remote storage instances.
+`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
+Then it sends the buffered data to the remote storage in order to prevent data gaps in the remote storage.
+
+
+#### Relabeling and filtering
+
+`vmagent` can add, remove or update labels on the collected data before sending it to remote storage. Additionally,
+it can remove unwanted samples via Prometheus-like relabeling before sending the collected data to remote storage.
+See [these docs](#relabeling) for details.
+
+
+#### Splitting data streams among multiple systems
+
+`vmagent` supports splitting the collected data between muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
+which is applied independently for each configured `-remoteWrite.url` destination. For instance, it is possible to replicate or split
+data among long-term remote storage, short-term remote storage and real-time analytical system [built on top of Kafka](https://github.com/Telefonica/prometheus-kafka-adapter).
+Note that each destination can receive its own subset of the collected data thanks to per-destination relabeling via `-remoteWrite.urlRelabelConfig`.
+
+
+#### Prometheus remote_write proxy
+
+`vmagent` may be used as a proxy for Prometheus data sent via Prometheus `remote_write` protocol. It can accept data via `remote_write` API
+at `/api/v1/write` endpoint, apply relabeling and filtering and then proxy it to another `remote_write` systems.
+The `vmagent` can be configured to encrypt the incoming `remote_write` requests with `-tls*` command-line flags.
+Additionally, Basic Auth can be enabled for the incoming `remote_write` requests with `-httpAuth.*` command-line flags.
+
+
+
+### How to collect metrics in Prometheus format
+
+Pass the path to `prometheus.yml` to `-promscrape.config` command-line flag. `vmagent` takes into account the following
+sections from [Prometheus config file](https://prometheus.io/docs/prometheus/latest/configuration/configuration/):
+
+* `global`
+* `scrape_configs`
+
+All the other sections are ignored, including [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) section.
+Use `-remoteWrite.*` command-line flags instead for configuring remote write settings.
+
+The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported:
+
+* `static_configs` - for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
+* `file_sd_configs` - for scraping targets defined in external files aka file-based service discover.
+  See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
+* `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s).
+  See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
+* `ec2_sd_configs` - for scraping targets in Amazon EC2.
+  See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
+  `vmagent` doesn't support `role_arn` config param yet.
+* `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
+  See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
+  `vmagent` provides the following additional functionality for `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
+* `consul_sd_configs` - for scraping targets registered in Consul.
+  See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
+* `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
+  See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
+
+Note that `vmagent` doesn't support `refresh_interval` option these scrape configs. Use the corresponding `-promscrape.*CheckInterval`
+command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs`
+entries to 60s. Run `vmagent -help` in order to see default values for `-promscrape.*CheckInterval` flags.
+
+
+File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
+
+
+### Adding labels to metrics
+
+Labels can be added to metrics via the following mechanisms:
+
+* Via `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured in `-promscrape.config` file.
+* Via `-remoteWrite.label` command-line flag. These labels are added to all the collected metrics before sending them to `-remoteWrite.url`.
+
+
+### Relabeling
+
+`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
+Additionally it provides the following extra actions:
+
+* `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
+* `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
+* `keep_if_equal`: keeps the entry if all label values from `source_labels` are equal.
+* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
+
+The relabeling can be defined in the following places:
+
+* At `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels.
+* At `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`.
+* At `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage.
+* At `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`.
+
+Read more about relabeling in the following articles:
+
+* [Life of a label](https://www.robustperception.io/life-of-a-label)
+* [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
+* [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
+* [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
+* [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
+
+
+### Monitoring
+
+`vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. It is recommended setting up regular scraping of this page
+either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
+
+`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format.
+
+
+### Troubleshooting
+
+* It is recommended increasing the maximum number of open files in the system (`ulimit -n`) when scraping big number of targets,
+  since `vmagent` establishes at least a single TCP connection per each target.
+
+* When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
+  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
+
+* It is recommended to increase `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
+  and `vmagent_remotewrite_pending_data_bytes` metric exported at `http://vmagent-host:8429/metrics` page constantly grows.
+
+* `vmagent` buffers scraped data at `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
+  The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
+  If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
+
+* If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports.
+  Just add the following relabeling rule to `relabel_configs` section in order to filter out targets with unneeded ports:
+
+```yml
+- action: keep_if_equal
+  source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number]
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmagent` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmagent` from the root folder of the repository.
+   It builds `vmagent` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmagent-prod` from the root folder of the repository.
+   It builds `vmagent-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
+
+By default the image is built on top of [alpine](https://hub.docker.com/_/alpine) image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
+
+```bash
+ROOT_IMAGE=scratch make package-vmagent
+```
+
+
+### Profiling
+
+`vmagent` provides handlers for collecting the following [Go profiles](https://blog.golang.org/profiling-go-programs):
+
+* Memory profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmagent-host>:8429/debug/pprof/heap > mem.pprof
+```
+
+* CPU profile. It can be collected with the following command:
+
+```bash
+curl -s http://<vmagent-host>:8429/debug/pprof/profile > cpu.pprof
+```
+
+The command for collecting CPU profile waits for 30 seconds before returning.
+
+The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).

app/vmagent/common/push_ctx.go

@@ -1,31 +1,39 @@
 package common
 
 import (
+	"runtime"
 	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // PushCtx is a context used for populating WriteRequest.
 type PushCtx struct {
-	// WriteRequest contains the WriteRequest, which must be pushed later to remote storage.
-	//
-	// The actual labels and samples for the time series are stored in Labels and Samples fields.
-	WriteRequest prompb.WriteRequest
+	WriteRequest prompbmarshal.WriteRequest
 
 	// Labels contains flat list of all the labels used in WriteRequest.
-	Labels []prompb.Label
+	Labels []prompbmarshal.Label
 
 	// Samples contains flat list of all the samples used in WriteRequest.
-	Samples []prompb.Sample
+	Samples []prompbmarshal.Sample
 }
 
 // Reset resets ctx.
 func (ctx *PushCtx) Reset() {
-	ctx.WriteRequest.Reset()
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
 
-	promrelabel.CleanLabels(ctx.Labels)
+	labels := ctx.Labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
 	ctx.Labels = ctx.Labels[:0]
 
 	ctx.Samples = ctx.Samples[:0]
@@ -35,10 +43,15 @@ func (ctx *PushCtx) Reset() {
 //
 // Call PutPushCtx when the ctx is no longer needed.
 func GetPushCtx() *PushCtx {
-	if v := pushCtxPool.Get(); v != nil {
-		return v.(*PushCtx)
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
 	}
-	return &PushCtx{}
 }
 
 // PutPushCtx returns ctx to the pool.
@@ -46,7 +59,12 @@ func GetPushCtx() *PushCtx {
 // ctx mustn't be used after returning to the pool.
 func PutPushCtx(ctx *PushCtx) {
 	ctx.Reset()
-	pushCtxPool.Put(ctx)
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
 }
 
 var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/csvimport/request_handler.go

@@ -5,33 +5,25 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="csvimport"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
 )
 
 // InsertHandler processes csv data from req.
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	return stream.Parse(req, func(rows []csvimport.Row) error {
-		return insertRows(at, rows, extraLabels)
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
 	})
 }
 
-func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompb.Label) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -41,23 +33,22 @@ func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompb.Label
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})
@@ -65,13 +56,8 @@ func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompb.Label
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(len(rows))
-	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil
 }

app/vmagent/datadogsketches/request_handler.go

@@ -1,95 +0,0 @@
-package datadogsketches
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
-		return insertRows(at, sketches, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for _, sketch := range sketches {
-		ms := sketch.ToSummary()
-		for _, m := range ms {
-			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
-				Name:  "__name__",
-				Value: m.Name,
-			})
-			for _, label := range m.Labels {
-				labels = append(labels, prompb.Label{
-					Name:  label.Name,
-					Value: label.Value,
-				})
-			}
-			for _, tag := range sketch.Tags {
-				name, value := datadogutil.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
-				labels = append(labels, prompb.Label{
-					Name:  name,
-					Value: value,
-				})
-			}
-			labels = append(labels, extraLabels...)
-			samplesLen := len(samples)
-			for _, p := range m.Points {
-				samples = append(samples, prompb.Sample{
-					Timestamp: p.Timestamp,
-					Value:     p.Value,
-				})
-			}
-			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompb.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[samplesLen:],
-			})
-		}
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/datadogv1/request_handler.go

@@ -1,99 +0,0 @@
-package datadogv1
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv1"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv1"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv1"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(series []datadogv1.Series) error {
-		return insertRows(at, series, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range series {
-		ss := &series[i]
-		rowsTotal += len(ss.Points)
-		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
-			Name:  "__name__",
-			Value: ss.Metric,
-		})
-		if ss.Host != "" {
-			labels = append(labels, prompb.Label{
-				Name:  "host",
-				Value: ss.Host,
-			})
-		}
-		if ss.Device != "" {
-			labels = append(labels, prompb.Label{
-				Name:  "device",
-				Value: ss.Device,
-			})
-		}
-		for _, tag := range ss.Tags {
-			name, value := datadogutil.SplitTag(tag)
-			if name == "host" {
-				name = "exported_host"
-			}
-			labels = append(labels, prompb.Label{
-				Name:  name,
-				Value: value,
-			})
-		}
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		for _, pt := range ss.Points {
-			samples = append(samples, prompb.Sample{
-				Timestamp: pt.Timestamp(),
-				Value:     pt.Value(),
-			})
-		}
-		tssDst = append(tssDst, prompb.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/datadogv2/request_handler.go

@@ -1,102 +0,0 @@
-package datadogv2
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv2"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv2"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv2"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/v2/series request.
-//
-// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ct := req.Header.Get("Content-Type")
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, ct, func(series []datadogv2.Series) error {
-		return insertRows(at, series, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range series {
-		ss := &series[i]
-		rowsTotal += len(ss.Points)
-		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
-			Name:  "__name__",
-			Value: ss.Metric,
-		})
-		for _, rs := range ss.Resources {
-			labels = append(labels, prompb.Label{
-				Name:  rs.Type,
-				Value: rs.Name,
-			})
-		}
-		if ss.SourceTypeName != "" {
-			labels = append(labels, prompb.Label{
-				Name:  "source_type_name",
-				Value: ss.SourceTypeName,
-			})
-		}
-		for _, tag := range ss.Tags {
-			name, value := datadogutil.SplitTag(tag)
-			if name == "host" {
-				name = "exported_host"
-			}
-			labels = append(labels, prompb.Label{
-				Name:  name,
-				Value: value,
-			})
-		}
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		for _, pt := range ss.Points {
-			samples = append(samples, prompb.Sample{
-				Timestamp: pt.Timestamp * 1000,
-				Value:     pt.Value,
-			})
-		}
-		tssDst = append(tssDst, prompb.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 8429
 
 ENTRYPOINT ["/vmagent-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./vmagent-prod

app/vmagent/graphite/request_handler.go

@@ -5,10 +5,9 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -21,12 +20,12 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, "", func(rows []parser.Row) error {
-		return insertRows(nil, rows)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
 	})
 }
 
-func insertRows(at *auth.Token, rows []parser.Row) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -36,22 +35,22 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})
@@ -59,9 +58,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -4,63 +4,52 @@ import (
 	"flag"
 	"io"
 	"net/http"
+	"runtime"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metric name if InfluxDB line contains only a single field")
-	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
-	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="influx"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
 )
 
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
-	return stream.Parse(r, encoding, true, "", "", func(db string, rows []influx.Row) error {
-		return insertRows(at, db, rows, nil)
+func InsertHandlerForReader(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, false, "", "", insertRows)
 	})
 }
 
 // InsertHandlerForHTTP processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	q := req.URL.Query()
-	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-	db := q.Get("db")
-	encoding := req.Header.Get("Content-Encoding")
-	isStreamMode := req.Header.Get("Stream-Mode") == "1"
-	return stream.Parse(req.Body, encoding, isStreamMode, precision, db, func(db string, rows []influx.Row) error {
-		return insertRows(at, db, rows, extraLabels)
+func InsertHandlerForHTTP(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, insertRows)
 	})
 }
 
-func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prompb.Label) error {
+func insertRows(db string, rows []parser.Row) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 
@@ -72,32 +61,26 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 	buf := ctx.buf[:0]
 	for i := range rows {
 		r := &rows[i]
-		rowsTotal += len(r.Fields)
 		commonLabels = commonLabels[:0]
-		hasDBKey := false
+		hasDBLabel := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			if tag.Key == *dbLabel {
-				hasDBKey = true
+			if tag.Key == "db" {
+				hasDBLabel = true
 			}
-			commonLabels = append(commonLabels, prompb.Label{
+			commonLabels = append(commonLabels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		if len(db) > 0 && !hasDBKey {
-			commonLabels = append(commonLabels, prompb.Label{
-				Name:  *dbLabel,
+		if len(db) > 0 && !hasDBLabel {
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  "db",
 				Value: db,
 			})
 		}
-		commonLabels = append(commonLabels, extraLabels...)
-		ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
-		if !*skipMeasurement {
-			ctx.metricGroupBuf = append(ctx.metricGroupBuf, r.Measurement...)
-		}
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1139
-		skipFieldKey := len(r.Measurement) > 0 && len(r.Fields) == 1 && *skipSingleField
+		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
 		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
 			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
 		}
@@ -110,33 +93,29 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 			}
 			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
 			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: metricGroup,
 			})
 			labels = append(labels, commonLabels...)
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Timestamp: r.Timestamp,
 				Value:     f.Value,
 			})
-			tssDst = append(tssDst, prompb.TimeSeries{
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
 				Labels:  labels[labelsLen:],
 				Samples: samples[len(samples)-1:],
 			})
 		}
+		rowsTotal += len(r.Fields)
 	}
 	ctx.buf = buf
 	ctx.ctx.WriteRequest.Timeseries = tssDst
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	if !remotewrite.TryPush(at, &ctx.ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
 	rowsPerInsert.Update(float64(rowsTotal))
 
 	return nil
@@ -144,7 +123,7 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 
 type pushCtx struct {
 	ctx            common.PushCtx
-	commonLabels   []prompb.Label
+	commonLabels   []prompbmarshal.Label
 	metricGroupBuf []byte
 	buf            []byte
 }
@@ -152,23 +131,37 @@ type pushCtx struct {
 func (ctx *pushCtx) reset() {
 	ctx.ctx.Reset()
 
-	promrelabel.CleanLabels(ctx.commonLabels)
-	ctx.commonLabels = ctx.commonLabels[:0]
+	commonLabels := ctx.commonLabels
+	for i := range commonLabels {
+		label := &commonLabels[i]
+		label.Name = ""
+		label.Value = ""
+	}
 
 	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
 	ctx.buf = ctx.buf[:0]
 }
 
 func getPushCtx() *pushCtx {
-	if v := pushCtxPool.Get(); v != nil {
-		return v.(*pushCtx)
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
 	}
-	return &pushCtx{}
 }
 
 func putPushCtx(ctx *pushCtx) {
 	ctx.reset()
-	pushCtxPool.Put(ctx)
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
 }
 
 var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/main.go

@@ -1,40 +1,24 @@
 package main
 
 import (
-	"embed"
 	"flag"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutil"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
@@ -42,46 +26,22 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
-		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
-		"See also -influxListenAddr.useProxyProtocol")
-	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. "+
-		"See also -graphiteListenAddr.useProxyProtocol")
-	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
-		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
-	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
-		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
-	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
-		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
-		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
-		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 0, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . See also -promscrape.config.dryRun")
 )
 
 var (
@@ -91,97 +51,63 @@ var (
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
 
-var (
-	//go:embed static
-	staticFiles  embed.FS
-	staticServer = http.FileServer(http.FS(staticFiles))
-)
-
 func main() {
-	// vmagent is optimized for reduced memory allocations,
-	// so it can run with the reduced GOGC in order to reduce the used memory,
-	// while keeping CPU usage spent in GC at low levels.
-	//
-	// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
-	// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
-	// exceeds 0.05 for extended periods of time.
-	cgroup.SetGOGC(50)
-
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 
-	if promscrape.IsDryRun() {
-		if err := promscrape.CheckConfig(); err != nil {
-			logger.Fatalf("error when checking -promscrape.config: %s", err)
-		}
-		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
-		return
-	}
 	if *dryRun {
-		if err := promscrape.CheckConfig(); err != nil {
-			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		if err := flag.Set("promscrape.config.strictParse", "true"); err != nil {
+			logger.Panicf("BUG: cannot set promscrape.config.strictParse=true: %s", err)
 		}
 		if err := remotewrite.CheckRelabelConfigs(); err != nil {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
-		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking Prometheus config: %s", err)
 		}
-		logger.Infof("all the configs are ok; exiting with 0 status code")
+		logger.Infof("all the configs are ok; exitting with 0 status code")
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8429"}
-	}
-	logger.Infof("starting vmagent at %q...", listenAddrs)
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
-	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
-	protoparserutil.StartUnmarshalWorkers()
+	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, "")
-		})
+		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
 	}
 	if len(*graphiteListenAddr) > 0 {
-		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
 	}
 
-	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
+	promscrape.Init(remotewrite.Push)
 
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, requestHandler)
+	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
-	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	remotewrite.StopIngestionRateLimiter()
-	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -197,495 +123,68 @@ func main() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
-	protoparserutil.StopUnmarshalWorkers()
 	remotewrite.Stop()
 
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
 }
 
-func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
-	if !remotewrite.MultitenancyEnabled() {
-		return func(req *http.Request) error {
-			path := strings.ReplaceAll(req.URL.Path, "//", "/")
-			if path != "/api/put" {
-				return fmt.Errorf("unsupported path requested: %q; expecting '/api/put'", path)
-			}
-			return opentsdbhttp.InsertHandler(nil, req)
-		}
-	}
-	return func(req *http.Request) error {
-		path := strings.ReplaceAll(req.URL.Path, "//", "/")
-		at, err := getAuthTokenFromPath(path)
-		if err != nil {
-			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
-		}
-		return opentsdbhttp.InsertHandler(at, req)
-	}
-}
-
-func getAuthTokenFromPath(path string) (*auth.Token, error) {
-	p, err := httpserver.ParsePath(path)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse multitenant path: %w", err)
-	}
-	if p.Prefix != "insert" {
-		return nil, fmt.Errorf(`unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
-	}
-	if p.Suffix != "opentsdb/api/put" {
-		return nil, fmt.Errorf("unsupported path requested: %q; expecting 'opentsdb/api/put'", p.Suffix)
-	}
-	return auth.NewTokenPossibleMultitenant(p.AuthToken)
-}
-
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
-			return false
-		}
-		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, "<h2>vmagent</h2>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints:</br>")
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"targets", "status for discovered active targets"},
-			{"service-discovery", "labels before and after relabeling for discovered targets"},
-			{"metric-relabel-debug", "debug metric relabeling"},
-			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
-			{"config", "-promscrape.config contents"},
-			{"metrics", "available service metrics"},
-			{"flags", "command-line flags"},
-			{"-/reload", "reload configuration"},
-		})
-		return true
-	}
-
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
-	if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus") || strings.HasPrefix(path, "/api/v1/import/prometheus") {
-		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(nil, r); err != nil {
-			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		statusCode := http.StatusNoContent
-		if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus/metrics/job/") ||
-			strings.HasPrefix(path, "/api/v1/import/prometheus/metrics/job/") {
-			// Return 200 status code for pushgateway requests.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
-			statusCode = http.StatusOK
-		}
-		w.WriteHeader(statusCode)
-		return true
-	}
-	if strings.HasPrefix(path, "/datadog/") {
-		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
-		path = strings.TrimSuffix(path, "/")
-	}
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	switch path {
-	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
-		if protoparserutil.HandleVMProtoServerHandshake(w, r) {
-			return true
-		}
+	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := promremotewrite.InsertHandler(nil, r); err != nil {
+		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import", "/api/v1/import":
+	case "/api/v1/import":
 		vmimportRequests.Inc()
-		if err := vmimport.InsertHandler(nil, r); err != nil {
+		if err := vmimport.InsertHandler(r); err != nil {
 			vmimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import/csv", "/api/v1/import/csv":
+	case "/api/v1/import/csv":
 		csvimportRequests.Inc()
-		if err := csvimport.InsertHandler(nil, r); err != nil {
+		if err := csvimport.InsertHandler(r); err != nil {
 			csvimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import/native", "/api/v1/import/native":
-		nativeimportRequests.Inc()
-		if err := native.InsertHandler(nil, r); err != nil {
-			nativeimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	case "/influx/write", "/influx/api/v2/write", "/write", "/api/v2/write":
+	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
-		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
+		if err := influx.InsertHandlerForHTTP(r); err != nil {
 			influxWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/influx/query", "/query":
+	case "/query":
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
 		influxQueryRequests.Inc()
-		influxutil.WriteDatabaseNames(w)
+		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
 		return true
-	case "/influx/health":
-		influxHealthRequests.Inc()
-		influxutil.WriteHealthCheckResponse(w)
-		return true
-	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(nil, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "/newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "/newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(nil, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv2WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
-	case "/datadog/api/v1/validate":
-		datadogValidateRequests.Inc()
-		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, `{"valid":true}`)
-		return true
-	case "/datadog/api/v1/check_run":
-		datadogCheckRunRequests.Inc()
-		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/datadog/intake":
-		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, `{}`)
-		return true
-	case "/datadog/api/v1/metadata":
-		datadogMetadataRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, `{}`)
-		return true
-	case "/prometheus/targets", "/targets":
+	case "/targets":
 		promscrapeTargetsRequests.Inc()
-		promscrape.WriteHumanReadableTargetsStatus(w, r)
+		w.Header().Set("Content-Type", "text/plain")
+		promscrape.WriteHumanReadableTargetsStatus(w)
 		return true
-	case "/prometheus/service-discovery", "/service-discovery":
-		promscrapeServiceDiscoveryRequests.Inc()
-		promscrape.WriteServiceDiscovery(w, r)
-		return true
-	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
-		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
-		return true
-	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
-		promscrapeTargetRelabelDebugRequests.Inc()
-		promscrape.WriteTargetRelabelDebug(w, r)
-		return true
-	case "/prometheus/api/v1/targets", "/api/v1/targets":
-		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		// https://prometheus.io/docs/prometheus/latest/querying/api/#targets
-		state := r.FormValue("state")
-		scrapePool := r.FormValue("scrapePool")
-		promscrape.WriteAPIV1Targets(w, state, scrapePool)
-		return true
-	case "/prometheus/target_response", "/target_response":
-		promscrapeTargetResponseRequests.Inc()
-		if err := promscrape.WriteTargetResponse(w, r); err != nil {
-			promscrapeTargetResponseErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		return true
-	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
-			return true
-		}
-		promscrapeConfigRequests.Inc()
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		promscrape.WriteConfigData(w)
-		return true
-	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
-		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
-			return true
-		}
-		promscrapeStatusConfigRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		var bb bytesutil.ByteBuffer
-		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
-		return true
-	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
-			return true
-		}
+	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
-	case "/ready":
-		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
-			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
-			http.Error(w, errMsg, http.StatusTooEarly)
-		} else {
-			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-			w.WriteHeader(http.StatusOK)
-			w.Write([]byte("OK"))
-		}
-		return true
-	default:
-		if strings.HasPrefix(r.URL.Path, "/static") {
-			staticServer.ServeHTTP(w, r)
-			return true
-		}
-		if remotewrite.MultitenancyEnabled() {
-			return processMultitenantRequest(w, r, path)
-		}
-		return false
-	}
-}
-
-func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
-	p, err := httpserver.ParsePath(path)
-	if err != nil {
-		// Cannot parse multitenant path. Skip it - probably it will be parsed later.
-		return false
-	}
-	if p.Prefix != "insert" {
-		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
-		return true
-	}
-	at, err := auth.NewTokenPossibleMultitenant(p.AuthToken)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
-		return true
-	}
-	if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus") {
-		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(at, r); err != nil {
-			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		statusCode := http.StatusNoContent
-		if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus/metrics/job/") {
-			// Return 200 status code for pushgateway requests.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
-			statusCode = http.StatusOK
-		}
-		w.WriteHeader(statusCode)
-		return true
-	}
-	if strings.HasPrefix(p.Suffix, "datadog/") {
-		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
-		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
-	}
-	switch p.Suffix {
-	case "prometheus/", "prometheus", "prometheus/api/v1/write", "prometheus/api/v1/push":
-		prometheusWriteRequests.Inc()
-		if err := promremotewrite.InsertHandler(at, r); err != nil {
-			prometheusWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	case "prometheus/api/v1/import":
-		vmimportRequests.Inc()
-		if err := vmimport.InsertHandler(at, r); err != nil {
-			vmimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	case "prometheus/api/v1/import/csv":
-		csvimportRequests.Inc()
-		if err := csvimport.InsertHandler(at, r); err != nil {
-			csvimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	case "prometheus/api/v1/import/native":
-		nativeimportRequests.Inc()
-		if err := native.InsertHandler(at, r); err != nil {
-			nativeimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	case "influx/write", "influx/api/v2/write":
-		influxWriteRequests.Inc()
-		if err := influx.InsertHandlerForHTTP(at, r); err != nil {
-			influxWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	case "influx/query":
-		influxQueryRequests.Inc()
-		influxutil.WriteDatabaseNames(w)
-		return true
-	case "influx/health":
-		influxHealthRequests.Inc()
-		influxutil.WriteHealthCheckResponse(w)
-		return true
-	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(at, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(at, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv2WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
-	case "datadog/api/v1/validate":
-		datadogValidateRequests.Inc()
-		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, `{"valid":true}`)
-		return true
-	case "datadog/api/v1/check_run":
-		datadogCheckRunRequests.Inc()
-		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "datadog/intake":
-		datadogIntakeRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, `{}`)
-		return true
-	case "datadog/api/v1/metadata":
-		datadogMetadataRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, `{}`)
-		return true
-	default:
-		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
-		return true
 	}
+	return false
 }
 
 var (
@@ -698,54 +197,12 @@ var (
 	csvimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
 	csvimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
 
-	prometheusimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
-	prometheusimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
 
-	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
-	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
 
-	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
-	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
-
-	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
-	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
-
-	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-
-	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-
-	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-
-	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
-	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
-	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
-	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
-
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-
-	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-
-	newrelicInventoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/inventory/deltas", protocol="newrelic"}`)
-	newrelicCheckRequest      = metrics.NewCounter(`vm_http_requests_total{path="/newrelic", protocol="newrelic"}`)
-
-	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
-	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
-
-	promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/metric-relabel-debug"}`)
-	promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target-relabel-debug"}`)
-
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
-
-	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
-	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
-
-	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
-	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)
+	promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
@@ -754,7 +211,10 @@ func usage() {
 	const s = `
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
 
-See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md .
 `
-	flagutil.Usage(s)
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
 }

app/vmagent/multiarch/Dockerfile

@@ -1,13 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 8429
-ENTRYPOINT ["/vmagent-prod"]
-ARG TARGETARCH
-ARG BINARY_SUFFIX=non-existing
-COPY vmagent-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmagent-prod

app/vmagent/native/request_handler.go

@@ -1,91 +0,0 @@
-package native
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="native"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
-)
-
-// InsertHandler processes `/api/v1/import` request.
-//
-// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(block *stream.Block) error {
-		return insertRows(at, block, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	// Update rowsInserted and rowsPerInsert before actual inserting,
-	// since relabeling can prevent from inserting the rows.
-	rowsLen := len(block.Values)
-	rowsInserted.Add(rowsLen)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsLen)
-	}
-	rowsPerInsert.Update(float64(rowsLen))
-
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	mn := &block.MetricName
-	labelsLen := len(labels)
-	labels = append(labels, prompb.Label{
-		Name:  "__name__",
-		Value: bytesutil.ToUnsafeString(mn.MetricGroup),
-	})
-	for j := range mn.Tags {
-		tag := &mn.Tags[j]
-		labels = append(labels, prompb.Label{
-			Name:  bytesutil.ToUnsafeString(tag.Key),
-			Value: bytesutil.ToUnsafeString(tag.Value),
-		})
-	}
-	labels = append(labels, extraLabels...)
-	values := block.Values
-	timestamps := block.Timestamps
-	if len(timestamps) != len(values) {
-		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
-	}
-	samplesLen := len(samples)
-	for j, value := range values {
-		samples = append(samples, prompb.Sample{
-			Value:     value,
-			Timestamp: timestamps[j],
-		})
-	}
-	tssDst = append(tssDst, prompb.TimeSeries{
-		Labels:  labels[labelsLen:],
-		Samples: samples[samplesLen:],
-	})
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	return nil
-}

app/vmagent/newrelic/request_handler.go

@@ -1,87 +0,0 @@
-package newrelic
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="newrelic"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="newrelic"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="newrelic"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(rows []newrelic.Row) error {
-		return insertRows(at, rows, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	samplesCount := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		tags := r.Tags
-		srcSamples := r.Samples
-		for j := range srcSamples {
-			s := &srcSamples[j]
-			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
-				Name:  "__name__",
-				Value: bytesutil.ToUnsafeString(s.Name),
-			})
-			for k := range tags {
-				t := &tags[k]
-				labels = append(labels, prompb.Label{
-					Name:  bytesutil.ToUnsafeString(t.Key),
-					Value: bytesutil.ToUnsafeString(t.Value),
-				})
-			}
-			samples = append(samples, prompb.Sample{
-				Value:     s.Value,
-				Timestamp: r.Timestamp,
-			})
-			tssDst = append(tssDst, prompb.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[len(samples)-1:],
-			})
-			labels = append(labels, extraLabels...)
-		}
-		samplesCount += len(srcSamples)
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(samplesCount)
-	}
-	rowsPerInsert.Update(float64(samplesCount))
-	return nil
-}

app/vmagent/opentelemetry/request_handler.go

@@ -1,99 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="opentelemetry"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="opentelemetry"}`)
-	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
-)
-
-// InsertHandler processes opentelemetry metrics.
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	encoding := req.Header.Get("Content-Encoding")
-	var processBody func([]byte) ([]byte, error)
-	if req.Header.Get("Content-Type") == "application/json" {
-		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
-			processBody = firehose.ProcessRequestBody
-		} else {
-			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-		}
-	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
-		return insertRows(at, tss, mms, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range tss {
-		ts := &tss[i]
-		rowsTotal += len(ts.Samples)
-		labelsLen := len(labels)
-		labels = append(labels, ts.Labels...)
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		samples = append(samples, ts.Samples...)
-		tssDst = append(tssDst, prompb.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-
-	var metadataTotal int
-	if promscrape.IsMetadataEnabled() {
-		var accountID, projectID uint32
-		if at != nil {
-			accountID = at.AccountID
-			projectID = at.ProjectID
-			for i := range mms {
-				mm := &mms[i]
-				mm.AccountID = accountID
-				mm.ProjectID = projectID
-			}
-		}
-		ctx.WriteRequest.Metadata = mms
-		metadataTotal = len(mms)
-	}
-
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	metadataInserted.Add(metadataTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-		metadataTenantInserted.Get(at).Add(metadataTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/opentsdb/request_handler.go

@@ -5,9 +5,9 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,7 +20,9 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, insertRows)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
 }
 
 func insertRows(rows []parser.Row) error {
@@ -33,22 +35,22 @@ func insertRows(rows []parser.Row) error {
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})
@@ -56,9 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(nil, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/opentsdbhttp/request_handler.go

@@ -5,11 +5,9 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,17 +18,13 @@ var (
 
 // InsertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	return stream.Parse(req, func(rows []opentsdbhttp.Row) error {
-		return insertRows(at, rows, extraLabels)
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
 	})
 }
 
-func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompb.Label) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -40,23 +34,22 @@ func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompb.La
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})
@@ -64,9 +57,7 @@ func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompb.La
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -1,110 +0,0 @@
-package prometheusimport
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="prometheus"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="prometheus"}`)
-
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
-)
-
-// InsertHandler processes `/api/v1/import/prometheus` request.
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	defaultTimestamp, err := protoparserutil.GetTimestamp(req)
-	if err != nil {
-		return err
-	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, promscrape.IsMetadataEnabled(), func(rows []prometheus.Row, mms []prometheus.Metadata) error {
-		return insertRows(at, rows, mms, extraLabels)
-	}, func(s string) {
-		httpserver.LogError(req, s)
-	})
-}
-
-func insertRows(at *auth.Token, rows []prometheus.Row, mms []prometheus.Metadata, extraLabels []prompb.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	mmsDst := ctx.WriteRequest.Metadata[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
-			Name:  "__name__",
-			Value: r.Metric,
-		})
-		for j := range r.Tags {
-			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
-				Name:  tag.Key,
-				Value: tag.Value,
-			})
-		}
-		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
-			Value:     r.Value,
-			Timestamp: r.Timestamp,
-		})
-		tssDst = append(tssDst, prompb.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[len(samples)-1:],
-		})
-	}
-	var accountID, projectID uint32
-	if at != nil {
-		accountID = at.AccountID
-		projectID = at.ProjectID
-	}
-	for i := range mms {
-		mm := &mms[i]
-		mmsDst = append(mmsDst, prompb.MetricMetadata{
-			MetricFamilyName: mm.Metric,
-			Help:             mm.Help,
-			Type:             mm.Type,
-			// there is no unit in Prometheus exposition formats
-
-			AccountID: accountID,
-			ProjectID: projectID,
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.WriteRequest.Metadata = mmsDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	metadataInserted.Add(len(mms))
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(len(rows))
-		metadataTenantInserted.Get(at).Add(len(mms))
-	}
-	rowsPerInsert.Update(float64(len(rows)))
-	return nil
-}

app/vmagent/prometheusimport/request_handler_test.go

@@ -1,60 +0,0 @@
-package prometheusimport
-
-import (
-	"bytes"
-	"flag"
-	"log"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	srv        *httptest.Server
-	testOutput *bytes.Buffer
-)
-
-func TestInsertHandler(t *testing.T) {
-	setUp()
-	defer tearDown()
-	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
-go_memstats_alloc_bytes_total 1`))
-	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unexpected error %s", err)
-	}
-	expectedMsg := "cannot unmarshal Prometheus line"
-	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
-	}
-}
-
-func setUp() {
-	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(204)
-	}))
-	flag.Parse()
-	remoteWriteFlag := "remoteWrite.url"
-	if err := flag.Lookup(remoteWriteFlag).Value.Set(srv.URL); err != nil {
-		log.Fatalf("unable to set %q with value %q, err: %v", remoteWriteFlag, srv.URL, err)
-	}
-	logger.Init()
-	protoparserutil.StartUnmarshalWorkers()
-	remotewrite.Init()
-	testOutput = &bytes.Buffer{}
-	logger.SetOutputForTests(testOutput)
-}
-
-func tearDown() {
-	protoparserutil.StopUnmarshalWorkers()
-	srv.Close()
-	logger.ResetOutputForTest()
-	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()
-	fs.MustRemoveDir(tmpDataDir)
-
-}

app/vmagent/promremotewrite/request_handler.go

@@ -5,105 +5,63 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="promremotewrite"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="promremotewrite"}`)
-	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
-		return insertRows(at, tss, mms, extraLabels)
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
 	})
 }
 
-func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
+func insertRows(timeseries []prompb.TimeSeries) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
 	rowsTotal := 0
 	tssDst := ctx.WriteRequest.Timeseries[:0]
-	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range timeseries {
 		ts := &timeseries[i]
-		rowsTotal += len(ts.Samples)
 		labelsLen := len(labels)
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
-			labels = append(labels, prompb.Label{
-				Name:  label.Name,
-				Value: label.Value,
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
 			})
 		}
-		labels = append(labels, extraLabels...)
 		samplesLen := len(samples)
 		for i := range ts.Samples {
 			sample := &ts.Samples[i]
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Value:     sample.Value,
 				Timestamp: sample.Timestamp,
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
+		rowsTotal += len(ts.Samples)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
-
-	var metadataTotal int
-	if promscrape.IsMetadataEnabled() {
-		var accountID, projectID uint32
-		if at != nil {
-			accountID = at.AccountID
-			projectID = at.ProjectID
-		}
-		for i := range mms {
-			mm := &mms[i]
-			mmsDst = append(mmsDst, prompb.MetricMetadata{
-				MetricFamilyName: mm.MetricFamilyName,
-				Help:             mm.Help,
-				Type:             mm.Type,
-				Unit:             mm.Unit,
-
-				AccountID: accountID,
-				ProjectID: projectID,
-			})
-		}
-		ctx.WriteRequest.Metadata = mmsDst
-		metadataTotal = len(mms)
-	}
-
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-		metadataTenantInserted.Get(at).Add(metadataTotal)
-	}
-	metadataInserted.Add(metadataTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmagent/remotewrite/client.go

@@ -1,210 +1,140 @@
 package remotewrite
 
 import (
-	"bytes"
-	"errors"
+	"crypto/tls"
+	"encoding/base64"
+	"flag"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/fasthttp"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 var (
-	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
-	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
+	sendTimeout = flag.Duration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to -remoteWrite.url")
 
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
-		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
-		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
-	sendTimeout      = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
-	// deprecated in the future. use -remoteWrite.retryMaxInterval instead
-	retryMaxTime     = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. This flag is deprecated, use -remoteWrite.retryMaxInterval instead")
-	retryMaxInterval = flagutil.NewArrayDuration("remoteWrite.retryMaxInterval", time.Minute, "The maximum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url.  The delay doubles with each retry until this maximum is reached, after which it remains constant. See also -remoteWrite.retryMinInterval")
-	proxyURL         = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
-		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
+	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsCertFile           = flagutil.NewArray("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsKeyFile = flagutil.NewArray("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsCAFile = flagutil.NewArray("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
+		"By default system CA is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	tlsServerName = flagutil.NewArray("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
-	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
-	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
-		"to the corresponding -remoteWrite.url")
-	tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
-	tlsCAFile  = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
-		"By default, system CA is used")
-	tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
-		"By default, the server name from -remoteWrite.url is used")
-
-	headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
-		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
-		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
-
-	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
-	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
-	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
-		"The file is re-read every second")
-	bearerToken     = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
-	bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
-		"The token is re-read from the file every second")
-
-	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
-	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
-	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
-	oauth2EndpointParams   = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
-		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
-	oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
-	oauth2Scopes   = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
-
-	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
-		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
-	awsEC2Endpoint = flagutil.NewArrayString("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsSTSEndpoint = flagutil.NewArrayString("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsRegion      = flagutil.NewArrayString("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsRoleARN     = flagutil.NewArrayString("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsAccessKey   = flagutil.NewArrayString("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsService     = flagutil.NewArrayString("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
-		"Defaults to \"aps\"")
-	awsSecretKey = flagutil.NewArrayString("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	basicAuthUsername = flagutil.NewArray("remoteWrite.basicAuth.username", "Optional basic auth username to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	basicAuthPassword = flagutil.NewArray("remoteWrite.basicAuth.password", "Optional basic auth password to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
+	bearerToken = flagutil.NewArray("remoteWrite.bearerToken", "Optional bearer auth token to use for -remoteWrite.url. "+
+		"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
 )
 
 type client struct {
-	sanitizedURL   string
+	urlLabelValue  string
 	remoteWriteURL string
+	host           string
+	requestURI     string
+	authHeader     string
+	fq             *persistentqueue.FastQueue
+	hc             *fasthttp.HostClient
 
-	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
-	useVMProto          atomic.Bool
-	canDowngradeVMProto atomic.Bool
-
-	fq *persistentqueue.FastQueue
-	hc *http.Client
-
-	retryMinInterval time.Duration
-	retryMaxInterval time.Duration
-
-	sendBlock func(block []byte) bool
-	authCfg   *promauth.Config
-	awsCfg    *awsapi.Config
-
-	rl *ratelimiter.RateLimiter
-
-	bytesSent       *metrics.Counter
-	blocksSent      *metrics.Counter
 	requestDuration *metrics.Histogram
 	requestsOKCount *metrics.Counter
 	errorsCount     *metrics.Counter
-	packetsDropped  *metrics.Counter
-	rateLimit       *metrics.Gauge
 	retriesCount    *metrics.Counter
-	sendDuration    *metrics.FloatCounter
 
 	wg     sync.WaitGroup
 	stopCh chan struct{}
 }
 
-func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
-	authCfg, err := getAuthConfig(argIdx)
-	if err != nil {
-		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+func newClient(argIdx int, remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue, concurrency int) *client {
+	authHeader := ""
+	username := basicAuthUsername.GetOptionalArg(argIdx)
+	password := basicAuthPassword.GetOptionalArg(argIdx)
+	if len(username) > 0 || len(password) > 0 {
+		// See https://en.wikipedia.org/wiki/Basic_access_authentication
+		token := username + ":" + password
+		token64 := base64.StdEncoding.EncodeToString([]byte(token))
+		authHeader = "Basic " + token64
 	}
-	awsCfg, err := getAWSAPIConfig(argIdx)
-	if err != nil {
-		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
-	}
-
-	tr := httputil.NewTransport(false, "vmagent_remotewrite")
-	tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
-	tr.MaxConnsPerHost = 2 * concurrency
-	tr.MaxIdleConnsPerHost = 2 * concurrency
-	tr.IdleConnTimeout = time.Minute
-	tr.WriteBufferSize = 64 * 1024
-
-	pURL := proxyURL.GetOptionalArg(argIdx)
-	if len(pURL) > 0 {
-		if !strings.Contains(pURL, "://") {
-			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
+	token := bearerToken.GetOptionalArg(argIdx)
+	if len(token) > 0 {
+		if authHeader != "" {
+			logger.Fatalf("`-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", token)
 		}
-		pu, err := url.Parse(pURL)
+		authHeader = "Bearer " + token
+	}
+
+	readTimeout := *sendTimeout
+	if readTimeout <= 0 {
+		readTimeout = time.Minute
+	}
+	writeTimeout := readTimeout
+	var u fasthttp.URI
+	u.Update(remoteWriteURL)
+	scheme := string(u.Scheme())
+	switch scheme {
+	case "http", "https":
+	default:
+		logger.Fatalf("unsupported scheme in -remoteWrite.url=%q: %q. It must be http or https", remoteWriteURL, scheme)
+	}
+	host := string(u.Host())
+	if len(host) == 0 {
+		logger.Fatalf("invalid -remoteWrite.url=%q: host cannot be empty. Make sure the url looks like `http://host:port/path`", remoteWriteURL)
+	}
+	requestURI := string(u.RequestURI())
+	isTLS := scheme == "https"
+	var tlsCfg *tls.Config
+	if isTLS {
+		var err error
+		tlsCfg, err = getTLSConfig(argIdx)
 		if err != nil {
-			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
+			logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
 		}
-		tr.Proxy = http.ProxyURL(pu)
 	}
-	hc := &http.Client{
-		Transport: authCfg.NewRoundTripper(tr),
-		Timeout:   sendTimeout.GetOptionalArg(argIdx),
+	if !strings.Contains(host, ":") {
+		if isTLS {
+			host += ":443"
+		} else {
+			host += ":80"
+		}
 	}
-	retryMaxIntervalFlag := retryMaxTime
-	if retryMaxInterval.String() != "" {
-		retryMaxIntervalFlag = retryMaxInterval
+	maxConns := 2 * concurrency
+	hc := &fasthttp.HostClient{
+		Addr:                host,
+		Name:                "vmagent",
+		Dial:                statDial,
+		IsTLS:               isTLS,
+		TLSConfig:           tlsCfg,
+		MaxConns:            maxConns,
+		MaxIdleConnDuration: 10 * readTimeout,
+		ReadTimeout:         readTimeout,
+		WriteTimeout:        writeTimeout,
+		MaxResponseBodySize: 1024 * 1024,
 	}
 	c := &client{
-		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
-		authCfg:          authCfg,
-		awsCfg:           awsCfg,
-		fq:               fq,
-		hc:               hc,
-		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
-		retryMaxInterval: retryMaxIntervalFlag.GetOptionalArg(argIdx),
-		stopCh:           make(chan struct{}),
+		urlLabelValue:  urlLabelValue,
+		remoteWriteURL: remoteWriteURL,
+		host:           host,
+		requestURI:     requestURI,
+		authHeader:     authHeader,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
 	}
-	c.sendBlock = c.sendBlockHTTP
-
-	useVMProto := forceVMProto.GetOptionalArg(argIdx)
-	usePromProto := forcePromProto.GetOptionalArg(argIdx)
-	if useVMProto && usePromProto {
-		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
-	}
-	if !useVMProto && !usePromProto {
-		// The VM protocol could be downgraded later at runtime if unsupported media type response status is received.
-		useVMProto = true
-		c.canDowngradeVMProto.Store(true)
-	}
-	c.useVMProto.Store(useVMProto)
-
-	return c
-}
-
-func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
-	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
-	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
-		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
-		c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
-	}
-	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
-	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
-	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(rateLimit.GetOptionalArg(argIdx))
-	})
-	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
-	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
-	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
-	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
-	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
-	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
-	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(*queues)
-	})
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
 	for i := 0; i < concurrency; i++ {
 		c.wg.Add(1)
 		go func() {
@@ -212,389 +142,133 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 			c.runWorker()
 		}()
 	}
-	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	return c
 }
 
 func (c *client) MustStop() {
 	close(c.stopCh)
 	c.wg.Wait()
-	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
 }
 
-func getAuthConfig(argIdx int) (*promauth.Config, error) {
-	headersValue := headers.GetOptionalArg(argIdx)
-	var hdrs []string
-	if headersValue != "" {
-		hdrs = strings.Split(headersValue, "^^")
-	}
-	username := basicAuthUsername.GetOptionalArg(argIdx)
-	password := basicAuthPassword.GetOptionalArg(argIdx)
-	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
-	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
-		basicAuthCfg = &promauth.BasicAuthConfig{
-			Username:     username,
-			Password:     promauth.NewSecret(password),
-			PasswordFile: passwordFile,
-		}
-	}
-
-	token := bearerToken.GetOptionalArg(argIdx)
-	tokenFile := bearerTokenFile.GetOptionalArg(argIdx)
-
-	var oauth2Cfg *promauth.OAuth2Config
-	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
-	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
-	if clientSecretFile != "" || clientSecret != "" {
-		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
-		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
-		}
-		oauth2Cfg = &promauth.OAuth2Config{
-			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
-			ClientSecret:     promauth.NewSecret(clientSecret),
-			ClientSecretFile: clientSecretFile,
-			EndpointParams:   endpointParams,
-			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
-			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
-		}
-	}
-
-	tlsCfg := &promauth.TLSConfig{
+func getTLSConfig(argIdx int) (*tls.Config, error) {
+	tlsConfig := &promauth.TLSConfig{
 		CAFile:             tlsCAFile.GetOptionalArg(argIdx),
 		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
 		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
 		ServerName:         tlsServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
+		InsecureSkipVerify: *tlsInsecureSkipVerify,
 	}
-
-	opts := &promauth.Options{
-		BasicAuth:       basicAuthCfg,
-		BearerToken:     token,
-		BearerTokenFile: tokenFile,
-		OAuth2:          oauth2Cfg,
-		TLSConfig:       tlsCfg,
-		Headers:         hdrs,
-	}
-	authCfg, err := opts.NewConfig()
+	cfg, err := promauth.NewConfig(".", nil, "", "", tlsConfig)
 	if err != nil {
-		return nil, fmt.Errorf("cannot populate auth config for remoteWrite idx: %d, err: %w", argIdx, err)
+		return nil, fmt.Errorf("cannot populate TLS config: %w", err)
 	}
-	return authCfg, nil
-}
-
-func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
-	if !awsUseSigv4.GetOptionalArg(argIdx) {
-		return nil, nil
-	}
-	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
-	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
-	region := awsRegion.GetOptionalArg(argIdx)
-	roleARN := awsRoleARN.GetOptionalArg(argIdx)
-	accessKey := awsAccessKey.GetOptionalArg(argIdx)
-	secretKey := awsSecretKey.GetOptionalArg(argIdx)
-	service := awsService.GetOptionalArg(argIdx)
-	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
-	if err != nil {
-		return nil, err
-	}
-	return cfg, nil
+	tlsCfg := cfg.NewTLSConfig()
+	return tlsCfg, nil
 }
 
 func (c *client) runWorker() {
 	var ok bool
 	var block []byte
-	ch := make(chan bool, 1)
+	ch := make(chan struct{})
 	for {
 		block, ok = c.fq.MustReadBlock(block[:0])
 		if !ok {
 			return
 		}
-		if len(block) == 0 {
-			// skip empty data blocks from sending
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241
-			continue
-		}
 		go func() {
-			startTime := time.Now()
-			ch <- c.sendBlock(block)
-			c.sendDuration.Add(time.Since(startTime).Seconds())
+			c.sendBlock(block)
+			ch <- struct{}{}
 		}()
 		select {
-		case ok := <-ch:
-			if ok {
-				// The block has been sent successfully
-				continue
-			}
-			// Return unsent block to the queue.
-			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
-			return
+		case <-ch:
+			// The block has been sent successfully
+			continue
 		case <-c.stopCh:
 			// c must be stopped. Wait for a while in the hope the block will be sent.
 			graceDuration := 5 * time.Second
 			select {
-			case ok := <-ch:
-				if !ok {
-					// Return unsent block to the queue.
-					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
-				}
+			case <-ch:
+				// The block has been sent successfully.
 			case <-time.After(graceDuration):
-				// Return unsent block to the queue.
-				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
+					len(block), c.remoteWriteURL, graceDuration.Seconds())
 			}
 			return
 		}
 	}
 }
 
-func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
-	req, err := c.newRequest(url, body)
-	if err != nil {
-		return nil, err
+func (c *client) sendBlock(block []byte) {
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(c.requestURI)
+	req.SetHost(c.host)
+	req.Header.SetMethod("POST")
+	req.Header.Add("Content-Type", "application/x-protobuf")
+	req.Header.Add("Content-Encoding", "snappy")
+	req.Header.Add("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.authHeader != "" {
+		req.Header.Set("Authorization", c.authHeader)
 	}
-	resp, err := c.hc.Do(req)
-	if err == nil {
-		return resp, nil
-	}
-	if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
-		return nil, err
-	}
-	// It is likely connection became stale or timed out during the first request.
-	// Make another attempt in hope request will succeed.
-	// If not, the error should be handled by the caller as usual.
-	// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
-	req, err = c.newRequest(url, body)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	resp, err = c.hc.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	return resp, nil
-}
+	req.SetBody(block)
 
-func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
-	reqBody := bytes.NewBuffer(body)
-	req, err := http.NewRequest(http.MethodPost, url, reqBody)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
-	}
-	err = c.authCfg.SetHeaders(req, true)
-	if err != nil {
-		return nil, err
-	}
-	h := req.Header
-	h.Set("User-Agent", "vmagent")
-	h.Set("Content-Type", "application/x-protobuf")
-	if encoding.IsZstd(body) {
-		h.Set("Content-Encoding", "zstd")
-		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
-	} else {
-		h.Set("Content-Encoding", "snappy")
-		h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	}
-	if c.awsCfg != nil {
-		sigv4Hash := awsapi.HashHex(body)
-		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
-			return nil, fmt.Errorf("cannot sign remoteWrite request with AWS sigv4: %w", err)
-		}
-	}
-	return req, nil
-}
-
-// sendBlockHTTP sends the given block to c.remoteWriteURL.
-//
-// The function returns false only if c.stopCh is closed.
-// Otherwise, it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxInterval)
-	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
-	retriesCount := 0
+	retryDuration := time.Second
+	resp := fasthttp.AcquireResponse()
 
 again:
+	select {
+	case <-c.stopCh:
+		fasthttp.ReleaseRequest(req)
+		fasthttp.ReleaseResponse(resp)
+		return
+	default:
+	}
+
 	startTime := time.Now()
-	resp, err := c.doRequest(c.remoteWriteURL, block)
+	err := doRequestWithPossibleRetry(c.hc, req, resp)
 	c.requestDuration.UpdateDuration(startTime)
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > maxRetryDuration {
-			retryDuration = maxRetryDuration
-		}
-		remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
-			len(block), c.sanitizedURL, err, retryDuration.Seconds())
-		t := timerpool.Get(retryDuration)
-		select {
-		case <-c.stopCh:
-			timerpool.Put(t)
-			return false
-		case <-t.C:
-			timerpool.Put(t)
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
 		}
+		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
+		time.Sleep(retryDuration)
 		c.retriesCount.Inc()
 		goto again
 	}
-
-	statusCode := resp.StatusCode
-	if statusCode/100 == 2 {
-		_ = resp.Body.Close()
-		c.requestsOKCount.Inc()
-		c.bytesSent.Add(len(block))
-		c.blocksSent.Inc()
-		return true
-	}
-
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
-	switch statusCode {
-	case 409:
-		logBlockRejected(block, c.sanitizedURL, resp)
-
-		// Just drop block on 409 status code like Prometheus does.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
-		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
-		_ = resp.Body.Close()
-		c.packetsDropped.Inc()
-		return true
-		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
-		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
-		// - Real-world implementations of v1 use both 400 and 415 status codes.
-		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
-	case 415, 400:
-		if encoding.IsZstd(block) {
-			logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
-				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-
-			zstdBlockLen := len(block)
-			block, err = repackBlockFromZstdToSnappy(block)
-			if err == nil {
-				if c.canDowngradeVMProto.Swap(false) {
-					logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
-						"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-					c.useVMProto.Store(false)
-				}
-
-				c.retriesCount.Inc()
-				_ = resp.Body.Close()
-				goto again
-			}
-
-			logger.Warnf("failed to repack zstd block (%s bytes) to snappy: %s; The block will be rejected. "+
-				"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
-				zstdBlockLen, err)
+	statusCode := resp.StatusCode()
+	if statusCode/100 != 2 {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
 		}
-
-		// Just drop snappy blocks on 400 or 415 status codes like Prometheus does.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
-		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
-		logBlockRejected(block, c.sanitizedURL, resp)
-		_ = resp.Body.Close()
-		c.packetsDropped.Inc()
-		return true
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q: %d; response body=%q; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, statusCode, resp.Body(), retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
 	}
+	c.requestsOKCount.Inc()
 
-	// Unexpected status code returned
-	retriesCount++
-	retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
-	retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
-
-	// Handle response
-	body, err := io.ReadAll(resp.Body)
-	_ = resp.Body.Close()
-	if err != nil {
-		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
-	} else {
-		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
-			"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
-	}
-	t := timerpool.Get(retryDuration)
-	select {
-	case <-c.stopCh:
-		timerpool.Put(t)
-		return false
-	case <-t.C:
-		timerpool.Put(t)
-	}
-	c.retriesCount.Inc()
-	goto again
+	// The block has been successfully sent to the remote storage.
+	fasthttp.ReleaseResponse(resp)
+	fasthttp.ReleaseRequest(req)
 }
 
-var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
-var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
-
-// getRetryDuration returns retry duration.
-// retryAfterDuration has the highest priority.
-// If retryAfterDuration is not specified, retryDuration gets doubled.
-// retryDuration can't exceed maxRetryDuration.
-//
-// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
-func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
-	// retryAfterDuration has the highest priority duration
-	if retryAfterDuration > 0 {
-		return timeutil.AddJitterToDuration(retryAfterDuration)
+func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response) error {
+	// There is no need in calling DoTimeout, since the timeout must be already set in hc.ReadTimeout.
+	err := hc.Do(req, resp)
+	if err == nil {
+		return nil
 	}
-
-	// default backoff retry policy
-	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
+	if err != fasthttp.ErrConnectionClosed {
+		return err
 	}
-
-	return retryDuration
-}
-
-// repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
-//
-// The input block may be corrupted, for example, if vmagent was shut down ungracefully and
-// failed to properly update the persisted queue files. In such cases, zstd decompression
-// will fail and an error will be returned.
-//
-// For more details, see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9417
-func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
-	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
-	if err != nil {
-		return nil, fmt.Errorf("zstd: decompress: %s", err)
-	}
-
-	return snappy.Encode(nil, plainBlock), nil
-}
-
-func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
-			"failed to read response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, err)
-	} else {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, string(body))
-	}
-}
-
-// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
-// retryAfterString should be in either HTTP-date or a number of seconds.
-// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
-func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
-	if retryAfterString == "" {
-		return retryAfterDuration
-	}
-
-	defer func() {
-		v := retryAfterDuration.Seconds()
-		logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
-	}()
-
-	// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
-	if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
-		return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
-	}
-	// Retry-After could be in seconds.
-	if seconds, err := strconv.Atoi(retryAfterString); err == nil {
-		return time.Duration(seconds) * time.Second
-	}
-
-	return 0
+	// Retry request if the server closed the keep-alive connection during the first attempt.
+	return hc.Do(req, resp)
 }

app/vmagent/remotewrite/client_test.go

@@ -1,132 +0,0 @@
-package remotewrite
-
-import (
-	"math"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/golang/snappy"
-)
-
-func TestCalculateRetryDuration(t *testing.T) {
-	// `testFunc` call `calculateRetryDuration` for `n` times
-	// and evaluate if the result of `calculateRetryDuration` is
-	// 1. >= expectMinDuration
-	// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
-	f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
-		t.Helper()
-
-		for i := 0; i < n; i++ {
-			retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
-		}
-
-		expectMaxDuration := helper(expectMinDuration)
-		expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
-
-		if retryDuration < expectMinDuration || retryDuration > expectMaxDuration {
-			t.Fatalf(
-				"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
-				expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
-				retryDuration.Milliseconds(),
-			)
-		}
-	}
-
-	// Call calculateRetryDuration for 1 time.
-	{
-		// default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-		// default backoff policy exceed max limit"
-		f(0, 10*time.Minute, 1, time.Minute)
-
-		// retry after > default backoff policy
-		f(10*time.Second, 1*time.Second, 1, 10*time.Second)
-		// retry after < default backoff policy
-		f(1*time.Second, 10*time.Second, 1, 1*time.Second)
-		// retry after invalid and < default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-
-	}
-
-	// Call calculateRetryDuration for multiple times.
-	{
-		// default backoff policy 2 times
-		f(0, time.Second, 2, 4*time.Second)
-		// default backoff policy 3 times
-		f(0, time.Second, 3, 8*time.Second)
-		// default backoff policy N times exceed max limit
-		f(0, time.Second, 10, time.Minute)
-
-		// retry after 120s 1 times
-		f(120*time.Second, time.Second, 1, 120*time.Second)
-		// retry after 120s 2 times
-		f(120*time.Second, time.Second, 2, 120*time.Second)
-	}
-}
-
-func TestParseRetryAfterHeader(t *testing.T) {
-	f := func(retryAfterString string, expectResult time.Duration) {
-		t.Helper()
-
-		result := parseRetryAfterHeader(retryAfterString)
-		// expect `expectResult == result` when retryAfterString is in seconds or invalid
-		// expect the difference between result and expectResult to be lower than 10%
-		if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
-			t.Fatalf(
-				"incorrect retry after duration, want (ms): %d, got (ms): %d",
-				expectResult.Milliseconds(), result.Milliseconds(),
-			)
-		}
-	}
-
-	// retry after header in seconds
-	f("10", 10*time.Second)
-	// retry after header in date time
-	f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
-	// retry after header invalid
-	f("invalid-retry-after", 0)
-	// retry after header not in GMT
-	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
-}
-
-// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
-func helper(d time.Duration) time.Duration {
-	dv := d / 10
-	if dv > 10*time.Second {
-		dv = 10 * time.Second
-	}
-
-	return d + dv
-}
-
-func TestRepackBlockFromZstdToSnappy(t *testing.T) {
-	expectedPlainBlock := []byte(`foobar`)
-
-	zstdBlock := encoding.CompressZSTDLevel(nil, expectedPlainBlock, 1)
-	snappyBlock, err := repackBlockFromZstdToSnappy(zstdBlock)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-
-	actualPlainBlock, err := snappy.Decode(nil, snappyBlock)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-
-	if string(actualPlainBlock) != string(expectedPlainBlock) {
-		t.Fatalf("unexpected plain block; got %q; want %q", actualPlainBlock, expectedPlainBlock)
-	}
-}
-
-func TestRepackBlockFromZstdToSnappyInvalidBlock(t *testing.T) {
-	snappyBlock, err := repackBlockFromZstdToSnappy([]byte("invalid zstd block"))
-
-	if err == nil {
-		t.Fatalf("expected error for invalid zstd block; got nil")
-	}
-	if len(snappyBlock) != 0 {
-		t.Fatalf("expected empty snappy block; got %d bytes", len(snappyBlock))
-	}
-}

app/vmagent/remotewrite/pendingseries.go

@@ -3,36 +3,27 @@ package remotewrite
 import (
 	"flag"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
 
 var (
 	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
-		"This option takes effect only when less than -remoteWrite.maxRowsPerBlock data points per -remoteWrite.flushInterval are pushed to -remoteWrite.url")
-	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
-	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
-	maxMetadataPerBlock  = flag.Int("remoteWrite.maxMetadataPerBlock", 5000, "The maximum number of metadata to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
-	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
-		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
-		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
+		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage. "+
+		"Minimum supported interval is 1 second")
+	maxUnpackedBlockSize = flag.Int("remoteWrite.maxBlockSize", 32*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
+		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
 )
 
+// the maximum number of rows to send per each block.
+const maxRowsPerBlock = 10000
+
 type pendingSeries struct {
 	mu sync.Mutex
 	wr writeRequest
@@ -41,12 +32,9 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(pushBlock func(block []byte)) *pendingSeries {
 	var ps pendingSeries
-	ps.wr.fq = fq
-	ps.wr.isVMRemoteWrite = isVMRemoteWrite
-	ps.wr.significantFigures = significantFigures
-	ps.wr.roundDigits = roundDigits
+	ps.wr.pushBlock = pushBlock
 	ps.stopCh = make(chan struct{})
 	ps.periodicFlusherWG.Add(1)
 	go func() {
@@ -61,18 +49,10 @@ func (ps *pendingSeries) MustStop() {
 	ps.periodicFlusherWG.Wait()
 }
 
-func (ps *pendingSeries) TryPushTimeSeries(tss []prompb.TimeSeries) bool {
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
 	ps.mu.Lock()
-	ok := ps.wr.tryPushTimeSeries(tss)
+	ps.wr.push(tss)
 	ps.mu.Unlock()
-	return ok
-}
-
-func (ps *pendingSeries) TryPushMetadata(mms []prompb.MetricMetadata) bool {
-	ps.mu.Lock()
-	ok := ps.wr.tryPushMetadata(mms)
-	ps.mu.Unlock()
-	return ok
 }
 
 func (ps *pendingSeries) periodicFlusher() {
@@ -80,348 +60,140 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	d := timeutil.AddJitterToDuration(*flushInterval)
-	ticker := time.NewTicker(d)
+	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
-	for {
+	mustStop := false
+	for !mustStop {
 		select {
 		case <-ps.stopCh:
-			ps.mu.Lock()
-			ps.wr.mustFlushOnStop()
-			ps.mu.Unlock()
-			return
+			mustStop = true
 		case <-ticker.C:
-			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime.Load() < uint64(flushSeconds) {
+			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime < uint64(flushSeconds) {
 				continue
 			}
 		}
 		ps.mu.Lock()
-		_ = ps.wr.tryFlush()
+		ps.wr.flush()
 		ps.mu.Unlock()
 	}
 }
 
 type writeRequest struct {
-	lastFlushTime atomic.Uint64
+	wr            prompbmarshal.WriteRequest
+	pushBlock     func(block []byte)
+	lastFlushTime uint64
 
-	// The queue to send blocks to.
-	fq *persistentqueue.FastQueue
+	tss []prompbmarshal.TimeSeries
 
-	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	isVMRemoteWrite *atomic.Bool
-
-	// How many significant figures must be left before sending the writeRequest to fq.
-	significantFigures int
-
-	// How many decimal digits after point must be left before sending the writeRequest to fq.
-	roundDigits int
-
-	wr prompb.WriteRequest
-
-	tss     []prompb.TimeSeries
-	mms     []prompb.MetricMetadata
-	labels  []prompb.Label
-	samples []prompb.Sample
-
-	// buf holds labels data
-	buf []byte
-	// metadatabuf holds metadata data
-	metadatabuf []byte
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
+	buf     []byte
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are reused.
-
 	wr.wr.Timeseries = nil
-	wr.wr.Metadata = nil
 
-	clear(wr.tss)
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
 	wr.tss = wr.tss[:0]
 
-	clear(wr.mms)
-	wr.mms = wr.mms[:0]
-
-	promrelabel.CleanLabels(wr.labels)
+	for i := range wr.labels {
+		label := &wr.labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
 	wr.buf = wr.buf[:0]
-	wr.metadatabuf = wr.metadatabuf[:0]
 }
 
-// mustFlushOnStop force pushes wr data into wr.fq
-//
-// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
-func (wr *writeRequest) mustFlushOnStop() {
+func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
-	wr.wr.Metadata = wr.mms
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
-		logger.Panicf("BUG: final flush must always return true")
-	}
+	wr.lastFlushTime = fasttime.UnixTimestamp()
+	pushWriteRequest(&wr.wr, wr.pushBlock)
 	wr.reset()
 }
 
-func (wr *writeRequest) mustWriteBlock(block []byte) bool {
-	wr.fq.MustWriteBlockIgnoreDisabledPQ(block)
-	return true
-}
-
-func (wr *writeRequest) tryFlush() bool {
-	wr.wr.Timeseries = wr.tss
-	wr.wr.Metadata = wr.mms
-	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
-		return false
-	}
-	wr.reset()
-	return true
-}
-
-func adjustSampleValues(samples []prompb.Sample, significantFigures, roundDigits int) {
-	if n := significantFigures; n > 0 {
-		for i := range samples {
-			s := &samples[i]
-			s.Value = decimal.RoundToSignificantFigures(s.Value, n)
-		}
-	}
-	if n := roundDigits; n < 100 {
-		for i := range samples {
-			s := &samples[i]
-			s.Value = decimal.RoundToDecimalDigits(s.Value, n)
-		}
-	}
-}
-
-func (wr *writeRequest) tryPushMetadata(mms []prompb.MetricMetadata) bool {
-	mmdDst := wr.mms
-	maxMetadataPerBlock := *maxMetadataPerBlock
-	for i := range mms {
-		if len(wr.mms) >= maxMetadataPerBlock {
-			if !wr.tryFlush() {
-				return false
-			}
-			mmdDst = wr.mms
-		}
-		mmSrc := &mms[i]
-		mmdDst = append(mmdDst, prompb.MetricMetadata{})
-		wr.copyMetadata(&mmdDst[len(mmdDst)-1], mmSrc)
-	}
-	wr.mms = mmdDst
-	return true
-}
-
-func (wr *writeRequest) copyMetadata(dst, src *prompb.MetricMetadata) {
-	// Direct copy for non-string fields, which are safe by value.
-	dst.Type = src.Type
-	dst.Unit = src.Unit
-
-	// Pre-allocate memory for all string fields.
-	neededBufLen := len(src.MetricFamilyName) + len(src.Help)
-	bufLen := len(wr.metadatabuf)
-	wr.metadatabuf = slicesutil.SetLength(wr.metadatabuf, bufLen+neededBufLen)
-	buf := wr.metadatabuf[:bufLen]
-
-	// Copy MetricFamilyName
-	bufLen = len(buf)
-	buf = append(buf, src.MetricFamilyName...)
-	dst.MetricFamilyName = bytesutil.ToUnsafeString(buf[bufLen:])
-
-	// Copy Help
-	bufLen = len(buf)
-	buf = append(buf, src.Help...)
-	dst.Help = bytesutil.ToUnsafeString(buf[bufLen:])
-
-	wr.metadatabuf = buf
-}
-
-func (wr *writeRequest) tryPushTimeSeries(src []prompb.TimeSeries) bool {
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
-	maxSamplesPerBlock := *maxRowsPerBlock
-	// Allow up to 10x of labels per each block on average.
-	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
-		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
-			wr.tss = tssDst
-			if !wr.tryFlush() {
-				return false
-			}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		dst := &tssDst[len(tssDst)-1]
+		wr.copyTimeSeries(dst, &src[i])
+		if len(wr.tss) >= maxRowsPerBlock {
+			wr.flush()
 			tssDst = wr.tss
 		}
-		tsSrc := &src[i]
-		adjustSampleValues(tsSrc.Samples, wr.significantFigures, wr.roundDigits)
-		tssDst = append(tssDst, prompb.TimeSeries{})
-		wr.copyTimeSeries(&tssDst[len(tssDst)-1], tsSrc)
 	}
-
 	wr.tss = tssDst
-	return true
 }
 
-func (wr *writeRequest) copyTimeSeries(dst, src *prompb.TimeSeries) {
-	labelsSrc := src.Labels
-
-	// Pre-allocate memory for labels.
+func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
+	labelsDst := wr.labels
 	labelsLen := len(wr.labels)
-	wr.labels = slicesutil.SetLength(wr.labels, labelsLen+len(labelsSrc))
-	labelsDst := wr.labels[labelsLen:]
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
 
-	// Pre-allocate memory for byte slice needed for storing label names and values.
-	neededBufLen := 0
-	for i := range labelsSrc {
-		label := &labelsSrc[i]
-		neededBufLen += len(label.Name) + len(label.Value)
-	}
-	bufLen := len(wr.buf)
-	wr.buf = slicesutil.SetLength(wr.buf, bufLen+neededBufLen)
-	buf := wr.buf[:bufLen]
-
-	// Copy labels
-	for i := range labelsSrc {
-		dstLabel := &labelsDst[i]
-		srcLabel := &labelsSrc[i]
-
-		bufLen := len(buf)
 		buf = append(buf, srcLabel.Name...)
-		dstLabel.Name = bytesutil.ToUnsafeString(buf[bufLen:])
-
-		bufLen = len(buf)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
 		buf = append(buf, srcLabel.Value...)
-		dstLabel.Value = bytesutil.ToUnsafeString(buf[bufLen:])
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
 	}
-	wr.buf = buf
-	dst.Labels = labelsDst
+	dst.Labels = labelsDst[labelsLen:]
 
-	// Copy samples
-	samplesLen := len(wr.samples)
-	wr.samples = append(wr.samples, src.Samples...)
-	dst.Samples = wr.samples[samplesLen:]
+	samplesDst = append(samplesDst, src.Samples...)
+	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
 }
 
-// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
-var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
-
-func tryPushWriteRequest(wr *prompb.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
-	if wr.IsEmpty() {
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
+	if len(wr.Timeseries) == 0 {
 		// Nothing to push
-		return true
+		return
 	}
-
-	marshalConcurrencyCh <- struct{}{}
-
 	bb := writeRequestBufPool.Get()
-	bb.B = wr.MarshalProtobuf(bb.B[:0])
-	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := compressBufPool.Get()
-		if isVMRemoteWrite {
-			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
-		} else {
-			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
-		}
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	if len(bb.B) <= *maxUnpackedBlockSize {
+		zb := snappyBufPool.Get()
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
-
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			zbLen := len(zb.B)
-			ok := tryPushBlock(zb.B)
-			compressBufPool.Put(zb)
-			if ok {
-				blockSizeRows.Update(float64(len(wr.Timeseries)))
-				blockMetadataRows.Update(float64(len(wr.Metadata)))
-				blockSizeBytes.Update(float64(zbLen))
-			}
-			return ok
+			pushBlock(zb.B)
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return
 		}
-		compressBufPool.Put(zb)
+		snappyBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
 	}
 
-	// Split timeseries or metadata into two smaller blocks
-	switch len(wr.Timeseries) {
-	case 0:
-		if len(wr.Metadata) == 1 {
-			logger.Warnf("dropping a metadata exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
-			return true
-		}
-		metadata := wr.Metadata
-		n := len(metadata) / 2
-		wr.Metadata = metadata[:n]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Metadata = metadata
-			return false
-		}
-		wr.Metadata = metadata[n:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Metadata = metadata
-			return false
-		}
-		wr.Metadata = metadata
-		return true
-
-	case 1:
-		// A single time series left. Recursively split its samples and metadata into smaller parts if possible.
-		samples := wr.Timeseries[0].Samples
-		metaData := wr.Metadata
-		if len(samples) == 1 && len(metaData) <= 1 {
-			logger.Warnf("dropping a sample for metric and %d metadata which are exceeding -remoteWrite.maxBlockSize=%d bytes", len(metaData), maxUnpackedBlockSize.N)
-			return true
-		}
-		n := len(samples) / 2
-		m := len(metaData) / 2
-		wr.Timeseries[0].Samples = samples[:n]
-		wr.Metadata = metaData[:m]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			wr.Metadata = metaData
-			return false
-		}
-		wr.Timeseries[0].Samples = samples[n:]
-		wr.Metadata = metaData[m:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			wr.Metadata = metaData
-			return false
-		}
-		wr.Timeseries[0].Samples = samples
-		wr.Metadata = metaData
-		return true
-
-	default:
-		// Split both timeseries and metadata.
-		timeseries := wr.Timeseries
-		metaData := wr.Metadata
-		n := len(timeseries) / 2
-		m := len(metaData) / 2
-		wr.Timeseries = timeseries[:n]
-		wr.Metadata = metaData[:m]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries = timeseries
-			wr.Metadata = metaData
-			return false
-		}
-		wr.Timeseries = timeseries[n:]
-		wr.Metadata = metaData[m:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries = timeseries
-			wr.Metadata = metaData
-			return false
-		}
-		wr.Timeseries = timeseries
-		wr.Metadata = metaData
-		return true
-	}
+	// Too big block. Recursively split it into smaller parts.
+	timeseries := wr.Timeseries
+	n := len(timeseries) / 2
+	wr.Timeseries = timeseries[:n]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries[n:]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries
 }
 
 var (
-	blockSizeBytes    = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
-	blockSizeRows     = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
-	blockMetadataRows = metrics.NewHistogram(`vmagent_remotewrite_block_metadata_rows`)
+	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var (
-	writeRequestBufPool bytesutil.ByteBufferPool
-	compressBufPool     bytesutil.ByteBufferPool
-)
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/pendingseries_test.go

@@ -1,73 +0,0 @@
-package remotewrite
-
-import (
-	"fmt"
-	"math"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-)
-
-func TestPushWriteRequest(t *testing.T) {
-	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
-	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
-	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
-	for i, rowsCount := range rowsCounts {
-		expectedBlockLenProm := expectedBlockLensProm[i]
-		expectedBlockLenVM := expectedBlockLensVM[i]
-		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
-			testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
-		})
-	}
-}
-
-func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
-	f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
-		t.Helper()
-		wr := newTestWriteRequest(rowsCount, 20)
-		pushBlockLen := 0
-		pushBlock := func(block []byte) bool {
-			if pushBlockLen > 0 {
-				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
-			}
-			pushBlockLen = len(block)
-			return true
-		}
-		if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
-			t.Fatalf("cannot push data to remote storage")
-		}
-		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
-			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
-				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
-		}
-	}
-
-	// Check Prometheus remote write
-	f(false, expectedBlockLenProm, 3)
-
-	// Check VictoriaMetrics remote write
-	f(true, expectedBlockLenVM, 15)
-}
-
-func newTestWriteRequest(seriesCount, labelsCount int) *prompb.WriteRequest {
-	var wr prompb.WriteRequest
-	for i := 0; i < seriesCount; i++ {
-		var labels []prompb.Label
-		for j := 0; j < labelsCount; j++ {
-			labels = append(labels, prompb.Label{
-				Name:  fmt.Sprintf("label_%d_%d", i, j),
-				Value: fmt.Sprintf("value_%d_%d", i, j),
-			})
-		}
-		wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
-			Labels: labels,
-			Samples: []prompb.Sample{
-				{
-					Value:     float64(i),
-					Timestamp: 1000 * int64(i),
-				},
-			},
-		})
-	}
-	return &wr
-}

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -1,35 +0,0 @@
-package remotewrite
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/golang/snappy"
-	"github.com/klauspost/compress/s2"
-)
-
-func BenchmarkCompressWriteRequestSnappy(b *testing.B) {
-	b.Run("snappy", func(b *testing.B) {
-		benchmarkCompressWriteRequest(b, snappy.Encode)
-	})
-	b.Run("s2", func(b *testing.B) {
-		benchmarkCompressWriteRequest(b, s2.EncodeSnappy)
-	})
-}
-
-func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []byte) []byte) {
-	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
-		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
-			wr := newTestWriteRequest(rowsCount, 10)
-			data := wr.MarshalProtobuf(nil)
-			b.ReportAllocs()
-			b.SetBytes(int64(rowsCount))
-			b.RunParallel(func(pb *testing.PB) {
-				var zb []byte
-				for pb.Next() {
-					zb = compressFunc(zb[:cap(zb)], data)
-				}
-			})
-		})
-	}
-}

app/vmagent/remotewrite/relabel.go

@@ -3,50 +3,24 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"strconv"
 	"strings"
 	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
-		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
-		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
-		"The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
-	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
-		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
-
-	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
-		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
-		"See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels")
+	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+		"Pass multiple -remoteWrite.label flags in order to add multiple flags to metrics before sending them to remote storage")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+		"before sending them to -remoteWrite.url. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config for details")
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
 )
 
-var labelsGlobal []prompb.Label
-
-var (
-	relabelConfigReloads      *metrics.Counter
-	relabelConfigReloadErrors *metrics.Counter
-	relabelConfigSuccess      *metrics.Gauge
-	relabelConfigTimestamp    *metrics.Counter
-)
-
-func initRelabelMetrics() {
-	relabelConfigReloads = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
-	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
-	relabelConfigTimestamp = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
-}
+var labelsGlobal []prompbmarshal.Label
 
 // CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
 func CheckRelabelConfigs() error {
@@ -54,39 +28,6 @@ func CheckRelabelConfigs() error {
 	return err
 }
 
-func initRelabelConfigs() {
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		logger.Fatalf("cannot initialize relabel configs: %s", err)
-	}
-	allRelabelConfigs.Store(rcs)
-	if rcs.isSet() {
-		initRelabelMetrics()
-		relabelConfigSuccess.Set(1)
-		relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
-	}
-}
-
-func reloadRelabelConfigs() {
-	rcs := allRelabelConfigs.Load()
-	if !rcs.isSet() {
-		return
-	}
-	relabelConfigReloads.Inc()
-	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		relabelConfigReloadErrors.Inc()
-		relabelConfigSuccess.Set(0)
-		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
-		return
-	}
-	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
-	logger.Infof("successfully reloaded relabel configs")
-}
-
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
@@ -98,14 +39,10 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 	}
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
+			len(*relabelConfigPaths), len(*remoteWriteURLs))
 	}
-	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
+	rcs.perURL = make([][]promrelabel.ParsedRelabelConfig, len(*remoteWriteURLs))
 	for i, path := range *relabelConfigPaths {
-		if len(path) == 0 {
-			// Skip empty relabel config.
-			continue
-		}
 		prc, err := promrelabel.LoadRelabelConfigs(path)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
@@ -116,65 +53,53 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 }
 
 type relabelConfigs struct {
-	global *promrelabel.ParsedConfigs
-	perURL []*promrelabel.ParsedConfigs
-}
-
-func (rcs *relabelConfigs) isSet() bool {
-	if rcs == nil {
-		return false
-	}
-	if rcs.global.Len() > 0 {
-		return true
-	}
-	for _, pc := range rcs.perURL {
-		if pc.Len() > 0 {
-			return true
-		}
-	}
-	return false
+	global []promrelabel.ParsedRelabelConfig
+	perURL [][]promrelabel.ParsedRelabelConfig
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.
 func initLabelsGlobal() {
+	// Init labelsGlobal
 	labelsGlobal = nil
 	for _, s := range *unparsedLabelsGlobal {
-		if len(s) == 0 {
-			continue
-		}
 		n := strings.IndexByte(s, '=')
 		if n < 0 {
 			logger.Fatalf("missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
 		}
-		labelsGlobal = append(labelsGlobal, prompb.Label{
+		labelsGlobal = append(labelsGlobal, prompbmarshal.Label{
 			Name:  s[:n],
 			Value: s[n+1:],
 		})
 	}
 }
 
-func (rctx *relabelCtx) applyRelabeling(tss []prompb.TimeSeries, pcs *promrelabel.ParsedConfigs) []prompb.TimeSeries {
-	if pcs.Len() == 0 && !*usePromCompatibleNaming {
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, prcs []promrelabel.ParsedRelabelConfig) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && len(prcs) == 0 {
 		// Nothing to change.
 		return tss
 	}
-	rctx.reset()
 	tssDst := tss[:0]
 	labels := rctx.labels[:0]
 	for i := range tss {
 		ts := &tss[i]
 		labelsLen := len(labels)
 		labels = append(labels, ts.Labels...)
-		labels = pcs.Apply(labels, labelsLen)
-		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		labels = promrelabel.ApplyRelabelConfigs(labels, labelsLen, prcs, true)
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue
 		}
-		if *usePromCompatibleNaming {
-			fixPromCompatibleNaming(labels[labelsLen:])
-		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: ts.Samples,
 		})
@@ -183,70 +108,23 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompb.TimeSeries, pcs *promrelabe
 	return tssDst
 }
 
-func (rctx *relabelCtx) appendExtraLabels(tss []prompb.TimeSeries, extraLabels []prompb.Label) {
-	if len(extraLabels) == 0 {
-		return
-	}
-	rctx.reset()
-	labels := rctx.labels[:0]
-	for i := range tss {
-		ts := &tss[i]
-		labelsLen := len(labels)
-		labels = append(labels, ts.Labels...)
-		for j := range extraLabels {
-			extraLabel := extraLabels[j]
-			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
-			if tmp != nil {
-				tmp.Value = extraLabel.Value
-			} else {
-				labels = append(labels, extraLabel)
-			}
-		}
-		ts.Labels = labels[labelsLen:]
-	}
-	rctx.labels = labels
-}
-
-func (rctx *relabelCtx) tenantToLabels(tss []prompb.TimeSeries, accountID, projectID uint32) {
-	rctx.reset()
-	accountIDStr := strconv.FormatUint(uint64(accountID), 10)
-	projectIDStr := strconv.FormatUint(uint64(projectID), 10)
-	labels := rctx.labels[:0]
-	for i := range tss {
-		ts := &tss[i]
-		labelsLen := len(labels)
-		for _, label := range ts.Labels {
-			labelName := label.Name
-			if labelName == "vm_account_id" || labelName == "vm_project_id" {
-				continue
-			}
-			labels = append(labels, label)
-		}
-		labels = append(labels, prompb.Label{
-			Name:  "vm_account_id",
-			Value: accountIDStr,
-		})
-		labels = append(labels, prompb.Label{
-			Name:  "vm_project_id",
-			Value: projectIDStr,
-		})
-		ts.Labels = labels[labelsLen:]
-	}
-	rctx.labels = labels
-}
-
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
-	labels []prompb.Label
+	labels []prompbmarshal.Label
 }
 
 func (rctx *relabelCtx) reset() {
-	promrelabel.CleanLabels(rctx.labels)
+	labels := rctx.labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
 	rctx.labels = rctx.labels[:0]
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		return &relabelCtx{}
 	},
 }
@@ -256,18 +134,6 @@ func getRelabelCtx() *relabelCtx {
 }
 
 func putRelabelCtx(rctx *relabelCtx) {
-	rctx.reset()
+	rctx.labels = rctx.labels[:0]
 	relabelCtxPool.Put(rctx)
 }
-
-func fixPromCompatibleNaming(labels []prompb.Label) {
-	// Replace unsupported Prometheus chars in label names and metric names with underscores.
-	for i := range labels {
-		label := &labels[i]
-		if label.Name == "__name__" {
-			label.Value = promrelabel.SanitizeMetricName(label.Value)
-		} else {
-			label.Name = promrelabel.SanitizeLabelName(label.Name)
-		}
-	}
-}

app/vmagent/remotewrite/relabel_test.go

@@ -1,69 +0,0 @@
-package remotewrite
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
-)
-
-func TestApplyRelabeling(t *testing.T) {
-	f := func(pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
-		rctx := &relabelCtx{}
-		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		gotTss := rctx.applyRelabeling(tss, pcs)
-		if !reflect.DeepEqual(gotTss, expTss) {
-			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
-		}
-	}
-
-	f(nil, "up", "up")
-
-	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
-- target_label: "foo"
-  replacement: "aaa"
-- action: labeldrop
-  regex: "env.*"
-`))
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	f(pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
-
-	oldVal := *usePromCompatibleNaming
-	*usePromCompatibleNaming = true
-	f(nil, `foo.bar`, `foo_bar`)
-	*usePromCompatibleNaming = oldVal
-}
-
-func TestAppendExtraLabels(t *testing.T) {
-	f := func(extraLabels []prompb.Label, sTss, sExpTss string) {
-		t.Helper()
-		rctx := &relabelCtx{}
-		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		rctx.appendExtraLabels(tss, extraLabels)
-		if !reflect.DeepEqual(tss, expTss) {
-			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, tss)
-		}
-	}
-
-	f(nil, "up", "up")
-	f([]prompb.Label{{Name: "foo", Value: "bar"}}, "up", `up{foo="bar"}`)
-	f([]prompb.Label{{Name: "foo", Value: "bar"}}, `up{foo="baz"}`, `up{foo="bar"}`)
-	f([]prompb.Label{{Name: "baz", Value: "qux"}}, `up{foo="baz"}`, `up{foo="baz",baz="qux"}`)
-
-	oldVal := *usePromCompatibleNaming
-	*usePromCompatibleNaming = true
-	f([]prompb.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo.bar="baz"}`)
-	*usePromCompatibleNaming = oldVal
-}
-
-func parseSeries(data string) []prompb.TimeSeries {
-	var tss []prompb.TimeSeries
-	tss = append(tss, prompb.TimeSeries{
-		Labels: promutil.MustNewLabelsFromString(data).GetLabels(),
-	})
-	return tss
-}

app/vmagent/remotewrite/remotewrite_test.go

@@ -1,349 +0,0 @@
-package remotewrite
-
-import (
-	"fmt"
-	"math"
-	"reflect"
-	"strconv"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
-
-	"github.com/VictoriaMetrics/metrics"
-)
-
-func TestGetLabelsHash_Distribution(t *testing.T) {
-	f := func(bucketsCount int) {
-		t.Helper()
-
-		// Distribute itemsCount hashes returned by getLabelsHash() across bucketsCount buckets.
-		itemsCount := 1_000 * bucketsCount
-		m := make([]int, bucketsCount)
-		var labels []prompb.Label
-		for i := 0; i < itemsCount; i++ {
-			labels = append(labels[:0], prompb.Label{
-				Name:  "__name__",
-				Value: fmt.Sprintf("some_name_%d", i),
-			})
-			for j := 0; j < 10; j++ {
-				labels = append(labels, prompb.Label{
-					Name:  fmt.Sprintf("label_%d", j),
-					Value: fmt.Sprintf("value_%d_%d", i, j),
-				})
-			}
-			h := getLabelsHash(labels)
-			m[h%uint64(bucketsCount)]++
-		}
-
-		// Verify that the distribution is even
-		expectedItemsPerBucket := itemsCount / bucketsCount
-		for _, n := range m {
-			if math.Abs(1-float64(n)/float64(expectedItemsPerBucket)) > 0.04 {
-				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want around %d", bucketsCount, n, expectedItemsPerBucket)
-			}
-		}
-	}
-
-	f(2)
-	f(3)
-	f(4)
-	f(5)
-	f(10)
-}
-
-func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
-		t.Helper()
-		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
-		if err != nil {
-			t.Fatalf("cannot load relabel configs: %s", err)
-		}
-		rcs := &relabelConfigs{
-			perURL: []*promrelabel.ParsedConfigs{
-				perURLRelabel,
-			},
-		}
-		allRelabelConfigs.Store(rcs)
-
-		pss := make([]*pendingSeries, 1)
-		isVMProto := &atomic.Bool{}
-		isVMProto.Store(true)
-		pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
-		rwctx := &remoteWriteCtx{
-			idx:                    0,
-			streamAggrKeepInput:    keepInput,
-			streamAggrDropInput:    dropInput,
-			pss:                    pss,
-			rowsPushedAfterRelabel: metrics.GetOrCreateCounter(`foo`),
-			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
-		}
-		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, enableWindows, dedupInterval, nil, "dedup-global")
-		}
-
-		if streamAggrConfig != "" {
-			pushNoop := func(_ []prompb.TimeSeries) {}
-			opts := streamaggr.Options{
-				EnableWindows: enableWindows,
-			}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, &opts, "global")
-			if err != nil {
-				t.Fatalf("cannot load streamaggr configs: %s", err)
-			}
-			defer sas.MustStop()
-			rwctx.sas.Store(sas)
-		}
-
-		offsetMsecs := time.Now().UnixMilli()
-		inputTss := prometheus.MustParsePromMetrics(input, offsetMsecs)
-		expectedTss := make([]prompb.TimeSeries, len(inputTss))
-
-		// copy inputTss to make sure it is not mutated during TryPush call
-		copy(expectedTss, inputTss)
-		if !rwctx.TryPushTimeSeries(inputTss, false) {
-			t.Fatalf("cannot push samples to rwctx")
-		}
-
-		if !reflect.DeepEqual(expectedTss, inputTss) {
-			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
-		}
-	}
-
-	f(`
-- interval: 1m
-  outputs: [sum_samples]
-- interval: 2m
-  outputs: [count_series]
-`, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, false, 0, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-	f(``, ``, true, time.Hour, false, false, `
-metric{env="dev"} 10
-metric{env="foo"} 20
-metric{env="dev"} 15
-metric{env="foo"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, true, false, `
-metric{env="test"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, false, true, `
-metric{env="foo"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, true, true, `
-metric{env="dev"} 10
-metric{env="test"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-}
-
-func TestShardAmountRemoteWriteCtx(t *testing.T) {
-	// 1. distribute 100000 series to n nodes.
-	// 2. remove the last node from healthy list.
-	// 3. distribute the same 10000 series to (n-1) node again.
-	// 4. check active time series change rate:
-	// change rate must < (3/total nodes). e.g. +30% if 10 you have 10 nodes.
-
-	f := func(remoteWriteCount int, healthyIdx []int, replicas int) {
-		t.Helper()
-		defer func() {
-			rwctxsGlobal = nil
-			rwctxsGlobalIdx = nil
-			rwctxConsistentHashGlobal = nil
-		}()
-
-		rwctxsGlobal = make([]*remoteWriteCtx, remoteWriteCount)
-		rwctxsGlobalIdx = make([]int, remoteWriteCount)
-		rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
-
-		for i := range remoteWriteCount {
-			rwCtx := &remoteWriteCtx{
-				idx: i,
-			}
-			rwctxsGlobalIdx[i] = i
-
-			if i >= len(healthyIdx) {
-				rwctxsGlobal[i] = rwCtx
-				continue
-			}
-			hIdx := healthyIdx[i]
-			if hIdx != i {
-				rwctxs = append(rwctxs, &remoteWriteCtx{
-					idx: hIdx,
-				})
-			} else {
-				rwctxs = append(rwctxs, rwCtx)
-			}
-			rwctxsGlobal[i] = rwCtx
-		}
-
-		seriesCount := 100000
-		// build 1000000 series
-		tssBlock := make([]prompb.TimeSeries, 0, seriesCount)
-		for i := 0; i < seriesCount; i++ {
-			tssBlock = append(tssBlock, prompb.TimeSeries{
-				Labels: []prompb.Label{
-					{
-						Name:  "label",
-						Value: strconv.Itoa(i),
-					},
-				},
-				Samples: []prompb.Sample{
-					{
-						Timestamp: 0,
-						Value:     0,
-					},
-				},
-			})
-		}
-
-		// build consistent hash for x remote write context
-		// build active time series set
-		nodes := make([]string, 0, remoteWriteCount)
-		activeTimeSeriesByNodes := make([]map[string]struct{}, remoteWriteCount)
-		for i := 0; i < remoteWriteCount; i++ {
-			nodes = append(nodes, fmt.Sprintf("node%d", i))
-			activeTimeSeriesByNodes[i] = make(map[string]struct{})
-		}
-		rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(nodes, 0)
-
-		// create shards
-		x := getTSSShards(len(rwctxs))
-		shards := x.shards
-
-		// execute
-		shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
-
-		for i, nodeIdx := range healthyIdx {
-			for _, ts := range shards[i] {
-				// add it to node[nodeIdx]'s active time series
-				activeTimeSeriesByNodes[nodeIdx][prompb.LabelsToString(ts.Labels)] = struct{}{}
-			}
-		}
-
-		totalActiveTimeSeries := 0
-		for _, activeTimeSeries := range activeTimeSeriesByNodes {
-			totalActiveTimeSeries += len(activeTimeSeries)
-		}
-		avgActiveTimeSeries1 := totalActiveTimeSeries / remoteWriteCount
-		putTSSShards(x)
-
-		// removed last node
-		rwctxs = rwctxs[:len(rwctxs)-1]
-		healthyIdx = healthyIdx[:len(healthyIdx)-1]
-
-		x = getTSSShards(len(rwctxs))
-		shards = x.shards
-
-		// execute
-		shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
-		for i, nodeIdx := range healthyIdx {
-			for _, ts := range shards[i] {
-				// add it to node[nodeIdx]'s active time series
-				activeTimeSeriesByNodes[nodeIdx][prompb.LabelsToString(ts.Labels)] = struct{}{}
-			}
-		}
-
-		totalActiveTimeSeries = 0
-		for _, activeTimeSeries := range activeTimeSeriesByNodes {
-			totalActiveTimeSeries += len(activeTimeSeries)
-		}
-		avgActiveTimeSeries2 := totalActiveTimeSeries / remoteWriteCount
-
-		changed := math.Abs(float64(avgActiveTimeSeries2-avgActiveTimeSeries1) / float64(avgActiveTimeSeries1))
-		threshold := 3 / float64(remoteWriteCount)
-
-		if changed >= threshold {
-			t.Fatalf("average active time series before: %d, after: %d, changed: %.2f. threshold: %.2f", avgActiveTimeSeries1, avgActiveTimeSeries2, changed, threshold)
-		}
-
-	}
-
-	f(5, []int{0, 1, 2, 3, 4}, 1)
-
-	f(5, []int{0, 1, 2, 3, 4}, 2)
-
-	f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 1)
-
-	f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 3)
-}
-
-func TestCalculateHealthyRwctxIdx(t *testing.T) {
-	f := func(total int, healthyIdx []int, unhealthyIdx []int) {
-		t.Helper()
-
-		healthyMap := make(map[int]bool)
-		for _, idx := range healthyIdx {
-			healthyMap[idx] = true
-		}
-		rwctxsGlobal = make([]*remoteWriteCtx, total)
-		rwctxsGlobalIdx = make([]int, total)
-		rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
-		for i := range rwctxsGlobal {
-			rwctx := &remoteWriteCtx{idx: i}
-			rwctxsGlobal[i] = rwctx
-			if healthyMap[i] {
-				rwctxs = append(rwctxs, rwctx)
-			}
-			rwctxsGlobalIdx[i] = i
-		}
-
-		gotHealthyIdx, gotUnhealthyIdx := calculateHealthyRwctxIdx(rwctxs)
-		if !reflect.DeepEqual(healthyIdx, gotHealthyIdx) {
-			t.Errorf("calculateHealthyRwctxIdx want healthyIdx = %v, got %v", healthyIdx, gotHealthyIdx)
-		}
-		if !reflect.DeepEqual(unhealthyIdx, gotUnhealthyIdx) {
-			t.Errorf("calculateHealthyRwctxIdx want unhealthyIdx = %v, got %v", unhealthyIdx, gotUnhealthyIdx)
-		}
-	}
-
-	f(5, []int{0, 1, 2, 3, 4}, nil)
-	f(5, []int{0, 1, 2, 4}, []int{3})
-	f(5, []int{2, 4}, []int{0, 1, 3})
-	f(5, []int{0, 2, 4}, []int{1, 3})
-	f(5, []int{}, []int{0, 1, 2, 3, 4})
-	f(5, []int{4}, []int{0, 1, 2, 3})
-	f(1, []int{0}, nil)
-	f(1, []int{}, []int{0})
-}

app/vmagent/remotewrite/statconn.go

@@ -0,0 +1,76 @@
+package remotewrite
+
+import (
+	"net"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/fasthttp"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func statDial(addr string) (conn net.Conn, err error) {
+	if netutil.TCP6Enabled() {
+		conn, err = fasthttp.DialDualStack(addr)
+	} else {
+		conn, err = fasthttp.Dial(addr)
+	}
+	dialsTotal.Inc()
+	if err != nil {
+		dialErrors.Inc()
+		return nil, err
+	}
+	conns.Inc()
+	sc := &statConn{
+		Conn: conn,
+	}
+	return sc, nil
+}
+
+var (
+	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
+	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
+	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
+)
+
+type statConn struct {
+	closed uint64
+	net.Conn
+}
+
+func (sc *statConn) Read(p []byte) (int, error) {
+	n, err := sc.Conn.Read(p)
+	connReadsTotal.Inc()
+	if err != nil {
+		connReadErrors.Inc()
+	}
+	connBytesRead.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Write(p []byte) (int, error) {
+	n, err := sc.Conn.Write(p)
+	connWritesTotal.Inc()
+	if err != nil {
+		connWriteErrors.Inc()
+	}
+	connBytesWritten.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Close() error {
+	err := sc.Conn.Close()
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
+		conns.Dec()
+	}
+	return err
+}
+
+var (
+	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
+	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
+	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
+	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
+	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
+	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
+)

app/vmagent/remotewrite/streamaggr.go

@@ -1,258 +0,0 @@
-package remotewrite
-
-import (
-	"flag"
-	"fmt"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	// Global config
-	streamAggrGlobalConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
-		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
-	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation "+
-		"with -streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
-		"aggregator before optional aggregation with -streamAggr.config . "+
-		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
-	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
-		"current aggregation interval for aggregator. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples")
-	streamAggrGlobalIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start for "+
-		"aggregator. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from "+
-		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignore-aggregation-intervals-on-start")
-	streamAggrGlobalDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples for aggregator "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels")
-	streamAggrGlobalEnableWindows = flag.Bool("streamAggr.enableWindows", false, "Enables aggregation within fixed windows for all global aggregators. "+
-		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
-
-	// Per URL config
-	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
-		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
-	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
-	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
-		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
-		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
-		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving buffered delayed data from clients pushing data into the vmagent. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignore-aggregation-intervals-on-start")
-	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
-		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
-		"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels")
-	streamAggrEnableWindows = flagutil.NewArrayBool("remoteWrite.streamAggr.enableWindows", "Enables aggregation within fixed windows for all remote write's aggregators. "+
-		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
-)
-
-// CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
-func CheckStreamAggrConfigs() error {
-	// Check global config
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		return err
-	}
-	sas.MustStop()
-
-	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
-		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
-	}
-
-	pushNoop := func(_ []prompb.TimeSeries) {}
-	for idx := range *streamAggrConfig {
-		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
-		if err != nil {
-			return err
-		}
-		sas.MustStop()
-	}
-	return nil
-}
-
-func reloadStreamAggrConfigs() {
-	reloadStreamAggrConfigGlobal()
-	for _, rwctx := range rwctxsGlobal {
-		rwctx.reloadStreamAggrConfig()
-	}
-}
-
-func reloadStreamAggrConfigGlobal() {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
-		return
-	}
-
-	sas := sasGlobal.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := sasGlobal.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
-}
-
-func initStreamAggrConfigGlobal() {
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		logger.Fatalf("cannot initialize global stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		sasGlobal.Store(sas)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	}
-	dedupInterval := *streamAggrGlobalDedupInterval
-	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushTimeSeriesToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
-	}
-}
-
-func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
-	idx := rwctx.idx
-
-	sas, err := rwctx.newStreamAggrConfig()
-	if err != nil {
-		logger.Fatalf("cannot initialize stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	}
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-	if dedupInterval > 0 {
-		alias := fmt.Sprintf("dedup-%d", idx+1)
-		var dropLabels []string
-		if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-			dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
-		}
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, dropLabels, alias)
-	}
-}
-
-func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
-	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if path == "" {
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := rwctx.newStreamAggrConfig()
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
-		return
-	}
-
-	sas := rwctx.sas.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := rwctx.sas.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
-}
-
-func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return nil, nil
-	}
-
-	opts := &streamaggr.Options{
-		DedupInterval:        *streamAggrGlobalDedupInterval,
-		DropInputLabels:      *streamAggrGlobalDropInputLabels,
-		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
-		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
-		KeepInput:            *streamAggrGlobalKeepInput,
-		EnableWindows:        *streamAggrGlobalEnableWindows,
-	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushTimeSeriesToRemoteStoragesTrackDropped, opts, "global")
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
-	}
-	return sas, nil
-}
-
-func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
-	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
-}
-
-func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path := streamAggrConfig.GetOptionalArg(idx)
-	if path == "" {
-		return nil, nil
-	}
-
-	alias := fmt.Sprintf("%d:secret-url", idx+1)
-	if *showRemoteWriteURL {
-		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
-	}
-	var dropLabels []string
-	if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-		dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
-	}
-	opts := &streamaggr.Options{
-		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
-		DropInputLabels:      dropLabels,
-		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
-		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
-		EnableWindows:        streamAggrEnableWindows.GetOptionalArg(idx),
-	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
-	}
-	return sas, nil
-}

app/vmagent/vmimport/request_handler.go

@@ -5,38 +5,28 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="vmimport"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(rows []vmimport.Row) error {
-		return insertRows(at, rows, extraLabels)
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
 	})
 }
 
-func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompb.Label) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -46,43 +36,35 @@ func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompb.Label)
 	samples := ctx.Samples[:0]
 	for i := range rows {
 		r := &rows[i]
-		rowsTotal += len(r.Values)
 		labelsLen := len(labels)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  bytesutil.ToUnsafeString(tag.Key),
 				Value: bytesutil.ToUnsafeString(tag.Value),
 			})
 		}
-		labels = append(labels, extraLabels...)
 		values := r.Values
 		timestamps := r.Timestamps
-		if len(timestamps) != len(values) {
-			logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
-		}
+		_ = timestamps[len(values)-1]
 		samplesLen := len(samples)
 		for j, value := range values {
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Value:     value,
 				Timestamp: timestamps[j],
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
+		rowsTotal += len(values)
 	}
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(&ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil
 }

app/vmalert-tool/README.md

@@ -1,3 +0,0 @@
-See vmalert-tool docs [here](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/).
-
-vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmalert-tool.md).

app/vmalert-tool/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-EXPOSE 8880
-
-ENTRYPOINT ["/vmalert-tool-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vmalert-tool-prod

app/vmalert-tool/multiarch/Dockerfile

@@ -1,13 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 8429
-ENTRYPOINT ["/vmalert-tool-prod"]
-ARG TARGETARCH
-ARG BINARY_SUFFIX=non-existing
-COPY vmalert-tool-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmalert-tool-prod

app/vmalert/Makefile

@@ -12,35 +12,20 @@ vmalert-prod:
 vmalert-pure-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-pure
 
-vmalert-linux-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-linux-amd64
+vmalert-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-amd64
 
-vmalert-linux-arm-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm
+vmalert-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm
 
-vmalert-linux-arm64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm64
+vmalert-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm64
 
-vmalert-linux-ppc64le-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-linux-ppc64le
+vmalert-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-ppc64le
 
-vmalert-linux-386-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-linux-386
-
-vmalert-darwin-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
-
-vmalert-darwin-arm64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64
-
-vmalert-freebsd-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-freebsd-amd64
-
-vmalert-openbsd-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-openbsd-amd64
-
-vmalert-windows-amd64-prod:
-	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
+vmalert-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-386
 
 package-vmalert:
 	APP_NAME=vmalert $(MAKE) package-via-docker
@@ -68,76 +53,33 @@ publish-vmalert:
 
 test-vmalert:
 	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
-	go test -v -race -cover ./app/vmalert/rule
-	go test -v -race -cover ./app/vmalert/templates
 	go test -v -race -cover ./app/vmalert/datasource
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
-	go test -v -race -cover ./app/vmalert/remotewrite
-	go test -v -race -cover ./app/vmalert/vmalertutil
 
 run-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
 		-datasource.url=http://localhost:8428 \
 		-notifier.url=http://localhost:9093 \
 		-notifier.url=http://127.0.0.1:9093 \
 		-remoteWrite.url=http://localhost:8428 \
 		-remoteRead.url=http://localhost:8428 \
-		-external.label=cluster=east-1 \
-		-external.label=replica=a \
-		-evaluationInterval=3s \
-		-configCheckInterval=10s
+		-evaluationInterval=3s
 
-run-vmalert-sd: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
-		-datasource.url=http://localhost:8428 \
-		-remoteWrite.url=http://localhost:8428 \
-		-notifier.config=app/vmalert/notifier/testdata/mixed.good.yaml \
-		-configCheckInterval=10s
+vmalert-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-amd64 ./app/vmalert
 
-replay-vmalert: vmalert
-	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules-replay-good.rules \
-		-datasource.url=http://localhost:8428 \
-		-remoteWrite.url=http://localhost:8428 \
-		-external.label=cluster=east-1 \
-		-external.label=replica=a \
-		-replay.timeFrom=2024-06-01T00:00:00Z
+vmalert-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm ./app/vmalert
 
-vmalert-linux-amd64:
-	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+vmalert-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm64 ./app/vmalert
 
-vmalert-linux-arm:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+vmalert-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-ppc64le ./app/vmalert
 
-vmalert-linux-arm64:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vmalert-linux-ppc64le:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vmalert-linux-s390x:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vmalert-linux-loong64:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vmalert-linux-386:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vmalert-darwin-amd64:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmalert-darwin-arm64:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vmalert-freebsd-amd64:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmalert-openbsd-amd64:
-	APP_NAME=vmalert CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vmalert-windows-amd64:
-	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-goarch
+vmalert-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-386 ./app/vmalert
 
 vmalert-pure:
 	APP_NAME=vmalert $(MAKE) app-local-pure

app/vmalert/README.md

@@ -1,3 +1,277 @@
-See vmalert docs [here](https://docs.victoriametrics.com/victoriametrics/vmalert/).
+## vmalert
 
-vmalert docs can be edited at [docs/vmalert.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmalert.md).
+`vmalert` executes a list of given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
+or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
+rules against configured address.
+
+### Features:
+* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
+* VictoriaMetrics [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
+ support and expressions validation;
+* Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
+ support;
+* Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Lightweight without extra dependencies.
+
+### Limitations:
+* `vmalert` execute queries against remote datasource which has reliability risks because of network. 
+It is recommended to configure alerts thresholds and rules expressions with understanding that network request
+may fail;
+* by default, rules execution is sequential within one group, but persisting of execution results to remote
+storage is asynchronous. Hence, user shouldn't rely on recording rules chaining when result of previous
+recording rule is reused in next one;
+* there is no `query` function support in templates yet;
+* `vmalert` has no UI, just an API for getting groups and rules statuses.
+
+### QuickStart
+
+To build `vmalert` from sources:
+```
+git clone https://github.com/VictoriaMetrics/VictoriaMetrics
+cd VictoriaMetrics
+make vmalert
+```
+The build binary will be placed to `VictoriaMetrics/bin` folder.
+
+To start using `vmalert` you will need the following things:
+* list of rules - PromQL/MetricsQL expressions to execute;
+* datasource address - reachable VictoriaMetrics instance for rules execution;
+* notifier address - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing, 
+aggregating alerts and sending notifications.
+* remote write address - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+compatible storage address for storing recording rules results and alerts state in for of timeseries. This is optional.
+
+Then configure `vmalert` accordingly:
+```
+./bin/vmalert -rule=alert.rules \
+		-datasource.url=http://localhost:8428 \
+        -notifier.url=http://localhost:9093
+```
+
+Configuration for [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) 
+and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very 
+similar to Prometheus rules and configured using YAML. Configuration examples may be found 
+in [testdata](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/config/testdata) folder.
+Every `rule` belongs to `group` and every configuration file may contain arbitrary number of groups:
+```yaml
+groups:
+  [ - <rule_group> ]
+```
+
+#### Groups
+
+Each group has following attributes:
+```yaml
+# The name of the group. Must be unique within a file.
+name: <string>
+
+# How often rules in the group are evaluated.
+[ interval: <duration> | default = global.evaluation_interval ]
+
+# How many rules execute at once. Increasing concurrency may speed
+# up round execution speed. 
+[ concurrency: <integer> | default = 1 ]
+
+rules:
+  [ - <rule> ... ]
+```
+
+#### Rules
+
+There are two types of Rules:
+* [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) - 
+Alerting rules allows to define alert conditions via [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
+and to send notifications about firing alerts to [Alertmanager](https://github.com/prometheus/alertmanager).
+* [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) - 
+Recording rules allow you to precompute frequently needed or computationally expensive expressions 
+and save their result as a new set of time series.
+
+`vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels
+within one group. 
+
+##### Alerting rules
+
+The syntax for alerting rule is following:
+```yaml
+# The name of the alert. Must be a valid metric name.
+alert: <string>
+
+# The MetricsQL expression to evaluate.
+expr: <string>
+
+# Alerts are considered firing once they have been returned for this long.
+# Alerts which have not yet fired for long enough are considered pending.
+[ for: <duration> | default = 0s ]
+
+# Labels to add or overwrite for each alert.
+labels:
+  [ <labelname>: <tmpl_string> ]
+
+# Annotations to add to each alert.
+annotations:
+  [ <labelname>: <tmpl_string> ]
+``` 
+
+`vmalert` has no local storage and alerts state is stored in process memory. Hence, after reloading of `vmalert` process
+alerts state will be lost. To avoid this situation, `vmalert` may be configured via following flags:
+* `-remoteWrite.url` - URL to Victoria Metrics or VMInsert. `vmalert` will persist alerts state into the configured
+address in form of timeseries with name `ALERTS` via remote-write protocol.
+* `-remoteRead.url` - URL to Victoria Metrics or VMSelect. `vmalert` will try to restore alerts state from configured
+address by querying `ALERTS` timeseries.
+
+
+##### Recording rules
+
+The syntax for recording rules is following:
+```yaml
+# The name of the time series to output to. Must be a valid metric name.
+record: <string>
+
+# The MetricsQL expression to evaluate.
+expr: <string>
+
+# Labels to add or overwrite before storing the result.
+labels:
+  [ <labelname>: <labelvalue> ]
+```
+
+For recording rules to work `-remoteWrite.url` must specified.
+
+
+#### WEB
+
+`vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>/api/v1/groups` - list of all loaded groups and rules;
+* `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
+* `http://<vmalert-addr>/api/v1/<groupName>/<alertID>/status" ` - get alert status by ID.
+Used as alert source in AlertManager.
+* `http://<vmalert-addr>/metrics` - application metrics.
+* `http://<vmalert-addr>/-/reload` - hot configuration reload.
+
+
+### Configuration
+
+The shortlist of configuration flags is the following:
+```
+Usage of vmalert:
+  -datasource.basicAuth.password string
+        Optional basic auth password for -datasource.url
+  -datasource.basicAuth.username string
+        Optional basic auth username for -datasource.url
+  -datasource.tlsCAFile value
+        Optional path to TLS CA file to use for verifying connections to -datasource.url. By default system CA is used.
+  -datasource.tlsCertFile value
+        Optional path to client-side TLS certificate file to use when connecting to -datasource.url.
+  -datasource.tlsInsecureSkipVerify
+        Whether to skip tls verification when connecting to -datasource.url
+  -datasource.tlsKeyFile value
+        Optional path to client-side TLS certificate key to use when connecting to -datasource.url.
+  -datasource.tlsServerName value
+        Optional TLS server name to use for connections to -datasource.url. By default the server name from -datasource.url is used.
+  -datasource.url string
+        Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428
+  -evaluationInterval duration
+        How often to evaluate the rules (default 1m0s)
+  -external.url string
+        External URL is used as alert's source for sent alerts to the notifier
+  -httpListenAddr string
+        Address to listen for http connections (default ":8880")
+  -metricsAuthKey string
+        Auth key for /metrics. It overrides httpAuth settings
+  -notifier.tlsCAFile value
+        Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used.
+  -notifier.tlsCertFile value
+        Optional path to client-side TLS certificate file to use when connecting to -notifier.url.
+  -notifier.tlsInsecureSkipVerify
+        Whether to skip tls verification when connecting to -notifier.url
+  -notifier.tlsKeyFile value
+        Optional path to client-side TLS certificate key to use when connecting to -notifier.url.
+  -notifier.tlsServerName value
+        Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used.
+  -notifier.url string
+        Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+  -remoteRead.basicAuth.password string
+        Optional basic auth password for -remoteRead.url
+  -remoteRead.basicAuth.username string
+        Optional basic auth username for -remoteRead.url
+  -remoteRead.lookback duration
+        Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
+  -remoteRead.tlsCAFile value
+        Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used.
+  -remoteRead.tlsCertFile value
+        Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url.
+  -remoteRead.tlsInsecureSkipVerify
+        Whether to skip tls verification when connecting to -remoteRead.url
+  -remoteRead.tlsKeyFile value
+        Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url.
+  -remoteRead.tlsServerName value
+        Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used.
+  -remoteRead.url vmalert
+        Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428
+  -remoteWrite.basicAuth.password string
+        Optional basic auth password for -remoteWrite.url
+  -remoteWrite.basicAuth.username string
+        Optional basic auth username for -remoteWrite.url
+  -remoteWrite.concurrency int
+        Defines number of readers that concurrently write into remote storage (default 1)
+  -remoteWrite.flushInterval duration
+        Defines interval of flushes to remote write endpoint (default 5s)
+  -remoteWrite.maxBatchSize int
+        Defines defines max number of timeseries to be flushed at once (default 1000)
+  -remoteWrite.maxQueueSize int
+        Defines the max number of pending datapoints to remote write endpoint (default 100000)
+  -remoteWrite.tlsCAFile value
+        Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used.
+  -remoteWrite.tlsCertFile value
+        Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url.
+  -remoteWrite.tlsInsecureSkipVerify
+        Whether to skip tls verification when connecting to -remoteWrite.url
+  -remoteWrite.tlsKeyFile value
+        Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url.
+  -remoteWrite.tlsServerName value
+        Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used.
+  -remoteWrite.url string
+        Optional URL to Victoria Metrics or VMInsert where to persist alerts state in form of timeseries. E.g. http://127.0.0.1:8428
+  -rule value
+        Path to the file with alert rules. 
+        Supports patterns. Flag can be specified multiple times. 
+        Examples:
+         -rule /path/to/file. Path to a single file with alerting rules
+         -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+        absolute path to all .yaml files in root.
+  -rule.validateExpressions
+        Whether to validate rules expressions via MetricsQL engine (default true)
+  -rule.validateTemplates
+        Whether to validate annotation and label templates (default true)
+```
+
+Pass `-help` to `vmalert` in order to see the full list of supported 
+command-line flags with their descriptions.
+
+To reload configuration without `vmalert` restart send SIGHUP signal
+or send GET request to `/-/reload` endpoint.
+
+### Contributing
+
+`vmalert` is mostly designed and built by VictoriaMetrics community.
+Feel free to share your experience and ideas for improving this 
+software. Please keep simplicity as the main priority.
+
+### How to build from sources
+
+It is recommended using 
+[binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) 
+- `vmalert` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmalert` from the root folder of the repository.
+   It builds `vmalert` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmalert-prod` from the root folder of the repository.
+   It builds `vmalert-prod` binary and puts it into the `bin` folder.

app/vmalert/alerting.go

@@ -0,0 +1,374 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// AlertingRule is basic alert entity
+type AlertingRule struct {
+	RuleID      uint64
+	Name        string
+	Expr        string
+	For         time.Duration
+	Labels      map[string]string
+	Annotations map[string]string
+	GroupID     uint64
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores list of active alerts
+	alerts map[uint64]*notifier.Alert
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+}
+
+func newAlertingRule(gID uint64, cfg config.Rule) *AlertingRule {
+	return &AlertingRule{
+		RuleID:      cfg.ID,
+		Name:        cfg.Alert,
+		Expr:        cfg.Expr,
+		For:         cfg.For,
+		Labels:      cfg.Labels,
+		Annotations: cfg.Annotations,
+		GroupID:     gID,
+		alerts:      make(map[uint64]*notifier.Alert),
+	}
+}
+
+// String implements Stringer interface
+func (ar *AlertingRule) String() string {
+	return ar.Name
+}
+
+// ID returns unique Rule ID
+// within the parent Group.
+func (ar *AlertingRule) ID() uint64 {
+	return ar.RuleID
+}
+
+// Exec executes AlertingRule expression via the given Querier.
+// Based on the Querier results AlertingRule maintains notifier.Alerts
+func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) {
+	qMetrics, err := q.Query(ctx, ar.Expr)
+	ar.mu.Lock()
+	defer ar.mu.Unlock()
+
+	ar.lastExecError = err
+	ar.lastExecTime = time.Now()
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
+	}
+
+	for h, a := range ar.alerts {
+		// cleanup inactive alerts from previous Exec
+		if a.State == notifier.StateInactive {
+			delete(ar.alerts, h)
+		}
+	}
+
+	updated := make(map[uint64]struct{})
+	// update list of active alerts
+	for _, m := range qMetrics {
+		h := hash(m)
+		updated[h] = struct{}{}
+		if a, ok := ar.alerts[h]; ok {
+			if a.Value != m.Value {
+				// update Value field with latest value
+				a.Value = m.Value
+				// and re-exec template since Value can be used
+				// in templates
+				err = ar.template(a)
+				if err != nil {
+					return nil, err
+				}
+			}
+			continue
+		}
+		a, err := ar.newAlert(m, ar.lastExecTime)
+		if err != nil {
+			ar.lastExecError = err
+			return nil, fmt.Errorf("failed to create alert: %w", err)
+		}
+		a.ID = h
+		a.State = notifier.StatePending
+		ar.alerts[h] = a
+	}
+
+	for h, a := range ar.alerts {
+		// if alert wasn't updated in this iteration
+		// means it is resolved already
+		if _, ok := updated[h]; !ok {
+			if a.State == notifier.StatePending {
+				// alert was in Pending state - it is not
+				// active anymore
+				delete(ar.alerts, h)
+				continue
+			}
+			a.State = notifier.StateInactive
+			continue
+		}
+		if a.State == notifier.StatePending && time.Since(a.Start) >= ar.For {
+			a.State = notifier.StateFiring
+			alertsFired.Inc()
+		}
+	}
+	if series {
+		return ar.toTimeSeries(ar.lastExecTime), nil
+	}
+	return nil, nil
+}
+
+func (ar *AlertingRule) toTimeSeries(timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		ts := ar.alertToTimeSeries(a, timestamp)
+		tss = append(tss, ts...)
+	}
+	return tss
+}
+
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (ar *AlertingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*AlertingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update alerting rule with wrong type %#v", r)
+	}
+	ar.Expr = nr.Expr
+	ar.For = nr.For
+	ar.Labels = nr.Labels
+	ar.Annotations = nr.Annotations
+	return nil
+}
+
+// TODO: consider hashing algorithm in VM
+func hash(m datasource.Metric) uint64 {
+	hash := fnv.New64a()
+	labels := m.Labels
+	sort.Slice(labels, func(i, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
+	for _, l := range labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		hash.Write([]byte(l.Name))
+		hash.Write([]byte(l.Value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time) (*notifier.Alert, error) {
+	a := &notifier.Alert{
+		GroupID: ar.GroupID,
+		Name:    ar.Name,
+		Labels:  map[string]string{},
+		Value:   m.Value,
+		Start:   start,
+		Expr:    ar.Expr,
+	}
+	for _, l := range m.Labels {
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		a.Labels[l.Name] = l.Value
+	}
+	return a, ar.template(a)
+}
+
+func (ar *AlertingRule) template(a *notifier.Alert) error {
+	// 1. template rule labels with data labels
+	rLabels, err := a.ExecTemplate(ar.Labels)
+	if err != nil {
+		return err
+	}
+
+	// 2. merge data labels and rule labels
+	// metric labels may be overridden by
+	// rule labels
+	for k, v := range rLabels {
+		a.Labels[k] = v
+	}
+
+	// 3. template merged labels
+	a.Labels, err = a.ExecTemplate(a.Labels)
+	if err != nil {
+		return err
+	}
+
+	a.Annotations, err = a.ExecTemplate(ar.Annotations)
+	return err
+}
+
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
+	ar.mu.RLock()
+	defer ar.mu.RUnlock()
+	a, ok := ar.alerts[id]
+	if !ok {
+		return nil
+	}
+	return ar.newAlertAPI(*a)
+}
+
+// RuleAPI returns Rule representation in form
+// of APIAlertingRule
+func (ar *AlertingRule) RuleAPI() APIAlertingRule {
+	var lastErr string
+	if ar.lastExecError != nil {
+		lastErr = ar.lastExecError.Error()
+	}
+	return APIAlertingRule{
+		// encode as strings to avoid rounding
+		ID:          fmt.Sprintf("%d", ar.ID()),
+		GroupID:     fmt.Sprintf("%d", ar.GroupID),
+		Name:        ar.Name,
+		Expression:  ar.Expr,
+		For:         ar.For.String(),
+		LastError:   lastErr,
+		LastExec:    ar.lastExecTime,
+		Labels:      ar.Labels,
+		Annotations: ar.Annotations,
+	}
+}
+
+// AlertsAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsAPI() []*APIAlert {
+	var alerts []*APIAlert
+	ar.mu.RLock()
+	for _, a := range ar.alerts {
+		alerts = append(alerts, ar.newAlertAPI(*a))
+	}
+	ar.mu.RUnlock()
+	return alerts
+}
+
+func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
+	return &APIAlert{
+		// encode as strings to avoid rounding
+		ID:      fmt.Sprintf("%d", a.ID),
+		GroupID: fmt.Sprintf("%d", a.GroupID),
+
+		Name:        a.Name,
+		Expression:  ar.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.Start,
+		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+	}
+}
+
+const (
+	// AlertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// AlertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// AlertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// AlertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+)
+
+// alertToTimeSeries converts the given alert with the given timestamp to timeseries
+func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(ar.Name, a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(ar.Name, a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertNameLabel] = name
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries(1, labels, timestamp)
+}
+
+// alertForToTimeSeries returns a timeseries that represents
+// state of active alerts, where value is time when alert become active
+func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	labels[alertNameLabel] = name
+	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
+}
+
+// Restore restores the state of active alerts basing on previously written timeseries.
+// Restore restores only Start field. Field State will be always Pending and supposed
+// to be updated on next Exec, as well as Value field.
+// Only rules with For > 0 will be restored.
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration) error {
+	if q == nil {
+		return fmt.Errorf("querier is nil")
+	}
+	// Get the last datapoint in range via MetricsQL `last_over_time`.
+	// We don't use plain PromQL since Prometheus doesn't support
+	// remote write protocol which is used for state persistence in vmalert.
+	expr := fmt.Sprintf("last_over_time(%s{alertname=%q}[%ds])",
+		alertForStateMetricName, ar.Name, int(lookback.Seconds()))
+	qMetrics, err := q.Query(ctx, expr)
+	if err != nil {
+		return err
+	}
+
+	for _, m := range qMetrics {
+		labels := m.Labels
+		m.Labels = make([]datasource.Label, 0)
+		// drop all extra labels, so hash key will
+		// be identical to timeseries received in Exec
+		for _, l := range labels {
+			if l.Name == alertNameLabel {
+				continue
+			}
+			// drop all overridden labels
+			if _, ok := ar.Labels[l.Name]; ok {
+				continue
+			}
+			m.Labels = append(m.Labels, l)
+		}
+
+		a, err := ar.newAlert(m, time.Unix(int64(m.Value), 0))
+		if err != nil {
+			return fmt.Errorf("failed to create alert: %w", err)
+		}
+		a.ID = hash(m)
+		a.State = notifier.StatePending
+		ar.alerts[a.ID] = a
+		logger.Infof("alert %q(%d) restored to state at %v", a.Name, a.ID, a.Start)
+	}
+	return nil
+}

app/vmalert/alerting_test.go

@@ -0,0 +1,455 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestAlertingRule_ToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule  *AlertingRule
+		alert *notifier.Alert
+		expTS []prompbmarshal.TimeSeries
+	}{
+		{
+			newTestAlertingRule("instant", 0),
+			&notifier.Alert{State: notifier.StateFiring},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("instant extra labels", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant extra labels",
+					"job":           "foo",
+					"instance":      "bar",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("instant labels override", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant labels override",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("for", time.Second),
+			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "for",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for",
+				}, timestamp),
+			},
+		},
+		{
+			newTestAlertingRule("for pending", 10*time.Second),
+			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StatePending.String(),
+					alertNameLabel:  "for pending",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for pending",
+				}, timestamp),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			tc.rule.alerts[tc.alert.ID] = tc.alert
+			tss := tc.rule.toTimeSeries(timestamp)
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Exec(t *testing.T) {
+	const defaultStep = 5 * time.Millisecond
+	testCases := []struct {
+		rule      *AlertingRule
+		steps     [][]datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestAlertingRule("empty", 0),
+			[][]datasource.Metric{},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestAlertingRule("empty labels", 0),
+			[][]datasource.Metric{
+				{datasource.Metric{}},
+			},
+			map[uint64]*notifier.Alert{
+				hash(datasource.Metric{}): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+			},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-firing", 0),
+			[][]datasource.Metric{
+				{
+					metricWithLabels(t, "name", "foo"),
+					metricWithLabels(t, "name", "foo1"),
+					metricWithLabels(t, "name", "foo2"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")):  {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-steps-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo1")},
+				{metricWithLabels(t, "name", "foo2")},
+			},
+			// 1: fire first alert
+			// 2: fire second alert, set first inactive
+			// 3: fire third alert, set second inactive, delete first one
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo1")): {State: notifier.StateInactive},
+				hash(metricWithLabels(t, "name", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("duplicate", 0),
+			[][]datasource.Metric{
+				{
+					// metrics with the same labelset should result in one alert
+					metricWithLabels(t, "name", "foo", "type", "bar"),
+					metricWithLabels(t, "type", "bar", "name", "foo"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo", "type", "bar")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending", time.Minute),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestAlertingRule("for-fired", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>empty", time.Second),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset and delete pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "name", "foo")): {State: notifier.StateFiring},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.GroupID = fakeGroup.ID()
+			for _, step := range tc.steps {
+				fq.reset()
+				fq.add(step...)
+				if _, err := tc.rule.Exec(context.TODO(), fq, false); err != nil {
+					t.Fatalf("unexpected err: %s", err)
+				}
+				// artificial delay between applying steps
+				time.Sleep(defaultStep)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Restore(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		metrics   []datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestRuleWithLabels("no extra labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(datasource.Metric{}): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("metric labels"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+					"foo", "bar",
+					"namespace", "baz",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t,
+					"foo", "bar",
+					"namespace", "baz",
+				)): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("rule labels", "source", "vm"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					alertNameLabel, "",
+					"foo", "bar",
+					"namespace", "baz",
+					// following pair supposed to be dropped
+					"source", "vm",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t,
+					"foo", "bar",
+					"namespace", "baz",
+				)): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+			},
+		},
+		{
+			newTestRuleWithLabels("multiple alerts"),
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-1",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(2*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-2",
+				),
+				metricWithValueAndLabels(t, float64(time.Now().Truncate(3*time.Hour).Unix()),
+					"__name__", alertForStateMetricName,
+					"host", "localhost-3",
+				),
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "host", "localhost-1")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(time.Hour)},
+				hash(metricWithLabels(t, "host", "localhost-2")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(2 * time.Hour)},
+				hash(metricWithLabels(t, "host", "localhost-3")): {State: notifier.StatePending,
+					Start: time.Now().Truncate(3 * time.Hour)},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.GroupID = fakeGroup.ID()
+			fq.add(tc.metrics...)
+			if err := tc.rule.Restore(context.TODO(), fq, time.Hour); err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+				if got.Start != exp.Start {
+					t.Fatalf("expected Start %v; got %v", exp.Start, got.Start)
+				}
+			}
+		})
+	}
+}
+
+func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
+	r := newTestAlertingRule(name, 0)
+	r.Labels = make(map[string]string)
+	for i := 0; i < len(labels); i += 2 {
+		r.Labels[labels[i]] = labels[i+1]
+	}
+	return r
+}
+
+func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
+	return &AlertingRule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
+}