app/vmselect: adds traces for series update API

VictoriaMetrics_ProjectID and VictoriaMetrics_AccountID labels.
It should help to migrate for new labels vm_account_id vm_project_id without service downtime

.dockerignore

@@ -4,3 +4,4 @@ gocache-for-docker
 victoria-metrics-data
 vmstorage-data
 vmselect-cache
+.vscode

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,86 @@
+name: Bug report
+description: Create a report to help us improve
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
+        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+        and verify whether the bug is reproducible there.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
+  - type: textarea
+    id: describe-the-bug
+    attributes:
+      label: Describe the bug
+      description: |
+        A clear and concise description of what the bug is.
+      placeholder: |
+        When I do `A` VictoriaMetrics does `B`. I expect it to do `C`.
+    validations:
+      required: true
+  - type: textarea
+    id: to-reproduce
+    attributes:
+      label: To Reproduce
+      description: |
+        Steps to reproduce the behavior.
+        If reproducing an issue requires some specific configuration file, please paste it here.
+      placeholder: |
+        Steps to reproduce the behavior.
+    validations:
+      required: true
+  - type: textarea
+    id: version
+    attributes:
+      label: Version
+      description: |
+        The line returned when passing `--version` command line flag to the binary. For example:
+        ```
+        $ ./victoria-metrics-prod --version
+        victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
+        ```
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs
+      description: |
+        Check if any warnings or errors were logged by VictoriaMetrics components
+        or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
+    validations:
+      required: false
+  - type: textarea
+    id: screenshots
+    attributes:
+      label: Screenshots
+      description: |
+        If applicable, add screenshots to help explain your problem.
+  
+        For VictoriaMetrics health-state issues please provide full-length screenshots
+        of Grafana dashboards if possible:
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
+        
+        See how to setup monitoring here:
+          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
+    validations:
+      required: false
+  - type: textarea
+    id: flags
+    attributes:
+      label: Used command-line flags
+      description: |
+        Please provide the command-line flags used for running VictoriaMetrics and its components.
+    validations:
+      required: false
+  - type: textarea
+    id: additional-info
+    attributes:
+      label: Additional information
+      placeholder: |
+        Additional information that doesn't fit elsewhere
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/configuration.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Ask on Slack
+    url: https://slack.victoriametrics.com/
+    about: You can ask for help here!

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,43 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: [enhancement]
+body:
+  - type: textarea
+    id: describe-the-problem
+    attributes:
+      label: Is your feature request related to a problem? Please describe
+      description: |
+        A clear and concise description of what the problem is.
+      placeholder: |
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: false
+  - type: textarea
+    id: describe-the-solution
+    attributes:
+      label: Describe the solution you'd like
+      description: |
+        A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative-solutions
+    attributes:
+      label: Describe alternatives you've considered
+      description: |
+        A clear and concise description of any alternative solutions or features you've considered.
+      placeholder: |
+        I have tried to do `A`, but that doesn't solve a problem completely.
+        I have tried to do `A` and `B`, but implementing this would be better.
+    validations:
+      required: false
+  - type: textarea
+    id: feature-additional-info
+    attributes:
+      label: Additional information
+      description: |
+        Additional information which you consider helpful for implementing this feature.
+      placeholder: |
+        Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/question.yml

@@ -0,0 +1,32 @@
+name: Question
+description: Ask a question regarding VictoriaMetrics or its components
+labels: [question]
+body:
+  - type: textarea
+    id: describe-the-component
+    attributes:
+      label: Is your question request related to a specific component?
+      placeholder: |
+        VictoriaMetrics, vmagent, vmalert, vmui, etc...
+    validations:
+      required: false
+  - type: textarea
+    id: describe-the-question
+    attributes:
+      label: Describe the question in detail
+      description: |
+        A clear and concise description of the issue and the question.
+    validations:
+      required: true
+  - type: checkboxes
+    id: troubleshooting
+    attributes:
+      label: Troubleshooting docs
+      description: I am familiar with the following troubleshooting docs
+      options:
+        - label: General - https://docs.victoriametrics.com/Troubleshooting.html
+          required: false
+        - label: vmagent - https://docs.victoriametrics.com/vmagent.html#troubleshooting
+          required: false
+        - label: vmalert - https://docs.victoriametrics.com/vmalert.html#troubleshooting
+          required: false

.github/dependabot.yml

@@ -0,0 +1,28 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "bundler"
+    directory: "/docs"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/app/vmui/packages/vmui/web"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/app/vmui/packages/vmui"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0

.github/workflows/check-licenses.yml

@@ -0,0 +1,26 @@
+name: license-check
+on:
+  push:
+    paths:
+      - 'vendor'
+  pull_request:
+    paths:
+      - 'vendor'
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.20.2
+        id: go
+      - name: Code checkout
+        uses: actions/checkout@master
+      - name: Check License
+        run: |
+          make check-licenses

.github/workflows/codeql-analysis-js.yml

@@ -0,0 +1,46 @@
+name: "CodeQL - JS"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+  
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["javascript"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["go"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.2
+          check-latest: true
+          cache: true
+        if: ${{ matrix.language == 'go' }}
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -0,0 +1,95 @@
+name: main
+on:
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.2
+          check-latest: true
+          cache: true
+
+      - name: Dependencies
+        run: |
+          make install-golangci-lint
+          make check-all
+          git diff --exit-code
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        scenario: ["test-full", "test-pure", "test-full-386"]
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.2
+          check-latest: true
+          cache: true
+
+      - name: run tests
+        run: |
+          make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.txt
+
+  build:
+    needs: test
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.2
+          check-latest: true
+          cache: true
+
+      - uses: actions/cache@v3
+        with:
+          path: gocache-for-docker
+          key:  gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
+
+      - name: Build
+        run: |
+          make vmcluster-crossbuild

.gitignore

@@ -1,11 +1,23 @@
+/tmp
 /tags
 /pkg
 *.pprof
 /bin
 .idea
+.vscode
 *.test
 *.swp
 /gocache-for-docker
 /victoria-metrics-data
+/vmagent-remotewrite-data
 /vmstorage-data
 /vmselect-cache
+/package/temp-deb-*
+/package/temp-rpm-*
+/package/*.deb
+/package/*.rpm
+.DS_store
+Gemfile.lock
+/_site
+_site
+*.tmp

.golangci.yml

@@ -0,0 +1,19 @@
+run:
+  timeout: 2m
+
+linters:
+  enable:
+  - revive
+
+issues:
+  exclude-rules:
+  - linters:
+    - staticcheck
+    text: "SA(4003|1019|5011):"
+  include:
+  - EXC0012
+  - EXC0014
+
+linters-settings:
+  errcheck:
+    exclude: ./errcheck_excludes.txt

.wwhrd.yml

@@ -0,0 +1,6 @@
+allowlist:
+  - Apache-2.0
+  - MIT
+  - BSD-3-Clause
+  - BSD-2-Clause
+  - ISC

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues and other contributions
+that are not aligned to this Code of Conduct or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing or otherwise unacceptable behavior may be
+reported by contacting the project team at info@victoriametrics.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate for the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at <https://www.contributor-covenant.org/version/1/4/code-of-conduct.html>
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+<https://www.contributor-covenant.org/faq>

CODE_OF_CONDUCT_RU.md

@@ -0,0 +1,120 @@
+
+# Кодекс Поведения участника
+
+## Наши обязательства
+
+Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
+свободным от притеснений для всех, независимо от возраста, телосложения,
+видимых или невидимых ограничений способности, этнической принадлежности,
+половых признаков, гендерной идентичности и выражения, уровня опыта,
+образования, социо-экономического статуса, национальности, внешности,
+расы, религии, или сексуальной идентичности и ориентации.
+
+Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
+дружелюбное, многообразное, инклюзивное и здоровое сообщество.
+
+## Наши стандарты
+
+Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
+
+* Проявление доброты и эмпатии к другим участникам проекта
+* Уважение к чужой точке зрения и опыту
+* Конструктивная критика и принятие конструктивной критики
+* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
+  и извлечение уроков из опыта
+* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
+
+Примеры неприемлемого поведения участников включают в себя:
+
+* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
+* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
+* Публичное или приватное домогательство
+* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
+* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
+
+## Обязанности
+
+Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
+поведения и будут предпринимать соответствующие и честные меры по исправлению положения
+в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
+
+Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
+комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
+с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
+
+## Область применения
+
+Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
+а также когда человек официально представляет сообщество в публичных местах.
+Примеры представления проекта или сообщества включают использование официальной электронной почты,
+публикации в официальном аккаунте в социальных сетях,
+или упоминания как представителя в онлайн или оффлайн мероприятии.
+
+## Приведение в исполнение
+
+О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
+поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
+Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
+
+Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
+неприкосновенность автора сообщения.
+
+## Руководство по исполнению
+
+Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
+чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
+
+### 1. Исправление
+
+**Общественное влияние**: Использование недопустимой лексики или другое поведение,
+считающиеся непрофессиональным или нежелательным в сообществе.
+
+**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
+объясняющее суть нарушения и почему такое поведение
+было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
+
+### 2. Предупреждение
+
+**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
+
+**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
+На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
+включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
+в публичных пространствах, а так же во внешних каналах,
+таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
+
+### 3. Временный бан
+
+**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
+включая продолжительное неуместное поведение.
+
+**Последствия**: Временный запрет (бан) на любое взаимодействие
+или публичное общение с сообществом на определенный период времени.
+На этот период не допускается публичное или личное взаимодействие с людьми,
+вовлеченными в инцидент, включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса.
+Нарушение этих правил влечет за собой вечный бан.
+
+### 4. Вечный бан
+
+**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
+включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
+или проявление агрессии либо пренебрежительного отношения к категориям лиц.
+
+**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
+
+## Атрибуция
+
+Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
+версии 2.0, доступной по адресу
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
+
+Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+If you like VictoriaMetrics and want to contribute, then we need the following:
+
+- Filing issues and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+- Spreading a word about VictoriaMetrics: conference talks, articles, comments, experience sharing with colleagues.
+- Updating documentation.
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019 VictoriaMetrics, Inc.
+   Copyright 2019-2023 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,64 +1,236 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
+DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | sha1sum | grep -oP '^.{8}')))
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
+LATEST_TAG ?= cluster-latest
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
 PKG_TAG := $(BUILDINFO_TAG)
 endif
 
-GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date -u +'%Y%m%d-%H%M%S')-$(BUILDINFO_TAG)'
+GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 
-all: \
-	victoria-metrics-prod
+.PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
 include deployment/*/Makefile
+include package/release/Makefile
+
+all: \
+	vminsert \
+	vmselect \
+	vmstorage
+
+all-pure: \
+	vminsert-pure \
+	vmselect-pure \
+	vmstorage-pure
 
 clean:
 	rm -rf bin/*
 
-release: victoria-metrics-prod
-	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod
+vmcluster-linux-amd64: \
+	vminsert-linux-amd64 \
+	vmselect-linux-amd64 \
+	vmstorage-linux-amd64
+
+vmcluster-linux-arm64: \
+	vminsert-linux-arm64 \
+	vmselect-linux-arm64 \
+	vmstorage-linux-arm64
+
+vmcluster-linux-arm: \
+	vminsert-linux-arm \
+	vmselect-linux-arm \
+	vmstorage-linux-arm
+
+vmcluster-linux-ppc64le: \
+	vminsert-linux-ppc64le \
+	vmselect-linux-ppc64le \
+	vmstorage-linux-ppc64le
+
+vmcluster-linux-386: \
+	vminsert-linux-386 \
+	vmselect-linux-386 \
+	vmstorage-linux-386
+
+vmcluster-freebsd-amd64: \
+	vminsert-freebsd-amd64 \
+	vmselect-freebsd-amd64 \
+	vmstorage-freebsd-amd64
+
+vmcluster-openbsd-amd64: \
+	vminsert-openbsd-amd64 \
+	vmselect-openbsd-amd64 \
+	vmstorage-openbsd-amd64
+
+vmcluster-crossbuild: \
+	vmcluster-linux-amd64 \
+	vmcluster-linux-arm64 \
+	vmcluster-linux-arm \
+	vmcluster-linux-ppc64le \
+	vmcluster-linux-386 \
+	vmcluster-freebsd-amd64 \
+	vmcluster-openbsd-amd64
+
+publish: docker-scan \
+	publish-vminsert \
+	publish-vmselect \
+	publish-vmstorage
+
+package: \
+	package-vminsert \
+	package-vmselect \
+	package-vmstorage
+
+publish-release:
+	git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && LATEST_TAG=enterprise-cluster-stable $(MAKE) release publish
+
+release: \
+	release-vmcluster
+
+release-vmcluster: \
+	release-vmcluster-linux-amd64 \
+	release-vmcluster-linux-arm64 \
+	release-vmcluster-freebsd-amd64 \
+	release-vmcluster-openbsd-amd64
+
+release-vmcluster-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
+
+release-vmcluster-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-vmcluster-goos-goarch
+
+release-vmcluster-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
+
+release-vmcluster-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
+
+release-vmcluster-goos-goarch: \
+	vminsert-$(GOOS)-$(GOARCH)-prod \
+	vmselect-$(GOOS)-$(GOARCH)-prod \
+	vmstorage-$(GOOS)-$(GOARCH)-prod
+	cd bin && \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vminsert-$(GOOS)-$(GOARCH)-prod \
+			vmselect-$(GOOS)-$(GOARCH)-prod \
+			vmstorage-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			vminsert-$(GOOS)-$(GOARCH)-prod \
+			vmselect-$(GOOS)-$(GOARCH)-prod \
+			vmstorage-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		vminsert-$(GOOS)-$(GOARCH)-prod \
+		vmselect-$(GOOS)-$(GOARCH)-prod \
+		vmstorage-$(GOOS)-$(GOARCH)-prod
+
+pprof-cpu:
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
-	go fmt $(PKG_PREFIX)/lib/...
-	go fmt $(PKG_PREFIX)/app/...
+	gofmt -l -w -s ./lib
+	gofmt -l -w -s ./app
 
 vet:
-	go vet $(PKG_PREFIX)/lib/...
-	go vet $(PKG_PREFIX)/app/...
+	go vet ./lib/...
+	go vet ./app/...
 
-lint: install-golint
-	golint lib/...
-	golint app/...
-
-install-golint:
-	which golint || GO111MODULE=off go get -u github.com/golang/lint/golint
-
-errcheck: install-errcheck
-	errcheck -exclude=errcheck_excludes.txt ./lib/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
-
-install-errcheck:
-	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
+check-all: fmt vet golangci-lint govulncheck
 
 test:
-	go test $(PKG_PREFIX)/lib/...
+	go test ./lib/... ./app/...
+
+test-race:
+	go test -race ./lib/... ./app/...
+
+test-pure:
+	CGO_ENABLED=0 go test ./lib/... ./app/...
+
+test-full:
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+
+test-full-386:
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	go test -bench=. $(PKG_PREFIX)/lib/...
+	go test -bench=. ./lib/...
+	go test -bench=. ./app/...
+
+benchmark-pure:
+	CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	go get -u
-	go mod tidy
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.19
 	go mod vendor
 
+app-local:
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-pure:
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-goos-goarch:
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-windows-goarch:
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+
 quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
-	which qtc || GO111MODULE=off go get -u github.com/valyala/quicktemplate/qtc
+	which qtc || go install github.com/valyala/quicktemplate/qtc@latest
+
+
+golangci-lint: install-golangci-lint
+	golangci-lint run
+
+install-golangci-lint:
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.1
+
+govulncheck: install-govulncheck
+	govulncheck ./...
+
+install-govulncheck:
+	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
+
+install-wwhrd:
+	which wwhrd || go install github.com/frapposelli/wwhrd@latest
+
+check-licenses: install-wwhrd
+	wwhrd check -f .wwhrd.yml
+
+copy-docs:
+	echo '' > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "---\nsort: ${ORDER}\n---\n" > ${DST}; \
+	fi
+	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	rm -rf docs/*.tmp
+
+# Copies docs for all components and adds the order tag.
+# For ORDER=0 it adds no order tag.
+# Images starting with <img src="docs/ are replaced with <img src="
+# Cluster docs are supposed to be ordered as 9th.
+# The rest of docs is ordered manually.
+docs-sync:
+	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md ORDER=2 $(MAKE) copy-docs
+	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=3 $(MAKE) copy-docs
+	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=4 $(MAKE) copy-docs
+	SRC=app/vmauth/README.md DST=docs/vmauth.md ORDER=5 $(MAKE) copy-docs
+	SRC=app/vmbackup/README.md DST=docs/vmbackup.md ORDER=6 $(MAKE) copy-docs
+	SRC=app/vmrestore/README.md DST=docs/vmrestore.md ORDER=7 $(MAKE) copy-docs
+	SRC=app/vmctl/README.md DST=docs/vmctl.md ORDER=8 $(MAKE) copy-docs
+	SRC=app/vmgateway/README.md DST=docs/vmgateway.md ORDER=9 $(MAKE) copy-docs
+	SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md ORDER=10 $(MAKE) copy-docs

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+|---------|--------------------|
+| 1.81.x  | :white_check_mark: |
+| 1.80.x  | :x:                |
+| 1.79.x  | :white_check_mark: |
+| < 1.78  | :x:                |
+
+## Reporting a Vulnerability
+
+Please report any security issues to security@victoriametrics.com

app/victoria-metrics/Makefile

@@ -1,21 +0,0 @@
-# All these commands must run from repository root.
-
-victoria-metrics-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker
-
-package-victoria-metrics:
-	APP_NAME=victoria-metrics \
-	$(MAKE) package-via-docker
-
-publish-victoria-metrics:
-	APP_NAME=victoria-metrics $(MAKE) publish-via-docker
-
-run-victoria-metrics:
-	mkdir -p victoria-metrics-data
-	DOCKER_OPTS='-v $(shell pwd)/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 -p 2003:2003 -p 2003:2003/udp' \
-	APP_NAME=victoria-metrics \
-	ARGS='-graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -retentionPeriod=12 -search.maxUniqueTimeseries=1000000 -search.maxQueryDuration=10m' \
-	$(MAKE) run-via-docker
-
-victoria-metrics-arm:
-	CC=arm-linux-gnueabi-gcc CGO_ENABLED=1 GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics

app/victoria-metrics/deployment/Dockerfile

@@ -1,5 +0,0 @@
-FROM scratch
-COPY --from=local/certs:1.0.2 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY bin/victoria-metrics-prod .
-EXPOSE 8428
-ENTRYPOINT ["/victoria-metrics-prod"]

app/victoria-metrics/main.go

@@ -1,60 +0,0 @@
-package main
-
-import (
-	"flag"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-)
-
-var httpListenAddr = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
-
-func main() {
-	flag.Parse()
-	buildinfo.Init()
-	logger.Init()
-	logger.Infof("starting VictoraMetrics at %q...", *httpListenAddr)
-	startTime := time.Now()
-	vmstorage.Init()
-	vmselect.Init()
-	vminsert.Init()
-
-	go httpserver.Serve(*httpListenAddr, requestHandler)
-	logger.Infof("started VictoriaMetrics in %s", time.Since(startTime))
-
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-
-	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
-	startTime = time.Now()
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	vminsert.Stop()
-	logger.Infof("successfully shut down the webservice in %s", time.Since(startTime))
-
-	vmstorage.Stop()
-	vmselect.Stop()
-
-	logger.Infof("the VictoriaMetrics has been stopped in %s", time.Since(startTime))
-}
-
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if vminsert.RequestHandler(w, r) {
-		return true
-	}
-	if vmselect.RequestHandler(w, r) {
-		return true
-	}
-	if vmstorage.RequestHandler(w, r) {
-		return true
-	}
-	return false
-}

app/vmagent/Makefile

@@ -0,0 +1,110 @@
+# All these commands must run from repository root.
+
+vmagent:
+	APP_NAME=vmagent $(MAKE) app-local
+
+vmagent-race:
+	APP_NAME=vmagent RACE=-race $(MAKE) app-local
+
+vmagent-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker
+
+vmagent-pure-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-pure
+
+vmagent-linux-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-amd64
+
+vmagent-linux-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm
+
+vmagent-linux-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-arm64
+
+vmagent-linux-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-ppc64le
+
+vmagent-linux-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-386
+
+vmagent-darwin-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
+
+vmagent-darwin-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-arm64
+
+vmagent-freebsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-freebsd-amd64
+
+vmagent-openbsd-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-openbsd-amd64
+
+vmagent-windows-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-windows-amd64
+
+package-vmagent:
+	APP_NAME=vmagent $(MAKE) package-via-docker
+
+package-vmagent-pure:
+	APP_NAME=vmagent $(MAKE) package-via-docker-pure
+
+package-vmagent-amd64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-amd64
+
+package-vmagent-arm:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm
+
+package-vmagent-arm64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm64
+
+package-vmagent-ppc64le:
+	APP_NAME=vmagent $(MAKE) package-via-docker-ppc64le
+
+package-vmagent-386:
+	APP_NAME=vmagent $(MAKE) package-via-docker-386
+
+publish-vmagent:
+	APP_NAME=vmagent $(MAKE) publish-via-docker
+
+run-vmagent:
+	mkdir -p vmagent-remotewrite-data
+	DOCKER_OPTS='-v $(shell pwd)/vmagent-remotewrite-data:/vmagent-remotewrite-data' \
+	ARGS='-remoteWrite.url=http://localhost:8428/api/v1/write' \
+	APP_NAME=vmagent \
+	$(MAKE) run-via-docker
+
+vmagent-linux-amd64:
+	APP_NAME=vmagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-linux-arm:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+
+vmagent-linux-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmagent-linux-ppc64le:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+
+vmagent-linux-s390x:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
+vmagent-linux-386:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+vmagent-darwin-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-darwin-arm64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmagent-freebsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-openbsd-amd64:
+	APP_NAME=vmagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmagent-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmagent $(MAKE) app-local-windows-goarch
+
+vmagent-pure:
+	APP_NAME=vmagent $(MAKE) app-local-pure

app/vmagent/common/push_ctx.go

@@ -0,0 +1,66 @@
+package common
+
+import (
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+// PushCtx is a context used for populating WriteRequest.
+type PushCtx struct {
+	WriteRequest prompbmarshal.WriteRequest
+
+	// Labels contains flat list of all the labels used in WriteRequest.
+	Labels []prompbmarshal.Label
+
+	// Samples contains flat list of all the samples used in WriteRequest.
+	Samples []prompbmarshal.Sample
+}
+
+// Reset resets ctx.
+func (ctx *PushCtx) Reset() {
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
+
+	promrelabel.CleanLabels(ctx.Labels)
+	ctx.Labels = ctx.Labels[:0]
+
+	ctx.Samples = ctx.Samples[:0]
+}
+
+// GetPushCtx returns PushCtx from pool.
+//
+// Call PutPushCtx when the ctx is no longer needed.
+func GetPushCtx() *PushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
+	}
+}
+
+// PutPushCtx returns ctx to the pool.
+//
+// ctx mustn't be used after returning to the pool.
+func PutPushCtx(ctx *PushCtx) {
+	ctx.Reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, cgroup.AvailableCPUs())

app/vmagent/csvimport/request_handler.go

@@ -0,0 +1,75 @@
+package csvimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="csvimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+)
+
+// InsertHandler processes csv data from req.
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return stream.Parse(req, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(len(rows))
+	}
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/datadog/request_handler.go

@@ -0,0 +1,98 @@
+package datadog
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	ce := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, ce, func(series []parser.Series) error {
+		return insertRows(at, series, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		if ss.Host != "" {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "host",
+				Value: ss.Host,
+			})
+		}
+		if ss.Device != "" {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "device",
+				Value: ss.Device,
+			})
+		}
+		for _, tag := range ss.Tags {
+			name, value := parser.SplitTag(tag)
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp(),
+				Value:     pt.Value(),
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8429
+
+ENTRYPOINT ["/vmagent-prod"]
+ARG src_binary
+COPY $src_binary ./vmagent-prod

app/vmagent/graphite/request_handler.go

@@ -0,0 +1,63 @@
+package graphite
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="graphite"}`)
+)
+
+// InsertHandler processes remote write for graphite plaintext protocol.
+//
+// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
+func InsertHandler(r io.Reader) error {
+	return stream.Parse(r, insertRows)
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(nil, &ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -0,0 +1,182 @@
+package influx
+
+import (
+	"flag"
+	"io"
+	"net/http"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
+	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
+	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="influx"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+)
+
+// InsertHandlerForReader processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+		return insertRows(nil, db, rows, nil)
+	})
+}
+
+// InsertHandlerForHTTP processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	q := req.URL.Query()
+	precision := q.Get("precision")
+	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	db := q.Get("db")
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+		return insertRows(at, db, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.ctx.Labels[:0]
+	samples := ctx.ctx.Samples[:0]
+	commonLabels := ctx.commonLabels[:0]
+	buf := ctx.buf[:0]
+	for i := range rows {
+		r := &rows[i]
+		rowsTotal += len(r.Fields)
+		commonLabels = commonLabels[:0]
+		hasDBKey := false
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if tag.Key == *dbLabel {
+				hasDBKey = true
+			}
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		if len(db) > 0 && !hasDBKey {
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  *dbLabel,
+				Value: db,
+			})
+		}
+		commonLabels = append(commonLabels, extraLabels...)
+		ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+		if !*skipMeasurement {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, r.Measurement...)
+		}
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1139
+		skipFieldKey := len(r.Measurement) > 0 && len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
+		for j := range r.Fields {
+			f := &r.Fields[j]
+			bufLen := len(buf)
+			buf = append(buf, ctx.metricGroupBuf...)
+			if !skipFieldKey {
+				buf = append(buf, f.Key...)
+			}
+			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: metricGroup,
+			})
+			labels = append(labels, commonLabels...)
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: r.Timestamp,
+				Value:     f.Value,
+			})
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[len(samples)-1:],
+			})
+		}
+	}
+	ctx.buf = buf
+	ctx.ctx.WriteRequest.Timeseries = tssDst
+	ctx.ctx.Labels = labels
+	ctx.ctx.Samples = samples
+	ctx.commonLabels = commonLabels
+	remotewrite.Push(at, &ctx.ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+
+	return nil
+}
+
+type pushCtx struct {
+	ctx            common.PushCtx
+	commonLabels   []prompbmarshal.Label
+	metricGroupBuf []byte
+	buf            []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.ctx.Reset()
+
+	promrelabel.CleanLabels(ctx.commonLabels)
+	ctx.commonLabels = ctx.commonLabels[:0]
+
+	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+	ctx.buf = ctx.buf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, cgroup.AvailableCPUs())

app/vmagent/main.go

@@ -0,0 +1,592 @@
+package main
+
+import (
+	"embed"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
+		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
+		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
+		"See also -influxListenAddr.useProxyProtocol")
+	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. "+
+		"See also -graphiteListenAddr.useProxyProtocol")
+	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
+	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
+	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
+		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	dryRun        = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
+)
+
+var (
+	influxServer       *influxserver.Server
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
+)
+
+var (
+	//go:embed static
+	staticFiles  embed.FS
+	staticServer = http.FileServer(http.FS(staticFiles))
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	remotewrite.InitSecretFlags()
+	buildinfo.Init()
+	logger.Init()
+	pushmetrics.Init()
+
+	if promscrape.IsDryRun() {
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		}
+		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
+		return
+	}
+	if *dryRun {
+		if err := remotewrite.CheckRelabelConfigs(); err != nil {
+			logger.Fatalf("error when checking relabel configs: %s", err)
+		}
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
+		}
+		logger.Infof("all the configs are ok; exiting with 0 status code")
+		return
+	}
+
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	startTime := time.Now()
+	remotewrite.Init()
+	common.StartUnmarshalWorkers()
+	if len(*influxListenAddr) > 0 {
+		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
+			return influx.InsertHandlerForReader(r, false)
+		})
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
+	}
+
+	promscrape.Init(remotewrite.Push)
+
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	}
+	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	startTime = time.Now()
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	}
+
+	promscrape.Stop()
+
+	if len(*influxListenAddr) > 0 {
+		influxServer.MustStop()
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer.MustStop()
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer.MustStop()
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
+	}
+	common.StopUnmarshalWorkers()
+	remotewrite.Stop()
+
+	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
+	if !remotewrite.MultitenancyEnabled() {
+		return func(req *http.Request) error {
+			path := strings.Replace(req.URL.Path, "//", "/", -1)
+			if path != "/api/put" {
+				return fmt.Errorf("unsupported path requested: %q; expecting '/api/put'", path)
+			}
+			return opentsdbhttp.InsertHandler(nil, req)
+		}
+	}
+	return func(req *http.Request) error {
+		path := strings.Replace(req.URL.Path, "//", "/", -1)
+		at, err := getAuthTokenFromPath(path)
+		if err != nil {
+			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
+		}
+		return opentsdbhttp.InsertHandler(at, req)
+	}
+}
+
+func getAuthTokenFromPath(path string) (*auth.Token, error) {
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse multitenant path: %w", err)
+	}
+	if p.Prefix != "insert" {
+		return nil, fmt.Errorf(`unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
+	}
+	if p.Suffix != "opentsdb/api/put" {
+		return nil, fmt.Errorf("unsupported path requested: %q; expecting 'opentsdb/api/put'", p.Suffix)
+	}
+	return auth.NewToken(p.AuthToken)
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.URL.Path == "/" {
+		if r.Method != http.MethodGet {
+			return false
+		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprintf(w, "<h2>vmagent</h2>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
+		fmt.Fprintf(w, "Useful endpoints:</br>")
+		httpserver.WriteAPIHelp(w, [][2]string{
+			{"targets", "status for discovered active targets"},
+			{"service-discovery", "labels before and after relabeling for discovered targets"},
+			{"metric-relabel-debug", "debug metric relabeling"},
+			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
+			{"config", "-promscrape.config contents"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+			{"-/reload", "reload configuration"},
+		})
+		return true
+	}
+
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus") || strings.HasPrefix(path, "/api/v1/import/prometheus") {
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(nil, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		statusCode := http.StatusNoContent
+		if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus/metrics/job/") ||
+			strings.HasPrefix(path, "/api/v1/import/prometheus/metrics/job/") {
+			// Return 200 status code for pushgateway requests.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
+			statusCode = http.StatusOK
+		}
+		w.WriteHeader(statusCode)
+		return true
+	}
+	if strings.HasPrefix(path, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		path = strings.TrimSuffix(path, "/")
+	}
+	switch path {
+	case "/prometheus/api/v1/write", "/api/v1/write":
+		if common.HandleVMProtoServerHandshake(w, r) {
+			return true
+		}
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(nil, r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/prometheus/api/v1/import", "/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(nil, r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/prometheus/api/v1/import/csv", "/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(nil, r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/prometheus/api/v1/import/native", "/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(nil, r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/influx/write", "/influx/api/v2/write", "/write", "/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/influx/query", "/query":
+		influxQueryRequests.Inc()
+		influxutils.WriteDatabaseNames(w)
+		return true
+	case "/datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "/datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "/datadog/intake":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
+	case "/datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
+	case "/prometheus/targets", "/targets":
+		promscrapeTargetsRequests.Inc()
+		promscrape.WriteHumanReadableTargetsStatus(w, r)
+		return true
+	case "/prometheus/service-discovery", "/service-discovery":
+		promscrapeServiceDiscoveryRequests.Inc()
+		promscrape.WriteServiceDiscovery(w, r)
+		return true
+	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
+		promscrapeMetricRelabelDebugRequests.Inc()
+		promscrape.WriteMetricRelabelDebug(w, r)
+		return true
+	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
+		promscrapeTargetRelabelDebugRequests.Inc()
+		promscrape.WriteTargetRelabelDebug(w, r)
+		return true
+	case "/prometheus/api/v1/targets", "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
+		return true
+	case "/prometheus/target_response", "/target_response":
+		promscrapeTargetResponseRequests.Inc()
+		if err := promscrape.WriteTargetResponse(w, r); err != nil {
+			promscrapeTargetResponseErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
+	case "/prometheus/config", "/config":
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+			return true
+		}
+		promscrapeConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		promscrape.WriteConfigData(w)
+		return true
+	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
+		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
+			return true
+		}
+		promscrapeStatusConfigRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		var bb bytesutil.ByteBuffer
+		promscrape.WriteConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
+		return true
+	case "/prometheus/-/reload", "/-/reload":
+		promscrapeConfigReloadRequests.Inc()
+		procutil.SelfSIGHUP()
+		w.WriteHeader(http.StatusOK)
+		return true
+	case "/ready":
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
+			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
+			http.Error(w, errMsg, http.StatusTooEarly)
+		} else {
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("OK"))
+		}
+		return true
+	default:
+		if strings.HasPrefix(r.URL.Path, "/static") {
+			staticServer.ServeHTTP(w, r)
+			return true
+		}
+		if remotewrite.MultitenancyEnabled() {
+			return processMultitenantRequest(w, r, path)
+		}
+		return false
+	}
+}
+
+func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
+	p, err := httpserver.ParsePath(path)
+	if err != nil {
+		// Cannot parse multitenant path. Skip it - probably it will be parsed later.
+		return false
+	}
+	if p.Prefix != "insert" {
+		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
+		return true
+	}
+	at, err := auth.NewToken(p.AuthToken)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
+		return true
+	}
+	if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus") {
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(at, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	}
+	if strings.HasPrefix(p.Suffix, "datadog/") {
+		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
+		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
+	}
+	switch p.Suffix {
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(at, r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(at, r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(at, r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "prometheus/api/v1/import/native":
+		nativeimportRequests.Inc()
+		if err := native.InsertHandler(at, r); err != nil {
+			nativeimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/write", "influx/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(at, r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "influx/query":
+		influxQueryRequests.Inc()
+		influxutils.WriteDatabaseNames(w)
+		return true
+	case "datadog/api/v1/series":
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/api/v1/validate":
+		datadogValidateRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"valid":true}`)
+		return true
+	case "datadog/api/v1/check_run":
+		datadogCheckRunRequests.Inc()
+		// See https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(202)
+		fmt.Fprintf(w, `{"status":"ok"}`)
+		return true
+	case "datadog/intake":
+		datadogIntakeRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
+	case "datadog/api/v1/metadata":
+		datadogMetadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{}`)
+		return true
+	default:
+		httpserver.Errorf(w, r, "unsupported multitenant path suffix: %q", p.Suffix)
+		return true
+	}
+}
+
+var (
+	prometheusWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/write", protocol="promremotewrite"}`)
+
+	vmimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import", protocol="vmimport"}`)
+	vmimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import", protocol="vmimport"}`)
+
+	csvimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+	csvimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+
+	prometheusimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
+	prometheusimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/prometheus", protocol="prometheusimport"}`)
+
+	nativeimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+	nativeimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/native", protocol="nativeimport"}`)
+
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
+
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+
+	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
+	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
+	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
+	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
+
+	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
+
+	promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/metric-relabel-debug"}`)
+	promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target-relabel-debug"}`)
+
+	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+
+	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
+	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)
+
+	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
+	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)
+
+	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
+)
+
+func usage() {
+	const s = `
+vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
+
+See the docs at https://docs.victoriametrics.com/vmagent.html .
+`
+	flagutil.Usage(s)
+}

app/vmagent/multiarch/Dockerfile

@@ -0,0 +1,12 @@
+# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
+ARG certs_image
+ARG root_image
+FROM $certs_image as certs
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
+
+FROM $root_image
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+EXPOSE 8429
+ENTRYPOINT ["/vmagent-prod"]
+ARG TARGETARCH
+COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod

app/vmagent/native/request_handler.go

@@ -0,0 +1,89 @@
+package native
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="native"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	isGzip := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
+		return insertRows(at, block, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	// Update rowsInserted and rowsPerInsert before actual inserting,
+	// since relabeling can prevent from inserting the rows.
+	rowsLen := len(block.Values)
+	rowsInserted.Add(rowsLen)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsLen)
+	}
+	rowsPerInsert.Update(float64(rowsLen))
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	mn := &block.MetricName
+	labelsLen := len(labels)
+	labels = append(labels, prompbmarshal.Label{
+		Name:  "__name__",
+		Value: bytesutil.ToUnsafeString(mn.MetricGroup),
+	})
+	for j := range mn.Tags {
+		tag := &mn.Tags[j]
+		labels = append(labels, prompbmarshal.Label{
+			Name:  bytesutil.ToUnsafeString(tag.Key),
+			Value: bytesutil.ToUnsafeString(tag.Value),
+		})
+	}
+	labels = append(labels, extraLabels...)
+	values := block.Values
+	timestamps := block.Timestamps
+	if len(timestamps) != len(values) {
+		logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+	}
+	samplesLen := len(samples)
+	for j, value := range values {
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     value,
+			Timestamp: timestamps[j],
+		})
+	}
+	tssDst = append(tssDst, prompbmarshal.TimeSeries{
+		Labels:  labels[labelsLen:],
+		Samples: samples[samplesLen:],
+	})
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	return nil
+}

app/vmagent/opentsdb/request_handler.go

@@ -0,0 +1,63 @@
+package opentsdb
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdb"}`)
+)
+
+// InsertHandler processes remote write for OpenTSDB put protocol.
+//
+// See http://opentsdb.net/docs/build/html/api_telnet/put.html
+func InsertHandler(r io.Reader) error {
+	return stream.Parse(r, insertRows)
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(nil, &ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/opentsdbhttp/request_handler.go

@@ -0,0 +1,71 @@
+package opentsdbhttp
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdbhttp"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdbhttp"}`)
+)
+
+// InsertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return stream.Parse(req, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/prometheusimport/request_handler.go

@@ -0,0 +1,83 @@
+package prometheusimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+)
+
+// InsertHandler processes `/api/v1/import/prometheus` request.
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	defaultTimestamp, err := parserCommon.GetTimestamp(req)
+	if err != nil {
+		return err
+	}
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
+	}, func(s string) {
+		httpserver.LogError(req, s)
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(len(rows))
+	}
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/prometheusimport/request_handler_test.go

@@ -0,0 +1,60 @@
+package prometheusimport
+
+import (
+	"bytes"
+	"flag"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+)
+
+var (
+	srv        *httptest.Server
+	testOutput *bytes.Buffer
+)
+
+func TestInsertHandler(t *testing.T) {
+	setUp()
+	defer tearDown()
+	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
+go_memstats_alloc_bytes_total 1`))
+	if err := InsertHandler(nil, req); err != nil {
+		t.Errorf("unxepected error %s", err)
+	}
+	expectedMsg := "cannot unmarshal Prometheus line"
+	if !strings.Contains(testOutput.String(), expectedMsg) {
+		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
+	}
+}
+
+func setUp() {
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(204)
+	}))
+	flag.Parse()
+	remoteWriteFlag := "remoteWrite.url"
+	if err := flag.Lookup(remoteWriteFlag).Value.Set(srv.URL); err != nil {
+		log.Fatalf("unable to set %q with value %q, err: %v", remoteWriteFlag, srv.URL, err)
+	}
+	logger.Init()
+	common.StartUnmarshalWorkers()
+	remotewrite.Init()
+	testOutput = &bytes.Buffer{}
+	logger.SetOutputForTests(testOutput)
+}
+
+func tearDown() {
+	common.StopUnmarshalWorkers()
+	srv.Close()
+	logger.ResetOutputForTest()
+	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()
+	fs.MustRemoveAll(tmpDataDir)
+
+}

app/vmagent/promremotewrite/request_handler.go

@@ -0,0 +1,79 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="promremotewrite"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
+		return insertRows(at, tss, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range timeseries {
+		ts := &timeseries[i]
+		rowsTotal += len(ts.Samples)
+		labelsLen := len(labels)
+		for i := range ts.Labels {
+			label := &ts.Labels[i]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for i := range ts.Samples {
+			sample := &ts.Samples[i]
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     sample.Value,
+				Timestamp: sample.Timestamp,
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -0,0 +1,479 @@
+package remotewrite
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
+	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
+
+	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
+		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
+		"is sent after temporary unavailability of the remote storage")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
+
+	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
+	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+		"to the corresponding -remoteWrite.url")
+	tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
+	tlsCAFile  = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
+		"By default system CA is used")
+	tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used")
+
+	headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
+
+	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
+	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
+		"The file is re-read every second")
+	bearerToken     = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
+	bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
+		"The token is re-read from the file every second")
+
+	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
+	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
+	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+
+	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
+		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
+	awsEC2Endpoint = flagutil.NewArrayString("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsSTSEndpoint = flagutil.NewArrayString("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRegion      = flagutil.NewArrayString("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRoleARN     = flagutil.NewArrayString("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsAccessKey   = flagutil.NewArrayString("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsService     = flagutil.NewArrayString("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"Defaults to \"aps\"")
+	awsSecretKey = flagutil.NewArrayString("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+)
+
+type client struct {
+	sanitizedURL   string
+	remoteWriteURL string
+
+	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
+	useVMProto bool
+
+	fq *persistentqueue.FastQueue
+	hc *http.Client
+
+	sendBlock func(block []byte) bool
+	authCfg   *promauth.Config
+	awsCfg    *awsapi.Config
+
+	rl rateLimiter
+
+	bytesSent       *metrics.Counter
+	blocksSent      *metrics.Counter
+	requestDuration *metrics.Histogram
+	requestsOKCount *metrics.Counter
+	errorsCount     *metrics.Counter
+	packetsDropped  *metrics.Counter
+	rateLimit       *metrics.Gauge
+	retriesCount    *metrics.Counter
+	sendDuration    *metrics.FloatCounter
+
+	wg     sync.WaitGroup
+	stopCh chan struct{}
+}
+
+func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
+	authCfg, err := getAuthConfig(argIdx)
+	if err != nil {
+		logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	tlsCfg := authCfg.NewTLSConfig()
+	awsCfg, err := getAWSAPIConfig(argIdx)
+	if err != nil {
+		logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	tr := &http.Transport{
+		DialContext:         statDial,
+		TLSClientConfig:     tlsCfg,
+		TLSHandshakeTimeout: 10 * time.Second,
+		MaxConnsPerHost:     2 * concurrency,
+		MaxIdleConnsPerHost: 2 * concurrency,
+		IdleConnTimeout:     time.Minute,
+		WriteBufferSize:     64 * 1024,
+	}
+	pURL := proxyURL.GetOptionalArg(argIdx)
+	if len(pURL) > 0 {
+		if !strings.Contains(pURL, "://") {
+			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
+		}
+		pu, err := url.Parse(pURL)
+		if err != nil {
+			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
+		}
+		tr.Proxy = http.ProxyURL(pu)
+	}
+	hc := &http.Client{
+		Transport: tr,
+		Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
+	}
+	c := &client{
+		sanitizedURL:   sanitizedURL,
+		remoteWriteURL: remoteWriteURL,
+		authCfg:        authCfg,
+		awsCfg:         awsCfg,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
+	}
+	c.sendBlock = c.sendBlockHTTP
+
+	useVMProto := forceVMProto.GetOptionalArg(argIdx)
+	usePromProto := forcePromProto.GetOptionalArg(argIdx)
+	if useVMProto && usePromProto {
+		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
+	}
+	if !useVMProto && !usePromProto {
+		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
+		doRequest := func(url string) (*http.Response, error) {
+			return c.doRequest(url, nil)
+		}
+		useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
+		if !useVMProto {
+			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
+				"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol", sanitizedURL)
+		}
+	}
+	c.useVMProto = useVMProto
+
+	return c
+}
+
+func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
+	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
+		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
+		c.rl.perSecondLimit = int64(bytesPerSec)
+	}
+	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
+
+	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
+	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
+	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
+		return float64(rateLimit.GetOptionalArgOrDefault(argIdx, 0))
+	})
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
+	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
+	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
+	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
+		return float64(*queues)
+	})
+	for i := 0; i < concurrency; i++ {
+		c.wg.Add(1)
+		go func() {
+			defer c.wg.Done()
+			c.runWorker()
+		}()
+	}
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
+}
+
+func (c *client) MustStop() {
+	close(c.stopCh)
+	c.wg.Wait()
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
+}
+
+func getAuthConfig(argIdx int) (*promauth.Config, error) {
+	headersValue := headers.GetOptionalArg(argIdx)
+	var hdrs []string
+	if headersValue != "" {
+		hdrs = strings.Split(headersValue, "^^")
+	}
+	username := basicAuthUsername.GetOptionalArg(argIdx)
+	password := basicAuthPassword.GetOptionalArg(argIdx)
+	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
+	var basicAuthCfg *promauth.BasicAuthConfig
+	if username != "" || password != "" || passwordFile != "" {
+		basicAuthCfg = &promauth.BasicAuthConfig{
+			Username:     username,
+			Password:     promauth.NewSecret(password),
+			PasswordFile: passwordFile,
+		}
+	}
+
+	token := bearerToken.GetOptionalArg(argIdx)
+	tokenFile := bearerTokenFile.GetOptionalArg(argIdx)
+
+	var oauth2Cfg *promauth.OAuth2Config
+	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
+	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
+	if clientSecretFile != "" || clientSecret != "" {
+		oauth2Cfg = &promauth.OAuth2Config{
+			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
+			ClientSecret:     promauth.NewSecret(clientSecret),
+			ClientSecretFile: clientSecretFile,
+			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
+			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
+		}
+	}
+
+	tlsCfg := &promauth.TLSConfig{
+		CAFile:             tlsCAFile.GetOptionalArg(argIdx),
+		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
+		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
+		ServerName:         tlsServerName.GetOptionalArg(argIdx),
+		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
+	}
+
+	opts := &promauth.Options{
+		BasicAuth:       basicAuthCfg,
+		BearerToken:     token,
+		BearerTokenFile: tokenFile,
+		OAuth2:          oauth2Cfg,
+		TLSConfig:       tlsCfg,
+		Headers:         hdrs,
+	}
+	authCfg, err := opts.NewConfig()
+	if err != nil {
+		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
+	}
+	return authCfg, nil
+}
+
+func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
+	if !awsUseSigv4.GetOptionalArg(argIdx) {
+		return nil, nil
+	}
+	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
+	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
+	region := awsRegion.GetOptionalArg(argIdx)
+	roleARN := awsRoleARN.GetOptionalArg(argIdx)
+	accessKey := awsAccessKey.GetOptionalArg(argIdx)
+	secretKey := awsSecretKey.GetOptionalArg(argIdx)
+	service := awsService.GetOptionalArg(argIdx)
+	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (c *client) runWorker() {
+	var ok bool
+	var block []byte
+	ch := make(chan bool, 1)
+	for {
+		block, ok = c.fq.MustReadBlock(block[:0])
+		if !ok {
+			return
+		}
+		go func() {
+			startTime := time.Now()
+			ch <- c.sendBlock(block)
+			c.sendDuration.Add(time.Since(startTime).Seconds())
+		}()
+		select {
+		case ok := <-ch:
+			if ok {
+				// The block has been sent successfully
+				continue
+			}
+			// Return unsent block to the queue.
+			c.fq.MustWriteBlock(block)
+			return
+		case <-c.stopCh:
+			// c must be stopped. Wait for a while in the hope the block will be sent.
+			graceDuration := 5 * time.Second
+			select {
+			case ok := <-ch:
+				if !ok {
+					// Return unsent block to the queue.
+					c.fq.MustWriteBlock(block)
+				}
+			case <-time.After(graceDuration):
+				// Return unsent block to the queue.
+				c.fq.MustWriteBlock(block)
+			}
+			return
+		}
+	}
+}
+
+func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
+	reqBody := bytes.NewBuffer(body)
+	req, err := http.NewRequest(http.MethodPost, url, reqBody)
+	if err != nil {
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
+	}
+	c.authCfg.SetHeaders(req, true)
+	h := req.Header
+	h.Set("User-Agent", "vmagent")
+	h.Set("Content-Type", "application/x-protobuf")
+	if c.useVMProto {
+		h.Set("Content-Encoding", "zstd")
+		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
+	} else {
+		h.Set("Content-Encoding", "snappy")
+		h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	}
+	if c.awsCfg != nil {
+		sigv4Hash := awsapi.HashHex(body)
+		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
+			// there is no need in retry, request will be rejected by client.Do and retried by code below
+			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
+		}
+	}
+	return c.hc.Do(req)
+}
+
+// sendBlockHTTP sends the given block to c.remoteWriteURL.
+//
+// The function returns false only if c.stopCh is closed.
+// Otherwise it tries sending the block to remote storage indefinitely.
+func (c *client) sendBlockHTTP(block []byte) bool {
+	c.rl.register(len(block), c.stopCh)
+	retryDuration := time.Second
+	retriesCount := 0
+
+again:
+	startTime := time.Now()
+	resp, err := c.doRequest(c.remoteWriteURL, block)
+	c.requestDuration.UpdateDuration(startTime)
+	if err != nil {
+		c.errorsCount.Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+			len(block), c.sanitizedURL, err, retryDuration.Seconds())
+		t := timerpool.Get(retryDuration)
+		select {
+		case <-c.stopCh:
+			timerpool.Put(t)
+			return false
+		case <-t.C:
+			timerpool.Put(t)
+		}
+		c.retriesCount.Inc()
+		goto again
+	}
+	statusCode := resp.StatusCode
+	if statusCode/100 == 2 {
+		_ = resp.Body.Close()
+		c.requestsOKCount.Inc()
+		c.bytesSent.Add(len(block))
+		c.blocksSent.Inc()
+		return true
+	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
+	if statusCode == 409 || statusCode == 400 {
+		body, err := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+				"failed to read response body: %s",
+				len(block), c.sanitizedURL, statusCode, err)
+		} else {
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+				len(block), c.sanitizedURL, statusCode, string(body))
+		}
+		// Just drop block on 409 and 400 status codes like Prometheus does.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
+		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
+		_ = resp.Body.Close()
+		c.packetsDropped.Inc()
+		return true
+	}
+
+	// Unexpected status code returned
+	retriesCount++
+	retryDuration *= 2
+	if retryDuration > time.Minute {
+		retryDuration = time.Minute
+	}
+	body, err := io.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if err != nil {
+		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
+	} else {
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
+			"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
+	}
+	t := timerpool.Get(retryDuration)
+	select {
+	case <-c.stopCh:
+		timerpool.Put(t)
+		return false
+	case <-t.C:
+		timerpool.Put(t)
+	}
+	c.retriesCount.Inc()
+	goto again
+}
+
+var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+
+type rateLimiter struct {
+	perSecondLimit int64
+
+	// mu protects budget and deadline from concurrent access.
+	mu sync.Mutex
+
+	// The current budget. It is increased by perSecondLimit every second.
+	budget int64
+
+	// The next deadline for increasing the budget by perSecondLimit
+	deadline time.Time
+
+	limitReached *metrics.Counter
+}
+
+func (rl *rateLimiter) register(dataLen int, stopCh <-chan struct{}) {
+	limit := rl.perSecondLimit
+	if limit <= 0 {
+		return
+	}
+
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+
+	for rl.budget <= 0 {
+		if d := time.Until(rl.deadline); d > 0 {
+			rl.limitReached.Inc()
+			t := timerpool.Get(d)
+			select {
+			case <-stopCh:
+				timerpool.Put(t)
+				return
+			case <-t.C:
+				timerpool.Put(t)
+			}
+		}
+		rl.budget += limit
+		rl.deadline = time.Now().Add(time.Second)
+	}
+	rl.budget -= int64(dataLen)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -0,0 +1,257 @@
+package remotewrite
+
+import (
+	"flag"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+)
+
+var (
+	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
+		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
+	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
+	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
+	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
+		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
+		"See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
+)
+
+type pendingSeries struct {
+	mu sync.Mutex
+	wr writeRequest
+
+	stopCh            chan struct{}
+	periodicFlusherWG sync.WaitGroup
+}
+
+func newPendingSeries(pushBlock func(block []byte), isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
+	var ps pendingSeries
+	ps.wr.pushBlock = pushBlock
+	ps.wr.isVMRemoteWrite = isVMRemoteWrite
+	ps.wr.significantFigures = significantFigures
+	ps.wr.roundDigits = roundDigits
+	ps.stopCh = make(chan struct{})
+	ps.periodicFlusherWG.Add(1)
+	go func() {
+		defer ps.periodicFlusherWG.Done()
+		ps.periodicFlusher()
+	}()
+	return &ps
+}
+
+func (ps *pendingSeries) MustStop() {
+	close(ps.stopCh)
+	ps.periodicFlusherWG.Wait()
+}
+
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
+	ps.mu.Lock()
+	ps.wr.push(tss)
+	ps.mu.Unlock()
+}
+
+func (ps *pendingSeries) periodicFlusher() {
+	flushSeconds := int64(flushInterval.Seconds())
+	if flushSeconds <= 0 {
+		flushSeconds = 1
+	}
+	ticker := time.NewTicker(*flushInterval)
+	defer ticker.Stop()
+	mustStop := false
+	for !mustStop {
+		select {
+		case <-ps.stopCh:
+			mustStop = true
+		case <-ticker.C:
+			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
+				continue
+			}
+		}
+		ps.mu.Lock()
+		ps.wr.flush()
+		ps.mu.Unlock()
+	}
+}
+
+type writeRequest struct {
+	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
+	lastFlushTime uint64
+
+	// pushBlock is called when whe write request is ready to be sent.
+	pushBlock func(block []byte)
+
+	// Whether to encode the write request with VictoriaMetrics remote write protocol.
+	isVMRemoteWrite bool
+
+	// How many significant figures must be left before sending the writeRequest to pushBlock.
+	significantFigures int
+
+	// How many decimal digits after point must be left before sending the writeRequest to pushBlock.
+	roundDigits int
+
+	wr prompbmarshal.WriteRequest
+
+	tss []prompbmarshal.TimeSeries
+
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
+	buf     []byte
+}
+
+func (wr *writeRequest) reset() {
+	// Do not reset lastFlushTime, pushBlock, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
+
+	wr.wr.Timeseries = nil
+
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	wr.tss = wr.tss[:0]
+
+	promrelabel.CleanLabels(wr.labels)
+	wr.labels = wr.labels[:0]
+
+	wr.samples = wr.samples[:0]
+	wr.buf = wr.buf[:0]
+}
+
+func (wr *writeRequest) flush() {
+	wr.wr.Timeseries = wr.tss
+	wr.adjustSampleValues()
+	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
+	pushWriteRequest(&wr.wr, wr.pushBlock, wr.isVMRemoteWrite)
+	wr.reset()
+}
+
+func (wr *writeRequest) adjustSampleValues() {
+	samples := wr.samples
+	if n := wr.significantFigures; n > 0 {
+		for i := range samples {
+			s := &samples[i]
+			s.Value = decimal.RoundToSignificantFigures(s.Value, n)
+		}
+	}
+	if n := wr.roundDigits; n < 100 {
+		for i := range samples {
+			s := &samples[i]
+			s.Value = decimal.RoundToDecimalDigits(s.Value, n)
+		}
+	}
+}
+
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
+	tssDst := wr.tss
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
+	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
+		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
+			wr.tss = tssDst
+			wr.flush()
+			tssDst = wr.tss
+		}
+	}
+	wr.tss = tssDst
+}
+
+func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
+	labelsDst := wr.labels
+	labelsLen := len(wr.labels)
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
+
+		buf = append(buf, srcLabel.Name...)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
+		buf = append(buf, srcLabel.Value...)
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
+	}
+	dst.Labels = labelsDst[labelsLen:]
+
+	samplesDst = append(samplesDst, src.Samples...)
+	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
+}
+
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte), isVMRemoteWrite bool) {
+	if len(wr.Timeseries) == 0 {
+		// Nothing to push
+		return
+	}
+	bb := writeRequestBufPool.Get()
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
+		zb := snappyBufPool.Get()
+		if isVMRemoteWrite {
+			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
+		} else {
+			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		}
+		writeRequestBufPool.Put(bb)
+		if len(zb.B) <= persistentqueue.MaxBlockSize {
+			pushBlock(zb.B)
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return
+		}
+		snappyBufPool.Put(zb)
+	} else {
+		writeRequestBufPool.Put(bb)
+	}
+
+	// Too big block. Recursively split it into smaller parts if possible.
+	if len(wr.Timeseries) == 1 {
+		// A single time series left. Recursively split its samples into smaller parts if possible.
+		samples := wr.Timeseries[0].Samples
+		if len(samples) == 1 {
+			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
+			return
+		}
+		n := len(samples) / 2
+		wr.Timeseries[0].Samples = samples[:n]
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+		wr.Timeseries[0].Samples = samples[n:]
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+		wr.Timeseries[0].Samples = samples
+		return
+	}
+	timeseries := wr.Timeseries
+	n := len(timeseries) / 2
+	wr.Timeseries = timeseries[:n]
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+	wr.Timeseries = timeseries[n:]
+	pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+	wr.Timeseries = timeseries
+}
+
+var (
+	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
+)
+
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/pendingseries_test.go

@@ -0,0 +1,70 @@
+package remotewrite
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestPushWriteRequest(t *testing.T) {
+	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
+	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
+	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
+	for i, rowsCount := range rowsCounts {
+		expectedBlockLenProm := expectedBlockLensProm[i]
+		expectedBlockLenVM := expectedBlockLensVM[i]
+		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
+			testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
+		})
+	}
+}
+
+func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
+	f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
+		t.Helper()
+		wr := newTestWriteRequest(rowsCount, 20)
+		pushBlockLen := 0
+		pushBlock := func(block []byte) {
+			if pushBlockLen > 0 {
+				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
+			}
+			pushBlockLen = len(block)
+		}
+		pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
+		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
+			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
+				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
+		}
+	}
+
+	// Check Prometheus remote write
+	f(false, expectedBlockLenProm, 0)
+
+	// Check VictoriaMetrics remote write
+	f(true, expectedBlockLenVM, 15)
+}
+
+func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {
+	var wr prompbmarshal.WriteRequest
+	for i := 0; i < seriesCount; i++ {
+		var labels []prompbmarshal.Label
+		for j := 0; j < labelsCount; j++ {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  fmt.Sprintf("label_%d_%d", i, j),
+				Value: fmt.Sprintf("value_%d_%d", i, j),
+			})
+		}
+		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
+			Labels: labels,
+			Samples: []prompbmarshal.Sample{
+				{
+					Value:     float64(i),
+					Timestamp: 1000 * int64(i),
+				},
+			},
+		})
+	}
+	return &wr
+}

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -0,0 +1,36 @@
+package remotewrite
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+	"github.com/klauspost/compress/s2"
+)
+
+func BenchmarkCompressWriteRequestSnappy(b *testing.B) {
+	b.Run("snappy", func(b *testing.B) {
+		benchmarkCompressWriteRequest(b, snappy.Encode)
+	})
+	b.Run("s2", func(b *testing.B) {
+		benchmarkCompressWriteRequest(b, s2.EncodeSnappy)
+	})
+}
+
+func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []byte) []byte) {
+	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
+		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
+			wr := newTestWriteRequest(rowsCount, 10)
+			data := prompbmarshal.MarshalWriteRequest(nil, wr)
+			b.ReportAllocs()
+			b.SetBytes(int64(rowsCount))
+			b.RunParallel(func(pb *testing.PB) {
+				var zb []byte
+				for pb.Next() {
+					zb = compressFunc(zb[:cap(zb)], data)
+				}
+			})
+		})
+	}
+}

app/vmagent/remotewrite/relabel.go

@@ -0,0 +1,161 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+var (
+	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
+		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
+		"The path can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
+	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
+		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
+
+	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
+		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
+		"See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels")
+)
+
+var labelsGlobal []prompbmarshal.Label
+
+// CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
+func CheckRelabelConfigs() error {
+	_, err := loadRelabelConfigs()
+	return err
+}
+
+func loadRelabelConfigs() (*relabelConfigs, error) {
+	var rcs relabelConfigs
+	if *relabelConfigPathGlobal != "" {
+		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
+		}
+		rcs.global = global
+	}
+	if len(*relabelConfigPaths) > (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)) {
+		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url or -remoteWrite.multitenantURL args: %d",
+			len(*relabelConfigPaths), (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
+	}
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
+	for i, path := range *relabelConfigPaths {
+		if len(path) == 0 {
+			// Skip empty relabel config.
+			continue
+		}
+		prc, err := promrelabel.LoadRelabelConfigs(path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
+		}
+		rcs.perURL[i] = prc
+	}
+	return &rcs, nil
+}
+
+type relabelConfigs struct {
+	global *promrelabel.ParsedConfigs
+	perURL []*promrelabel.ParsedConfigs
+}
+
+// initLabelsGlobal must be called after parsing command-line flags.
+func initLabelsGlobal() {
+	labelsGlobal = nil
+	for _, s := range *unparsedLabelsGlobal {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labelsGlobal = append(labelsGlobal, prompbmarshal.Label{
+			Name:  s[:n],
+			Value: s[n+1:],
+		})
+	}
+}
+
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && pcs.Len() == 0 && !*usePromCompatibleNaming {
+		// Nothing to change.
+		return tss
+	}
+	tssDst := tss[:0]
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		if *usePromCompatibleNaming {
+			// Replace unsupported Prometheus chars in label names and metric names with underscores.
+			tmpLabels := labels[labelsLen:]
+			for j := range tmpLabels {
+				label := &tmpLabels[j]
+				if label.Name == "__name__" {
+					label.Value = promrelabel.SanitizeName(label.Value)
+				} else {
+					label.Name = promrelabel.SanitizeName(label.Name)
+				}
+			}
+		}
+		labels = pcs.Apply(labels, labelsLen)
+		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
+		if len(labels) == labelsLen {
+			// Drop the current time series, since relabeling removed all the labels.
+			continue
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: ts.Samples,
+		})
+	}
+	rctx.labels = labels
+	return tssDst
+}
+
+type relabelCtx struct {
+	// pool for labels, which are used during the relabeling.
+	labels []prompbmarshal.Label
+}
+
+func (rctx *relabelCtx) reset() {
+	promrelabel.CleanLabels(rctx.labels)
+	rctx.labels = rctx.labels[:0]
+}
+
+var relabelCtxPool = &sync.Pool{
+	New: func() interface{} {
+		return &relabelCtx{}
+	},
+}
+
+func getRelabelCtx() *relabelCtx {
+	return relabelCtxPool.Get().(*relabelCtx)
+}
+
+func putRelabelCtx(rctx *relabelCtx) {
+	rctx.labels = rctx.labels[:0]
+	relabelCtxPool.Put(rctx)
+}

app/vmagent/remotewrite/relabel_test.go

@@ -0,0 +1,49 @@
+package remotewrite
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func TestApplyRelabeling(t *testing.T) {
+	f := func(extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
+		rctx := &relabelCtx{}
+		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
+		gotTss := rctx.applyRelabeling(tss, extraLabels, pcs)
+		if !reflect.DeepEqual(gotTss, expTss) {
+			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
+		}
+	}
+
+	f(nil, nil, "up", "up")
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, "up", `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, `up{foo="baz"}`, `up{foo="bar"}`)
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- target_label: "foo"
+  replacement: "aaa"
+- action: labeldrop
+  regex: "env.*"
+`))
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	f(nil, pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
+
+	oldVal := *usePromCompatibleNaming
+	*usePromCompatibleNaming = true
+	f(nil, nil, `foo.bar`, `foo_bar`)
+	*usePromCompatibleNaming = oldVal
+}
+
+func parseSeries(data string) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, prompbmarshal.TimeSeries{
+		Labels: promutils.MustNewLabelsFromString(data).GetLabels(),
+	})
+	return tss
+}

app/vmagent/remotewrite/remotewrite.go

@@ -0,0 +1,601 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+	"net/url"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
+)
+
+var (
+	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
+		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
+	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
+		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
+		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
+	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
+		"See also -remoteWrite.maxDiskUsagePerURL")
+	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
+		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500MB. "+
+		"Disk usage is unlimited if the value is set to 0")
+	significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", "The number of significant figures to leave in metric values before writing them "+
+		"to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
+		"This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits")
+	roundDigits = flagutil.NewArrayInt("remoteWrite.roundDigits", "Round metric values to this number of decimal digits after the point before writing them to remote storage. "+
+		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
+		"By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
+		"This option may be used for improving data compression for the stored metrics")
+	sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
+		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
+		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
+		`Enabled sorting for labels can slow down ingestion performance a bit`)
+	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
+	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
+
+	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
+		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
+		"See also -remoteWrite.streamAggr.keepInput and -remoteWrite.streamAggr.dedupInterval")
+	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep input samples after the aggregation with -remoteWrite.streamAggr.config. "+
+		"By default the input is dropped after the aggregation, so only the aggregate data is sent to the -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/stream-aggregation.html")
+	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", "Input samples are de-duplicated with this interval before being aggregated. "+
+		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
+)
+
+var (
+	// rwctxsDefault contains statically populated entries when -remoteWrite.url is specified.
+	rwctxsDefault []*remoteWriteCtx
+
+	// rwctxsMap contains dynamically populated entries when -remoteWrite.multitenantURL is specified.
+	rwctxsMap     = make(map[tenantmetrics.TenantID][]*remoteWriteCtx)
+	rwctxsMapLock sync.Mutex
+
+	// Data without tenant id is written to defaultAuthToken if -remoteWrite.multitenantURL is specified.
+	defaultAuthToken = &auth.Token{}
+)
+
+// MultitenancyEnabled returns true if -remoteWrite.multitenantURL is specified.
+func MultitenancyEnabled() bool {
+	return len(*remoteWriteMultitenantURLs) > 0
+}
+
+// Contains the current relabelConfigs.
+var allRelabelConfigs atomic.Value
+
+// maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
+// since it may lead to high memory usage due to big number of buffers.
+var maxQueues = cgroup.AvailableCPUs() * 16
+
+// InitSecretFlags must be called after flag.Parse and before any logging.
+func InitSecretFlags() {
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		flagutil.RegisterSecretFlag("remoteWrite.url")
+	}
+}
+
+// Init initializes remotewrite.
+//
+// It must be called after flag.Parse().
+//
+// Stop must be called for graceful shutdown.
+func Init() {
+	if len(*remoteWriteURLs) == 0 && len(*remoteWriteMultitenantURLs) == 0 {
+		logger.Fatalf("at least one `-remoteWrite.url` or `-remoteWrite.multitenantURL` command-line flag must be set")
+	}
+	if len(*remoteWriteURLs) > 0 && len(*remoteWriteMultitenantURLs) > 0 {
+		logger.Fatalf("cannot set both `-remoteWrite.url` and `-remoteWrite.multitenantURL` command-line flags")
+	}
+	if *maxHourlySeries > 0 {
+		hourlySeriesLimiter = bloomfilter.NewLimiter(*maxHourlySeries, time.Hour)
+		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
+			return float64(hourlySeriesLimiter.MaxItems())
+		})
+		_ = metrics.NewGauge(`vmagent_hourly_series_limit_current_series`, func() float64 {
+			return float64(hourlySeriesLimiter.CurrentItems())
+		})
+	}
+	if *maxDailySeries > 0 {
+		dailySeriesLimiter = bloomfilter.NewLimiter(*maxDailySeries, 24*time.Hour)
+		_ = metrics.NewGauge(`vmagent_daily_series_limit_max_series`, func() float64 {
+			return float64(dailySeriesLimiter.MaxItems())
+		})
+		_ = metrics.NewGauge(`vmagent_daily_series_limit_current_series`, func() float64 {
+			return float64(dailySeriesLimiter.CurrentItems())
+		})
+	}
+	if *queues > maxQueues {
+		*queues = maxQueues
+	}
+	if *queues <= 0 {
+		*queues = 1
+	}
+	initLabelsGlobal()
+
+	// Register SIGHUP handler for config reload before loadRelabelConfigs.
+	// This guarantees that the config will be re-read if the signal arrives just after loadRelabelConfig.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		logger.Fatalf("cannot load relabel configs: %s", err)
+	}
+	allRelabelConfigs.Store(rcs)
+
+	configSuccess.Set(1)
+	configTimestamp.Set(fasttime.UnixTimestamp())
+
+	if len(*remoteWriteURLs) > 0 {
+		rwctxsDefault = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	}
+
+	// Start config reloader.
+	configReloaderWG.Add(1)
+	go func() {
+		defer configReloaderWG.Done()
+		for {
+			select {
+			case <-sighupCh:
+			case <-stopCh:
+				return
+			}
+			configReloads.Inc()
+			logger.Infof("SIGHUP received; reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+			rcs, err := loadRelabelConfigs()
+			if err != nil {
+				configReloadErrors.Inc()
+				configSuccess.Set(0)
+				logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+				continue
+			}
+
+			allRelabelConfigs.Store(rcs)
+			configSuccess.Set(1)
+			configTimestamp.Set(fasttime.UnixTimestamp())
+			logger.Infof("Successfully reloaded relabel configs")
+		}
+	}()
+}
+
+var (
+	configReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
+	configReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
+	configSuccess      = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`)
+	configTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
+)
+
+func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
+	if len(urls) == 0 {
+		logger.Panicf("BUG: urls must be non-empty")
+	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks / *queues > 100 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 100 * *queues
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
+	rwctxs := make([]*remoteWriteCtx, len(urls))
+	for i, remoteWriteURLRaw := range urls {
+		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
+		if err != nil {
+			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		}
+		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
+		if at != nil {
+			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format
+			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
+			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
+		}
+		if *showRemoteWriteURL {
+			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
+		}
+		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+	}
+	return rwctxs
+}
+
+var stopCh = make(chan struct{})
+var configReloaderWG sync.WaitGroup
+
+// Stop stops remotewrite.
+//
+// It is expected that nobody calls Push during and after the call to this func.
+func Stop() {
+	close(stopCh)
+	configReloaderWG.Wait()
+
+	for _, rwctx := range rwctxsDefault {
+		rwctx.MustStop()
+	}
+	rwctxsDefault = nil
+
+	// There is no need in locking rwctxsMapLock here, since nobody should call Push during the Stop call.
+	for _, rwctxs := range rwctxsMap {
+		for _, rwctx := range rwctxs {
+			rwctx.MustStop()
+		}
+	}
+	rwctxsMap = nil
+
+	if sl := hourlySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
+	if sl := dailySeriesLimiter; sl != nil {
+		sl.MustStop()
+	}
+}
+
+// Push sends wr to remote storage systems set via `-remoteWrite.url`.
+//
+// If at is nil, then the data is pushed to the configured `-remoteWrite.url`.
+// If at isn't nil, the data is pushed to the configured `-remoteWrite.multitenantURL`.
+//
+// Note that wr may be modified by Push due to relabeling and rounding.
+func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
+	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
+		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
+		at = defaultAuthToken
+	}
+	var rwctxs []*remoteWriteCtx
+	if at == nil {
+		rwctxs = rwctxsDefault
+	} else {
+		if len(*remoteWriteMultitenantURLs) == 0 {
+			logger.Panicf("BUG: -remoteWrite.multitenantURL command-line flag must be set when __tenant_id__=%q label is set", at)
+		}
+		rwctxsMapLock.Lock()
+		tenantID := tenantmetrics.TenantID{
+			AccountID: at.AccountID,
+			ProjectID: at.ProjectID,
+		}
+		rwctxs = rwctxsMap[tenantID]
+		if rwctxs == nil {
+			rwctxs = newRemoteWriteCtxs(at, *remoteWriteMultitenantURLs)
+			rwctxsMap[tenantID] = rwctxs
+		}
+		rwctxsMapLock.Unlock()
+	}
+
+	var rctx *relabelCtx
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	pcsGlobal := rcs.global
+	if pcsGlobal.Len() > 0 || len(labelsGlobal) > 0 {
+		rctx = getRelabelCtx()
+	}
+	tss := wr.Timeseries
+	rowsCount := getRowsCount(tss)
+	globalRowsPushedBeforeRelabel.Add(rowsCount)
+	maxSamplesPerBlock := *maxRowsPerBlock
+	// Allow up to 10x of labels per each block on average.
+	maxLabelsPerBlock := 10 * maxSamplesPerBlock
+	for len(tss) > 0 {
+		// Process big tss in smaller blocks in order to reduce the maximum memory usage
+		samplesCount := 0
+		labelsCount := 0
+		i := 0
+		for i < len(tss) {
+			samplesCount += len(tss[i].Samples)
+			labelsCount += len(tss[i].Labels)
+			i++
+			if samplesCount >= maxSamplesPerBlock || labelsCount >= maxLabelsPerBlock {
+				break
+			}
+		}
+		tssBlock := tss
+		if i < len(tss) {
+			tssBlock = tss[:i]
+			tss = tss[i:]
+		} else {
+			tss = nil
+		}
+		if rctx != nil {
+			rowsCountBeforeRelabel := getRowsCount(tssBlock)
+			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
+			rowsCountAfterRelabel := getRowsCount(tssBlock)
+			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
+		}
+		sortLabelsIfNeeded(tssBlock)
+		tssBlock = limitSeriesCardinality(tssBlock)
+		pushBlockToRemoteStorages(rwctxs, tssBlock)
+		if rctx != nil {
+			rctx.reset()
+		}
+	}
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
+}
+
+func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries) {
+	if len(tssBlock) == 0 {
+		// Nothing to push
+		return
+	}
+	// Push block to remote storages in parallel in order to reduce the time needed for sending the data to multiple remote storage systems.
+	var wg sync.WaitGroup
+	for _, rwctx := range rwctxs {
+		wg.Add(1)
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
+			rwctx.Push(tssBlock)
+		}(rwctx)
+	}
+	wg.Wait()
+}
+
+// sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set.
+func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) {
+	if !*sortLabels {
+		return
+	}
+	for i := range tss {
+		promrelabel.SortLabels(tss[i].Labels)
+	}
+}
+
+func limitSeriesCardinality(tss []prompbmarshal.TimeSeries) []prompbmarshal.TimeSeries {
+	if hourlySeriesLimiter == nil && dailySeriesLimiter == nil {
+		return tss
+	}
+	dst := make([]prompbmarshal.TimeSeries, 0, len(tss))
+	for i := range tss {
+		labels := tss[i].Labels
+		h := getLabelsHash(labels)
+		if hourlySeriesLimiter != nil && !hourlySeriesLimiter.Add(h) {
+			hourlySeriesLimitRowsDropped.Add(len(tss[i].Samples))
+			logSkippedSeries(labels, "-remoteWrite.maxHourlySeries", hourlySeriesLimiter.MaxItems())
+			continue
+		}
+		if dailySeriesLimiter != nil && !dailySeriesLimiter.Add(h) {
+			dailySeriesLimitRowsDropped.Add(len(tss[i].Samples))
+			logSkippedSeries(labels, "-remoteWrite.maxDailySeries", dailySeriesLimiter.MaxItems())
+			continue
+		}
+		dst = append(dst, tss[i])
+	}
+	return dst
+}
+
+var (
+	hourlySeriesLimiter *bloomfilter.Limiter
+	dailySeriesLimiter  *bloomfilter.Limiter
+
+	hourlySeriesLimitRowsDropped = metrics.NewCounter(`vmagent_hourly_series_limit_rows_dropped_total`)
+	dailySeriesLimitRowsDropped  = metrics.NewCounter(`vmagent_daily_series_limit_rows_dropped_total`)
+)
+
+func getLabelsHash(labels []prompbmarshal.Label) uint64 {
+	bb := labelsHashBufPool.Get()
+	b := bb.B[:0]
+	for _, label := range labels {
+		b = append(b, label.Name...)
+		b = append(b, label.Value...)
+	}
+	h := xxhash.Sum64(b)
+	bb.B = b
+	labelsHashBufPool.Put(bb)
+	return h
+}
+
+var labelsHashBufPool bytesutil.ByteBufferPool
+
+func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) {
+	select {
+	case <-logSkippedSeriesTicker.C:
+		// Do not use logger.WithThrottler() here, since this will increase CPU usage
+		// because every call to logSkippedSeries will result to a call to labelsToString.
+		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
+	default:
+	}
+}
+
+var logSkippedSeriesTicker = time.NewTicker(5 * time.Second)
+
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b []byte
+	b = append(b, '{')
+	for i, label := range labels {
+		b = append(b, label.Name...)
+		b = append(b, '=')
+		b = strconv.AppendQuote(b, label.Value)
+		if i+1 < len(labels) {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, '}')
+	return string(b)
+}
+
+var (
+	globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
+	rowsDroppedByGlobalRelabel    = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+)
+
+type remoteWriteCtx struct {
+	idx int
+	fq  *persistentqueue.FastQueue
+	c   *client
+
+	sas                 *streamaggr.Aggregators
+	streamAggrKeepInput bool
+
+	pss        []*pendingSeries
+	pssNextIdx uint64
+
+	rowsPushedAfterRelabel *metrics.Counter
+	rowsDroppedByRelabel   *metrics.Counter
+}
+
+func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+	// strip query params, otherwise changing params resets pq
+	pqURL := *remoteWriteURL
+	pqURL.RawQuery = ""
+	pqURL.Fragment = ""
+	h := xxhash.Sum64([]byte(pqURL.String()))
+	queuePath := fmt.Sprintf("%s/persistent-queue/%d_%016X", *tmpDataPath, argIdx+1, h)
+	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0)
+	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
+		return float64(fq.GetPendingBytes())
+	})
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
+		return float64(fq.GetInmemoryQueueLen())
+	})
+
+	var c *client
+	switch remoteWriteURL.Scheme {
+	case "http", "https":
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+	default:
+		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
+	}
+	c.init(argIdx, *queues, sanitizedURL)
+
+	// Initialize pss
+	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
+	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
+	pssLen := *queues
+	if n := cgroup.AvailableCPUs(); pssLen > n {
+		// There is no sense in running more than availableCPUs concurrent pendingSeries,
+		// since every pendingSeries can saturate up to a single CPU.
+		pssLen = n
+	}
+	pss := make([]*pendingSeries, pssLen)
+	for i := range pss {
+		pss[i] = newPendingSeries(fq.MustWriteBlock, c.useVMProto, sf, rd)
+	}
+
+	rwctx := &remoteWriteCtx{
+		idx: argIdx,
+		fq:  fq,
+		c:   c,
+		pss: pss,
+
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+	}
+
+	// Initialize sas
+	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
+	if sasFile != "" {
+		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(argIdx, 0)
+		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggrFile=%q: %s", sasFile, err)
+		}
+		rwctx.sas = sas
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+	}
+
+	return rwctx
+}
+
+func (rwctx *remoteWriteCtx) MustStop() {
+	for _, ps := range rwctx.pss {
+		ps.MustStop()
+	}
+	rwctx.idx = 0
+	rwctx.pss = nil
+	rwctx.fq.UnblockAllReaders()
+	rwctx.c.MustStop()
+	rwctx.c = nil
+	rwctx.sas.MustStop()
+	rwctx.sas = nil
+	rwctx.fq.MustClose()
+	rwctx.fq = nil
+
+	rwctx.rowsPushedAfterRelabel = nil
+	rwctx.rowsDroppedByRelabel = nil
+}
+
+func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
+	// Apply relabeling
+	var rctx *relabelCtx
+	var v *[]prompbmarshal.TimeSeries
+	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	pcs := rcs.perURL[rwctx.idx]
+	if pcs.Len() > 0 {
+		rctx = getRelabelCtx()
+		// Make a copy of tss before applying relabeling in order to prevent
+		// from affecting time series for other remoteWrite.url configs.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
+		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
+		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
+		tss = append(*v, tss...)
+		rowsCountBeforeRelabel := getRowsCount(tss)
+		tss = rctx.applyRelabeling(tss, nil, pcs)
+		rowsCountAfterRelabel := getRowsCount(tss)
+		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
+	}
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
+
+	// Apply stream aggregation if any
+	rwctx.sas.Push(tss)
+	if rwctx.sas == nil || rwctx.streamAggrKeepInput {
+		// Push samples to the remote storage
+		rwctx.pushInternal(tss)
+	}
+
+	// Return back relabeling contexts to the pool
+	if rctx != nil {
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssRelabelPool.Put(v)
+		putRelabelCtx(rctx)
+	}
+}
+
+func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
+	pss := rwctx.pss
+	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	pss[idx].Push(tss)
+}
+
+var tssRelabelPool = &sync.Pool{
+	New: func() interface{} {
+		a := []prompbmarshal.TimeSeries{}
+		return &a
+	},
+}
+
+func getRowsCount(tss []prompbmarshal.TimeSeries) int {
+	rowsCount := 0
+	for _, ts := range tss {
+		rowsCount += len(ts.Samples)
+	}
+	return rowsCount
+}

app/vmagent/remotewrite/statconn.go

@@ -0,0 +1,92 @@
+package remotewrite
+
+import (
+	"context"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func getStdDialer() *net.Dialer {
+	stdDialerOnce.Do(func() {
+		stdDialer = &net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: netutil.TCP6Enabled(),
+		}
+	})
+	return stdDialer
+}
+
+var (
+	stdDialer     *net.Dialer
+	stdDialerOnce sync.Once
+)
+
+func statDial(ctx context.Context, networkUnused, addr string) (conn net.Conn, err error) {
+	network := netutil.GetTCPNetwork()
+	d := getStdDialer()
+	conn, err = d.DialContext(ctx, network, addr)
+	dialsTotal.Inc()
+	if err != nil {
+		dialErrors.Inc()
+		return nil, err
+	}
+	conns.Inc()
+	sc := &statConn{
+		Conn: conn,
+	}
+	return sc, nil
+}
+
+var (
+	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
+	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
+	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
+)
+
+type statConn struct {
+	closed uint64
+	net.Conn
+}
+
+func (sc *statConn) Read(p []byte) (int, error) {
+	n, err := sc.Conn.Read(p)
+	connReadsTotal.Inc()
+	if err != nil {
+		connReadErrors.Inc()
+	}
+	connBytesRead.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Write(p []byte) (int, error) {
+	n, err := sc.Conn.Write(p)
+	connWritesTotal.Inc()
+	if err != nil {
+		connWriteErrors.Inc()
+	}
+	connBytesWritten.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Close() error {
+	err := sc.Conn.Close()
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
+		conns.Dec()
+	}
+	return err
+}
+
+var (
+	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
+	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
+	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
+	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
+	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
+	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
+)

app/vmagent/vmimport/request_handler.go

@@ -0,0 +1,86 @@
+package vmimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="vmimport"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		rowsTotal += len(r.Values)
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		labels = append(labels, extraLabels...)
+		values := r.Values
+		timestamps := r.Timestamps
+		if len(timestamps) != len(values) {
+			logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
+		}
+		samplesLen := len(samples)
+		for j, value := range values {
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     value,
+				Timestamp: timestamps[j],
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/Makefile

@@ -0,0 +1,139 @@
+# All these commands must run from repository root.
+
+vmalert:
+	APP_NAME=vmalert $(MAKE) app-local
+
+vmalert-race:
+	APP_NAME=vmalert RACE=-race $(MAKE) app-local
+
+vmalert-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker
+
+vmalert-pure-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-pure
+
+vmalert-linux-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-amd64
+
+vmalert-linux-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm
+
+vmalert-linux-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-arm64
+
+vmalert-linux-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-ppc64le
+
+vmalert-linux-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-386
+
+vmalert-darwin-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
+
+vmalert-darwin-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-arm64
+
+vmalert-freebsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-freebsd-amd64
+
+vmalert-openbsd-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-openbsd-amd64
+
+vmalert-windows-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-windows-amd64
+
+package-vmalert:
+	APP_NAME=vmalert $(MAKE) package-via-docker
+
+package-vmalert-pure:
+	APP_NAME=vmalert $(MAKE) package-via-docker-pure
+
+package-vmalert-amd64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-amd64
+
+package-vmalert-arm:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm
+
+package-vmalert-arm64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm64
+
+package-vmalert-ppc64le:
+	APP_NAME=vmalert $(MAKE) package-via-docker-ppc64le
+
+package-vmalert-386:
+	APP_NAME=vmalert $(MAKE) package-via-docker-386
+
+publish-vmalert:
+	APP_NAME=vmalert $(MAKE) publish-via-docker
+
+test-vmalert:
+	go test -v -race -cover ./app/vmalert -loggerLevel=ERROR
+	go test -v -race -cover ./app/vmalert/templates
+	go test -v -race -cover ./app/vmalert/datasource
+	go test -v -race -cover ./app/vmalert/notifier
+	go test -v -race -cover ./app/vmalert/config
+	go test -v -race -cover ./app/vmalert/remotewrite
+
+run-vmalert: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-notifier.url=http://localhost:9093 \
+		-notifier.url=http://127.0.0.1:9093 \
+		-remoteWrite.url=http://localhost:8428 \
+		-remoteRead.url=http://localhost:8428 \
+		-external.label=cluster=east-1 \
+		-external.label=replica=a \
+		-evaluationInterval=3s \
+		-configCheckInterval=10s
+
+run-vmalert-sd: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules2-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-remoteWrite.url=http://localhost:8428 \
+		-notifier.config=app/vmalert/notifier/testdata/mixed.good.yaml \
+		-configCheckInterval=10s
+
+replay-vmalert: vmalert
+	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules-replay-good.rules \
+		-datasource.url=http://localhost:8428 \
+		-remoteWrite.url=http://localhost:8428 \
+		-external.label=cluster=east-1 \
+		-external.label=replica=a \
+		-replay.timeFrom=2021-05-11T07:21:43Z \
+		-replay.timeTo=2021-05-29T18:40:43Z
+
+vmalert-linux-amd64:
+	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-linux-arm:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+
+vmalert-linux-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmalert-linux-ppc64le:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+
+vmalert-linux-s390x:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
+vmalert-linux-386:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+vmalert-darwin-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-darwin-arm64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+vmalert-freebsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-openbsd-amd64:
+	APP_NAME=vmalert CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+vmalert-windows-amd64:
+	GOARCH=amd64 APP_NAME=vmalert $(MAKE) app-local-windows-goarch
+
+vmalert-pure:
+	APP_NAME=vmalert $(MAKE) app-local-pure

app/vmalert/alerting.go

@@ -0,0 +1,695 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// AlertingRule is basic alert entity
+type AlertingRule struct {
+	Type         config.Type
+	RuleID       uint64
+	Name         string
+	Expr         string
+	For          time.Duration
+	Labels       map[string]string
+	Annotations  map[string]string
+	GroupID      uint64
+	GroupName    string
+	EvalInterval time.Duration
+	Debug        bool
+
+	q datasource.Querier
+
+	alertsMu sync.RWMutex
+	// stores list of active alerts
+	alerts map[uint64]*notifier.Alert
+
+	// state stores recent state changes
+	// during evaluations
+	state *ruleState
+
+	metrics *alertingRuleMetrics
+}
+
+type alertingRuleMetrics struct {
+	errors  *utils.Gauge
+	pending *utils.Gauge
+	active  *utils.Gauge
+	samples *utils.Gauge
+}
+
+func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
+	ar := &AlertingRule{
+		Type:         group.Type,
+		RuleID:       cfg.ID,
+		Name:         cfg.Alert,
+		Expr:         cfg.Expr,
+		For:          cfg.For.Duration(),
+		Labels:       cfg.Labels,
+		Annotations:  cfg.Annotations,
+		GroupID:      group.ID(),
+		GroupName:    group.Name,
+		EvalInterval: group.Interval,
+		Debug:        cfg.Debug,
+		q: qb.BuildWithParams(datasource.QuerierParams{
+			DataSourceType:     group.Type.String(),
+			EvaluationInterval: group.Interval,
+			QueryParams:        group.Params,
+			Headers:            group.Headers,
+			Debug:              cfg.Debug,
+		}),
+		alerts:  make(map[uint64]*notifier.Alert),
+		metrics: &alertingRuleMetrics{},
+	}
+
+	if cfg.UpdateEntriesLimit != nil {
+		ar.state = newRuleState(*cfg.UpdateEntriesLimit)
+	} else {
+		ar.state = newRuleState(*ruleUpdateEntriesLimit)
+	}
+
+	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
+	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+		func() float64 {
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StatePending {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+		func() float64 {
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StateFiring {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	ar.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_error{%s}`, labels),
+		func() float64 {
+			e := ar.state.getLast()
+			if e.err == nil {
+				return 0
+			}
+			return 1
+		})
+	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			e := ar.state.getLast()
+			return float64(e.samples)
+		})
+	return ar
+}
+
+// Close unregisters rule metrics
+func (ar *AlertingRule) Close() {
+	ar.metrics.active.Unregister()
+	ar.metrics.pending.Unregister()
+	ar.metrics.errors.Unregister()
+	ar.metrics.samples.Unregister()
+}
+
+// String implements Stringer interface
+func (ar *AlertingRule) String() string {
+	return ar.Name
+}
+
+// ID returns unique Rule ID
+// within the parent Group.
+func (ar *AlertingRule) ID() uint64 {
+	return ar.RuleID
+}
+
+func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...interface{}) {
+	if !ar.Debug {
+		return
+	}
+	prefix := fmt.Sprintf("DEBUG rule %q:%q (%d) at %v: ",
+		ar.GroupName, ar.Name, ar.RuleID, at.Format(time.RFC3339))
+
+	if a != nil {
+		labelKeys := make([]string, len(a.Labels))
+		var i int
+		for k := range a.Labels {
+			labelKeys[i] = k
+			i++
+		}
+		sort.Strings(labelKeys)
+		labels := make([]string, len(labelKeys))
+		for i, l := range labelKeys {
+			labels[i] = fmt.Sprintf("%s=%q", l, a.Labels[l])
+		}
+		labelsStr := strings.Join(labels, ",")
+		prefix += fmt.Sprintf("alert %d {%s} ", a.ID, labelsStr)
+	}
+	msg := fmt.Sprintf(format, args...)
+	logger.Infof("%s", prefix+msg)
+}
+
+type labelSet struct {
+	// origin labels extracted from received time series
+	// plus extra labels (group labels, service labels like alertNameLabel).
+	// in case of conflicts, origin labels from time series preferred.
+	// used for templating annotations
+	origin map[string]string
+	// processed labels includes origin labels
+	// plus extra labels (group labels, service labels like alertNameLabel).
+	// in case of conflicts, extra labels are preferred.
+	// used as labels attached to notifier.Alert and ALERTS series written to remote storage.
+	processed map[string]string
+}
+
+// toLabels converts labels from given Metric
+// to labelSet which contains original and processed labels.
+func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
+	ls := &labelSet{
+		origin:    make(map[string]string),
+		processed: make(map[string]string),
+	}
+	for _, l := range m.Labels {
+		ls.origin[l.Name] = l.Value
+		// drop __name__ to be consistent with Prometheus alerting
+		if l.Name == "__name__" {
+			continue
+		}
+		ls.processed[l.Name] = l.Value
+	}
+
+	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
+		Labels: ls.origin,
+		Value:  m.Values[0],
+		Expr:   ar.Expr,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %s", err)
+	}
+	for k, v := range extraLabels {
+		ls.processed[k] = v
+		if _, ok := ls.origin[k]; !ok {
+			ls.origin[k] = v
+		}
+	}
+
+	// set additional labels to identify group and rule name
+	if ar.Name != "" {
+		ls.processed[alertNameLabel] = ar.Name
+		if _, ok := ls.origin[alertNameLabel]; !ok {
+			ls.origin[alertNameLabel] = ar.Name
+		}
+	}
+	if !*disableAlertGroupLabel && ar.GroupName != "" {
+		ls.processed[alertGroupNameLabel] = ar.GroupName
+		if _, ok := ls.origin[alertGroupNameLabel]; !ok {
+			ls.origin[alertGroupNameLabel] = ar.GroupName
+		}
+	}
+	return ls, nil
+}
+
+// ExecRange executes alerting rule on the given time range similarly to Exec.
+// It doesn't update internal states of the Rule and meant to be used just
+// to get time series for backfilling.
+// It returns ALERT and ALERT_FOR_STATE time series as result.
+func (ar *AlertingRule) ExecRange(ctx context.Context, start, end time.Time) ([]prompbmarshal.TimeSeries, error) {
+	series, err := ar.q.QueryRange(ctx, ar.Expr, start, end)
+	if err != nil {
+		return nil, err
+	}
+	var result []prompbmarshal.TimeSeries
+	qFn := func(query string) ([]datasource.Metric, error) {
+		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
+	}
+	for _, s := range series {
+		a, err := ar.newAlert(s, nil, time.Time{}, qFn) // initial alert
+		if err != nil {
+			return nil, fmt.Errorf("failed to create alert: %s", err)
+		}
+		if ar.For == 0 { // if alert is instant
+			a.State = notifier.StateFiring
+			for i := range s.Values {
+				result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+			}
+			continue
+		}
+
+		// if alert with For > 0
+		prevT := time.Time{}
+		for i := range s.Values {
+			at := time.Unix(s.Timestamps[i], 0)
+			if at.Sub(prevT) > ar.EvalInterval {
+				// reset to Pending if there are gaps > EvalInterval between DPs
+				a.State = notifier.StatePending
+				a.ActiveAt = at
+			} else if at.Sub(a.ActiveAt) >= ar.For {
+				a.State = notifier.StateFiring
+				a.Start = at
+			}
+			prevT = at
+			result = append(result, ar.alertToTimeSeries(a, s.Timestamps[i])...)
+		}
+	}
+	return result, nil
+}
+
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequently repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
+// Exec executes AlertingRule expression via the given Querier.
+// Based on the Querier results AlertingRule maintains notifier.Alerts
+func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]prompbmarshal.TimeSeries, error) {
+	start := time.Now()
+	qMetrics, req, err := ar.q.Query(ctx, ar.Expr, ts)
+	curState := ruleStateEntry{
+		time:     start,
+		at:       ts,
+		duration: time.Since(start),
+		samples:  len(qMetrics),
+		err:      err,
+		curl:     requestToCurl(req),
+	}
+
+	defer func() {
+		ar.state.add(curState)
+	}()
+
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
+	}
+
+	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s)", curState.samples, curState.duration)
+
+	for h, a := range ar.alerts {
+		// cleanup inactive alerts from previous Exec
+		if a.State == notifier.StateInactive && ts.Sub(a.ResolvedAt) > resolvedRetention {
+			ar.logDebugf(ts, a, "deleted as inactive")
+			delete(ar.alerts, h)
+		}
+	}
+
+	qFn := func(query string) ([]datasource.Metric, error) {
+		res, _, err := ar.q.Query(ctx, query, ts)
+		return res, err
+	}
+	updated := make(map[uint64]struct{})
+	// update list of active alerts
+	for _, m := range qMetrics {
+		ls, err := ar.toLabels(m, qFn)
+		if err != nil {
+			curState.err = fmt.Errorf("failed to expand labels: %s", err)
+			return nil, curState.err
+		}
+		h := hash(ls.processed)
+		if _, ok := updated[h]; ok {
+			// duplicate may be caused by extra labels
+			// conflicting with the metric labels
+			curState.err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
+			return nil, curState.err
+		}
+		updated[h] = struct{}{}
+		if a, ok := ar.alerts[h]; ok {
+			if a.State == notifier.StateInactive {
+				// alert could be in inactive state for resolvedRetention
+				// so when we again receive metrics for it - we switch it
+				// back to notifier.StatePending
+				a.State = notifier.StatePending
+				a.ActiveAt = ts
+				ar.logDebugf(ts, a, "INACTIVE => PENDING")
+			}
+			a.Value = m.Values[0]
+			// re-exec template since Value or query can be used in annotations
+			a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+		a, err := ar.newAlert(m, ls, start, qFn)
+		if err != nil {
+			curState.err = fmt.Errorf("failed to create alert: %w", err)
+			return nil, curState.err
+		}
+		a.ID = h
+		a.State = notifier.StatePending
+		a.ActiveAt = ts
+		ar.alerts[h] = a
+		ar.logDebugf(ts, a, "created in state PENDING")
+	}
+	var numActivePending int
+	for h, a := range ar.alerts {
+		// if alert wasn't updated in this iteration
+		// means it is resolved already
+		if _, ok := updated[h]; !ok {
+			if a.State == notifier.StatePending {
+				// alert was in Pending state - it is not
+				// active anymore
+				delete(ar.alerts, h)
+				ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round")
+				continue
+			}
+			if a.State == notifier.StateFiring {
+				a.State = notifier.StateInactive
+				a.ResolvedAt = ts
+				ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
+			}
+			continue
+		}
+		numActivePending++
+		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
+			a.State = notifier.StateFiring
+			a.Start = ts
+			alertsFired.Inc()
+			ar.logDebugf(ts, a, "PENDING => FIRING: %s since becoming active at %v", ts.Sub(a.ActiveAt), a.ActiveAt)
+		}
+	}
+	if limit > 0 && numActivePending > limit {
+		ar.alerts = map[uint64]*notifier.Alert{}
+		curState.err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
+		return nil, curState.err
+	}
+	return ar.toTimeSeries(ts.Unix()), nil
+}
+
+func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		ts := ar.alertToTimeSeries(a, timestamp)
+		tss = append(tss, ts...)
+	}
+	return tss
+}
+
+// UpdateWith copies all significant fields.
+// alerts state isn't copied since
+// it should be updated in next 2 Execs
+func (ar *AlertingRule) UpdateWith(r Rule) error {
+	nr, ok := r.(*AlertingRule)
+	if !ok {
+		return fmt.Errorf("BUG: attempt to update alerting rule with wrong type %#v", r)
+	}
+	ar.Expr = nr.Expr
+	ar.For = nr.For
+	ar.Labels = nr.Labels
+	ar.Annotations = nr.Annotations
+	ar.EvalInterval = nr.EvalInterval
+	ar.Debug = nr.Debug
+	ar.q = nr.q
+	ar.state = nr.state
+	return nil
+}
+
+// TODO: consider hashing algorithm in VM
+func hash(labels map[string]string) uint64 {
+	hash := fnv.New64a()
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		// drop __name__ to be consistent with Prometheus alerting
+		if k == "__name__" {
+			continue
+		}
+		name, value := k, labels[k]
+		hash.Write([]byte(name))
+		hash.Write([]byte(value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (ar *AlertingRule) newAlert(m datasource.Metric, ls *labelSet, start time.Time, qFn templates.QueryFn) (*notifier.Alert, error) {
+	var err error
+	if ls == nil {
+		ls, err = ar.toLabels(m, qFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to expand labels: %s", err)
+		}
+	}
+	a := &notifier.Alert{
+		GroupID:  ar.GroupID,
+		Name:     ar.Name,
+		Labels:   ls.processed,
+		Value:    m.Values[0],
+		ActiveAt: start,
+		Expr:     ar.Expr,
+		For:      ar.For,
+	}
+	a.Annotations, err = a.ExecTemplate(qFn, ls.origin, ar.Annotations)
+	return a, err
+}
+
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (ar *AlertingRule) AlertAPI(id uint64) *APIAlert {
+	ar.alertsMu.RLock()
+	defer ar.alertsMu.RUnlock()
+	a, ok := ar.alerts[id]
+	if !ok {
+		return nil
+	}
+	return ar.newAlertAPI(*a)
+}
+
+// ToAPI returns Rule representation in form of APIRule
+// Isn't thread-safe. Call must be protected by AlertingRule mutex.
+func (ar *AlertingRule) ToAPI() APIRule {
+	lastState := ar.state.getLast()
+	r := APIRule{
+		Type:           "alerting",
+		DatasourceType: ar.Type.String(),
+		Name:           ar.Name,
+		Query:          ar.Expr,
+		Duration:       ar.For.Seconds(),
+		Labels:         ar.Labels,
+		Annotations:    ar.Annotations,
+		LastEvaluation: lastState.time,
+		EvaluationTime: lastState.duration.Seconds(),
+		Health:         "ok",
+		State:          "inactive",
+		Alerts:         ar.AlertsToAPI(),
+		LastSamples:    lastState.samples,
+		MaxUpdates:     ar.state.size(),
+		Updates:        ar.state.getAll(),
+		Debug:          ar.Debug,
+
+		// encode as strings to avoid rounding in JSON
+		ID:      fmt.Sprintf("%d", ar.ID()),
+		GroupID: fmt.Sprintf("%d", ar.GroupID),
+	}
+	if lastState.err != nil {
+		r.LastError = lastState.err.Error()
+		r.Health = "err"
+	}
+	// satisfy APIRule.State logic
+	if len(r.Alerts) > 0 {
+		r.State = notifier.StatePending.String()
+		stateFiring := notifier.StateFiring.String()
+		for _, a := range r.Alerts {
+			if a.State == stateFiring {
+				r.State = stateFiring
+				break
+			}
+		}
+	}
+	return r
+}
+
+// AlertsToAPI generates list of APIAlert objects from existing alerts
+func (ar *AlertingRule) AlertsToAPI() []*APIAlert {
+	var alerts []*APIAlert
+	ar.alertsMu.RLock()
+	for _, a := range ar.alerts {
+		if a.State == notifier.StateInactive {
+			continue
+		}
+		alerts = append(alerts, ar.newAlertAPI(*a))
+	}
+	ar.alertsMu.RUnlock()
+	return alerts
+}
+
+func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
+	aa := &APIAlert{
+		// encode as strings to avoid rounding
+		ID:      fmt.Sprintf("%d", a.ID),
+		GroupID: fmt.Sprintf("%d", a.GroupID),
+		RuleID:  fmt.Sprintf("%d", ar.RuleID),
+
+		Name:        a.Name,
+		Expression:  ar.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.ActiveAt,
+		Restored:    a.Restored,
+		Value:       strconv.FormatFloat(a.Value, 'f', -1, 32),
+	}
+	if alertURLGeneratorFn != nil {
+		aa.SourceLink = alertURLGeneratorFn(a)
+	}
+	return aa
+}
+
+const (
+	// alertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// alertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// alertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// alertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+
+	// alertGroupNameLabel defines the label name attached for generated time series.
+	// attaching this label may be disabled via `-disableAlertgroupLabel` flag.
+	alertGroupNameLabel = "alertgroup"
+)
+
+// alertToTimeSeries converts the given alert with the given timestamp to time series
+func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(a, timestamp))
+	if ar.For > 0 {
+		tss = append(tss, alertForToTimeSeries(a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
+}
+
+// alertForToTimeSeries returns a timeseries that represents
+// state of active alerts, where value is time when alert become active
+func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
+}
+
+// Restore restores the value of ActiveAt field for active alerts,
+// based on previously written time series `alertForStateMetricName`.
+// Only rules with For > 0 can be restored.
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, ts time.Time, lookback time.Duration) error {
+	if ar.For < 1 {
+		return nil
+	}
+
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()
+
+	if len(ar.alerts) < 1 {
+		return nil
+	}
+
+	for _, a := range ar.alerts {
+		if a.Restored || a.State != notifier.StatePending {
+			continue
+		}
+
+		var labelsFilter []string
+		for k, v := range a.Labels {
+			labelsFilter = append(labelsFilter, fmt.Sprintf("%s=%q", k, v))
+		}
+		sort.Strings(labelsFilter)
+		expr := fmt.Sprintf("last_over_time(%s{%s}[%ds])",
+			alertForStateMetricName, strings.Join(labelsFilter, ","), int(lookback.Seconds()))
+
+		ar.logDebugf(ts, nil, "restoring alert state via query %q", expr)
+
+		qMetrics, _, err := q.Query(ctx, expr, ts)
+		if err != nil {
+			return err
+		}
+
+		if len(qMetrics) < 1 {
+			ar.logDebugf(ts, nil, "no response was received from restore query")
+			continue
+		}
+
+		// only one series expected in response
+		m := qMetrics[0]
+		// __name__ supposed to be alertForStateMetricName
+		m.DelLabel("__name__")
+
+		// we assume that restore query contains all label matchers,
+		// so all received labels will match anyway if their number is equal.
+		if len(m.Labels) != len(a.Labels) {
+			ar.logDebugf(ts, nil, "state restore query returned not expected label-set %v", m.Labels)
+			continue
+		}
+		a.ActiveAt = time.Unix(int64(m.Values[0]), 0)
+		a.Restored = true
+		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
+	}
+	return nil
+}
+
+// alertsToSend walks through the current alerts of AlertingRule
+// and returns only those which should be sent to notifier.
+// Isn't concurrent safe.
+func (ar *AlertingRule) alertsToSend(ts time.Time, resolveDuration, resendDelay time.Duration) []notifier.Alert {
+	needsSending := func(a *notifier.Alert) bool {
+		if a.State == notifier.StatePending {
+			return false
+		}
+		if a.ResolvedAt.After(a.LastSent) {
+			return true
+		}
+		return a.LastSent.Add(resendDelay).Before(ts)
+	}
+
+	var alerts []notifier.Alert
+	for _, a := range ar.alerts {
+		if !needsSending(a) {
+			continue
+		}
+		a.End = ts.Add(resolveDuration)
+		if a.State == notifier.StateInactive {
+			a.End = a.ResolvedAt
+		}
+		a.LastSent = ts
+		alerts = append(alerts, *a)
+	}
+	return alerts
+}

app/vmalert/alerting_test.go

@@ -0,0 +1,980 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func TestAlertingRule_ToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule  *AlertingRule
+		alert *notifier.Alert
+		expTS []prompbmarshal.TimeSeries
+	}{
+		{
+			newTestAlertingRule("instant", 0),
+			&notifier.Alert{State: notifier.StateFiring},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+				}),
+			},
+		},
+		{
+			newTestAlertingRule("instant extra labels", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					"job":           "foo",
+					"instance":      "bar",
+				}),
+			},
+		},
+		{
+			newTestAlertingRule("instant labels override", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+				}),
+			},
+		},
+		{
+			newTestAlertingRule("for", time.Second),
+			&notifier.Alert{State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
+			},
+		},
+		{
+			newTestAlertingRule("for pending", 10*time.Second),
+			&notifier.Alert{State: notifier.StatePending, ActiveAt: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StatePending.String(),
+				}),
+				newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
+					[]int64{timestamp.UnixNano()},
+					map[string]string{
+						"__name__": alertForStateMetricName,
+					}),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			tc.rule.alerts[tc.alert.ID] = tc.alert
+			tss := tc.rule.toTimeSeries(timestamp.Unix())
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
+
+func TestAlertingRule_Exec(t *testing.T) {
+	const defaultStep = 5 * time.Millisecond
+	type testAlert struct {
+		labels []string
+		alert  *notifier.Alert
+	}
+	testCases := []struct {
+		rule      *AlertingRule
+		steps     [][]datasource.Metric
+		expAlerts []testAlert
+	}{
+		{
+			newTestAlertingRule("empty", 0),
+			[][]datasource.Metric{},
+			nil,
+		},
+		{
+			newTestAlertingRule("empty labels", 0),
+			[][]datasource.Metric{
+				{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
+			},
+			[]testAlert{
+				{alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{},
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-firing", 0),
+			[][]datasource.Metric{
+				{
+					metricWithLabels(t, "name", "foo"),
+					metricWithLabels(t, "name", "foo1"),
+					metricWithLabels(t, "name", "foo2"),
+				},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("multiple-steps-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo1")},
+				{metricWithLabels(t, "name", "foo2")},
+			},
+			// 1: fire first alert
+			// 2: fire second alert, set first inactive
+			// 3: fire third alert, set second inactive
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending", time.Minute),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
+			},
+		},
+		{
+			newTestAlertingRule("for-fired", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>empty", time.Second),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset and delete pending alerts
+				{},
+			},
+			nil,
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			[]testAlert{
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.q = fq
+			tc.rule.GroupID = fakeGroup.ID()
+			for _, step := range tc.steps {
+				fq.reset()
+				fq.add(step...)
+				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
+					t.Fatalf("unexpected err: %s", err)
+				}
+				// artificial delay between applying steps
+				time.Sleep(defaultStep)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			expAlerts := make(map[uint64]*notifier.Alert)
+			for _, ta := range tc.expAlerts {
+				labels := make(map[string]string)
+				for i := 0; i < len(ta.labels); i += 2 {
+					k, v := ta.labels[i], ta.labels[i+1]
+					labels[k] = v
+				}
+				labels[alertNameLabel] = tc.rule.Name
+				h := hash(labels)
+				expAlerts[h] = ta.alert
+			}
+			for key, exp := range expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertingRule_ExecRange(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		data      []datasource.Metric
+		expAlerts []*notifier.Alert
+	}{
+		{
+			newTestAlertingRule("empty", 0),
+			[]datasource.Metric{},
+			nil,
+		},
+		{
+			newTestAlertingRule("empty labels", 0),
+			[]datasource.Metric{
+				{Values: []float64{1}, Timestamps: []int64{1}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing", 0),
+			[]datasource.Metric{
+				metricWithLabels(t, "name", "foo"),
+			},
+			[]*notifier.Alert{
+				{
+					Labels: map[string]string{"name": "foo"},
+					State:  notifier.StateFiring,
+				},
+			},
+		},
+		{
+			newTestAlertingRule("single-firing-on-range", 0),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1e3, 2e3, 3e3}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+				{State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestAlertingRule("for-pending", time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(3, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("for-firing", 3*time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("for=>pending=>firing=>pending=>firing=>pending", time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1, 1, 1}, Timestamps: []int64{1, 2, 5, 6, 20}},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(5, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(20, 0)},
+			},
+		},
+		{
+			newTestAlertingRule("multi-series-for=>pending=>pending=>firing", 3*time.Second),
+			[]datasource.Metric{
+				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+				},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
+				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
+				//
+				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
+					Labels: map[string]string{
+						"foo": "bar",
+					}},
+				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0),
+					Labels: map[string]string{
+						"foo": "bar",
+					}},
+			},
+		},
+		{
+			newTestRuleWithLabels("multi-series-firing", "source", "vm"),
+			[]datasource.Metric{
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
+				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+				},
+			},
+			[]*notifier.Alert{
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"source": "vm",
+				}},
+				//
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+				{State: notifier.StateFiring, Labels: map[string]string{
+					"foo":    "bar",
+					"source": "vm",
+				}},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_ExecRange"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.q = fq
+			tc.rule.GroupID = fakeGroup.ID()
+			fq.add(tc.data...)
+			gotTS, err := tc.rule.ExecRange(context.TODO(), time.Now(), time.Now())
+			if err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			var expTS []prompbmarshal.TimeSeries
+			var j int
+			for _, series := range tc.data {
+				for _, timestamp := range series.Timestamps {
+					a := tc.expAlerts[j]
+					if a.Labels == nil {
+						a.Labels = make(map[string]string)
+					}
+					a.Labels[alertNameLabel] = tc.rule.Name
+					expTS = append(expTS, tc.rule.alertToTimeSeries(a, timestamp)...)
+					j++
+				}
+			}
+			if len(gotTS) != len(expTS) {
+				t.Fatalf("expected %d time series; got %d", len(expTS), len(gotTS))
+			}
+			for i := range expTS {
+				got, exp := gotTS[i], expTS[i]
+				if !reflect.DeepEqual(got, exp) {
+					t.Fatalf("%d: expected \n%v but got \n%v", i, exp, got)
+				}
+			}
+		})
+	}
+}
+
+func TestGroup_Restore(t *testing.T) {
+	defaultTS := time.Now()
+	fqr := &fakeQuerierWithRegistry{}
+	fn := func(rules []config.Rule, expAlerts map[uint64]*notifier.Alert) {
+		t.Helper()
+		defer fqr.reset()
+
+		for _, r := range rules {
+			fqr.set(r.Expr, metricWithValueAndLabels(t, 0, "__name__", r.Alert))
+		}
+
+		fg := newGroup(config.Group{Name: "TestRestore", Rules: rules}, fqr, time.Second, nil)
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			nts := func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} }
+			fg.start(context.Background(), nts, nil, fqr)
+			wg.Done()
+		}()
+		fg.close()
+		wg.Wait()
+
+		gotAlerts := make(map[uint64]*notifier.Alert)
+		for _, rs := range fg.Rules {
+			alerts := rs.(*AlertingRule).alerts
+			for k, v := range alerts {
+				if !v.Restored {
+					// set not restored alerts to predictable timestamp
+					v.ActiveAt = defaultTS
+				}
+				gotAlerts[k] = v
+			}
+		}
+
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected %d alerts; got %d", len(expAlerts), len(gotAlerts))
+		}
+		for key, exp := range expAlerts {
+			got, ok := gotAlerts[key]
+			if !ok {
+				t.Fatalf("expected to have key %d", key)
+			}
+			if got.State != notifier.StatePending {
+				t.Fatalf("expected state %d; got %d", notifier.StatePending, got.State)
+			}
+			if got.ActiveAt != exp.ActiveAt {
+				t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
+			}
+		}
+	}
+
+	stateMetric := func(name string, value time.Time, labels ...string) datasource.Metric {
+		labels = append(labels, "__name__", alertForStateMetricName)
+		labels = append(labels, alertNameLabel, name)
+		labels = append(labels, alertGroupNameLabel, "TestRestore")
+		return metricWithValueAndLabels(t, float64(value.Unix()), labels...)
+	}
+
+	// one active alert, no previous state
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+		})
+	fqr.reset()
+
+	// one active alert with state restore
+	ts := time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+		stateMetric("foo", ts))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// two rules, two active alerts, one with state restored
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+		stateMetric("foo", ts))
+	fn(
+		[]config.Rule{
+			{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)},
+			{Alert: "bar", Expr: "bar", For: promutils.NewDuration(time.Second)},
+		},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// two rules, two active alerts, two with state restored
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+		stateMetric("foo", ts))
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+		stateMetric("bar", ts))
+	fn(
+		[]config.Rule{
+			{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)},
+			{Alert: "bar", Expr: "bar", For: promutils.NewDuration(time.Second)},
+		},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts,
+			},
+			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// one active alert but wrong state restore
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
+		stateMetric("wrong alert", ts))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+		})
+
+	// one active alert with labels
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+		stateMetric("foo", ts, "env", "dev"))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
+				ActiveAt: ts,
+			},
+		})
+
+	// one active alert with restore labels missmatch
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+		stateMetric("foo", ts, "env", "dev", "team", "foo"))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
+				ActiveAt: defaultTS,
+			},
+		})
+}
+
+func TestAlertingRule_Exec_Negative(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+
+	// successful attempt
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	_, err := ar.Exec(context.TODO(), time.Now(), 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// label `job` will collide with rule extra label and will make both time series equal
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
+	if !errors.Is(err, errDuplicate) {
+		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
+	}
+
+	fq.reset()
+
+	expErr := "connection reset by peer"
+	fq.setErr(errors.New(expErr))
+	_, err = ar.Exec(context.TODO(), time.Now(), 0)
+	if err == nil {
+		t.Fatalf("expected to get err; got nil")
+	}
+	if !strings.Contains(err.Error(), expErr) {
+		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+	}
+}
+
+func TestAlertingRuleLimit(t *testing.T) {
+	fq := &fakeQuerier{}
+	ar := newTestAlertingRule("test", 0)
+	ar.Labels = map[string]string{"job": "test"}
+	ar.q = fq
+	ar.For = time.Minute
+	testCases := []struct {
+		limit  int
+		err    string
+		tssNum int
+	}{
+		{
+			limit:  0,
+			tssNum: 4,
+		},
+		{
+			limit:  -1,
+			tssNum: 4,
+		},
+		{
+			limit:  1,
+			err:    "exec exceeded limit of 1 with 2 alerts",
+			tssNum: 0,
+		},
+		{
+			limit:  4,
+			tssNum: 4,
+		},
+	}
+	var (
+		err       error
+		timestamp = time.Now()
+	)
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "bar", "job"))
+	for _, testCase := range testCases {
+		_, err = ar.Exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
+		}
+	}
+	fq.reset()
+}
+
+func TestAlertingRule_Template(t *testing.T) {
+	testCases := []struct {
+		rule      *AlertingRule
+		metrics   []datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			&AlertingRule{
+				Name: "common",
+				Labels: map[string]string{
+					"region": "east",
+				},
+				Annotations: map[string]string{
+					"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "instance", "foo"),
+				metricWithValueAndLabels(t, 1, "instance", "bar"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
+					Annotations: map[string]string{
+						"summary": `common: Too high connection number for "foo"`,
+					},
+					Labels: map[string]string{
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "foo",
+					},
+				},
+				hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
+					Annotations: map[string]string{
+						"summary": `common: Too high connection number for "bar"`,
+					},
+					Labels: map[string]string{
+						alertNameLabel: "common",
+						"region":       "east",
+						"instance":     "bar",
+					},
+				},
+			},
+		},
+		{
+			&AlertingRule{
+				Name: "override label",
+				Labels: map[string]string{
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
+					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
+				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
+					Labels: map[string]string{
+						alertNameLabel: "override label",
+						"instance":     "foo",
+					},
+					Annotations: map[string]string{
+						"summary":     `first: Too high connection number for "foo"`,
+						"description": `override: It is 2 connections for "foo"`,
+					},
+				},
+				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
+					Labels: map[string]string{
+						alertNameLabel: "override label",
+						"instance":     "bar",
+					},
+					Annotations: map[string]string{
+						"summary":     `second: Too high connection number for "bar"`,
+						"description": `override: It is 10 connections for "bar"`,
+					},
+				},
+			},
+		},
+		{
+			&AlertingRule{
+				Name:      "OriginLabels",
+				GroupName: "Testing",
+				Labels: map[string]string{
+					"instance": "{{ $labels.instance }}",
+				},
+				Annotations: map[string]string{
+					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+				},
+				alerts: make(map[uint64]*notifier.Alert),
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1,
+					alertNameLabel, "originAlertname",
+					alertGroupNameLabel, "originGroupname",
+					"instance", "foo"),
+			},
+			map[uint64]*notifier.Alert{
+				hash(map[string]string{
+					alertNameLabel:      "OriginLabels",
+					alertGroupNameLabel: "Testing",
+					"instance":          "foo"}): {
+					Labels: map[string]string{
+						alertNameLabel:      "OriginLabels",
+						alertGroupNameLabel: "Testing",
+						"instance":          "foo",
+					},
+					Annotations: map[string]string{
+						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
+					},
+				},
+			},
+		},
+	}
+	fakeGroup := Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.GroupID = fakeGroup.ID()
+			tc.rule.q = fq
+			tc.rule.state = newRuleState(10)
+			fq.add(tc.metrics...)
+			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+			for hash, expAlert := range tc.expAlerts {
+				gotAlert := tc.rule.alerts[hash]
+				if gotAlert == nil {
+					t.Fatalf("alert %d is missing; labels: %v; annotations: %v",
+						hash, expAlert.Labels, expAlert.Annotations)
+				}
+				if !reflect.DeepEqual(expAlert.Annotations, gotAlert.Annotations) {
+					t.Fatalf("expected to have annotations %#v; got %#v", expAlert.Annotations, gotAlert.Annotations)
+				}
+				if !reflect.DeepEqual(expAlert.Labels, gotAlert.Labels) {
+					t.Fatalf("expected to have labels %#v; got %#v", expAlert.Labels, gotAlert.Labels)
+				}
+			}
+		})
+	}
+}
+
+func TestAlertsToSend(t *testing.T) {
+	ts := time.Now()
+	f := func(alerts, expAlerts []*notifier.Alert, resolveDuration, resendDelay time.Duration) {
+		t.Helper()
+		ar := &AlertingRule{alerts: make(map[uint64]*notifier.Alert)}
+		for i, a := range alerts {
+			ar.alerts[uint64(i)] = a
+		}
+		gotAlerts := ar.alertsToSend(ts, resolveDuration, resendDelay)
+		if gotAlerts == nil && expAlerts == nil {
+			return
+		}
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected to get %d alerts; got %d instead",
+				len(expAlerts), len(gotAlerts))
+		}
+		sort.Slice(expAlerts, func(i, j int) bool {
+			return expAlerts[i].Name < expAlerts[j].Name
+		})
+		sort.Slice(gotAlerts, func(i, j int) bool {
+			return gotAlerts[i].Name < gotAlerts[j].Name
+		})
+		for i, exp := range expAlerts {
+			got := gotAlerts[i]
+			if got.LastSent != exp.LastSent {
+				t.Fatalf("expected LastSent to be %v; got %v", exp.LastSent, got.LastSent)
+			}
+			if got.End != exp.End {
+				t.Fatalf("expected End to be %v; got %v", exp.End, got.End)
+			}
+		}
+	}
+
+	f( // send firing alert with custom resolve time
+		[]*notifier.Alert{{State: notifier.StateFiring}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(5 * time.Minute)}},
+		5*time.Minute, time.Minute,
+	)
+	f( // resolve inactive alert at the current timestamp
+		[]*notifier.Alert{{State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+	f( // mixed case of firing and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StateFiring}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "a", LastSent: ts, End: ts.Add(5 * time.Minute)}, {Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // mixed case of pending and resolved alerts. Names are added for deterministic sorting
+		[]*notifier.Alert{{Name: "a", State: notifier.StatePending}, {Name: "b", State: notifier.StateInactive, ResolvedAt: ts}},
+		[]*notifier.Alert{{Name: "b", LastSent: ts, End: ts}},
+		5*time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was already sent in the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // attempt to send alert that was sent out of the resendDelay interval
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-2 * time.Minute)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, time.Minute,
+	)
+	f( // alert must be sent even if resendDelay interval is 0
+		[]*notifier.Alert{{State: notifier.StateFiring, LastSent: ts.Add(-time.Second)}},
+		[]*notifier.Alert{{LastSent: ts, End: ts.Add(time.Minute)}},
+		time.Minute, 0,
+	)
+	f( // inactive alert which has been sent already
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts.Add(-2 * time.Second)}},
+		nil,
+		time.Minute, time.Minute,
+	)
+	f( // inactive alert which has been resolved after last send
+		[]*notifier.Alert{{State: notifier.StateInactive, LastSent: ts.Add(-time.Second), ResolvedAt: ts}},
+		[]*notifier.Alert{{LastSent: ts, End: ts}},
+		time.Minute, time.Minute,
+	)
+}
+
+func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
+	r := newTestAlertingRule(name, 0)
+	r.Labels = make(map[string]string)
+	for i := 0; i < len(labels); i += 2 {
+		r.Labels[labels[i]] = labels[i+1]
+	}
+	return r
+}
+
+func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
+	return &AlertingRule{
+		Name:         name,
+		For:          waitFor,
+		EvalInterval: waitFor,
+		alerts:       make(map[uint64]*notifier.Alert),
+		state:        newRuleState(10),
+	}
+}

app/vmalert/config/config.go

@@ -0,0 +1,304 @@
+package config
+
+import (
+	"crypto/md5"
+	"fmt"
+	"hash/fnv"
+	"net/url"
+	"sort"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// Group contains list of Rules grouped into
+// entity with one name and evaluation interval
+type Group struct {
+	Type        Type `yaml:"type,omitempty"`
+	File        string
+	Name        string              `yaml:"name"`
+	Interval    *promutils.Duration `yaml:"interval,omitempty"`
+	Limit       int                 `yaml:"limit,omitempty"`
+	Rules       []Rule              `yaml:"rules"`
+	Concurrency int                 `yaml:"concurrency"`
+	// Labels is a set of label value pairs, that will be added to every rule.
+	// It has priority over the external labels.
+	Labels map[string]string `yaml:"labels"`
+	// Checksum stores the hash of yaml definition for this group.
+	// May be used to detect any changes like rules re-ordering etc.
+	Checksum string
+	// Optional HTTP URL parameters added to each rule request
+	Params url.Values `yaml:"params"`
+	// Headers contains optional HTTP headers added to each rule request
+	Headers []Header `yaml:"headers,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type group Group
+	if err := unmarshal((*group)(g)); err != nil {
+		return err
+	}
+	b, err := yaml.Marshal(g)
+	if err != nil {
+		return fmt.Errorf("failed to marshal group configuration for checksum: %w", err)
+	}
+	// change default value to prometheus datasource.
+	if g.Type.Get() == "" {
+		g.Type.Set(NewPrometheusType())
+	}
+
+	h := md5.New()
+	h.Write(b)
+	g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	return nil
+}
+
+// Validate check for internal Group or Rule configuration errors
+func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool) error {
+	if g.Name == "" {
+		return fmt.Errorf("group name must be set")
+	}
+
+	uniqueRules := map[uint64]struct{}{}
+	for _, r := range g.Rules {
+		ruleName := r.Record
+		if r.Alert != "" {
+			ruleName = r.Alert
+		}
+		if _, ok := uniqueRules[r.ID]; ok {
+			return fmt.Errorf("%q is a duplicate within the group %q", r.String(), g.Name)
+		}
+		uniqueRules[r.ID] = struct{}{}
+		if err := r.Validate(); err != nil {
+			return fmt.Errorf("invalid rule %q.%q: %w", g.Name, ruleName, err)
+		}
+		if validateExpressions {
+			// its needed only for tests.
+			// because correct types must be inherited after unmarshalling.
+			exprValidator := g.Type.ValidateExpr
+			if err := exprValidator(r.Expr); err != nil {
+				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
+			}
+		}
+		if validateTplFn != nil {
+			if err := validateTplFn(r.Annotations); err != nil {
+				return fmt.Errorf("invalid annotations for rule %q.%q: %w", g.Name, ruleName, err)
+			}
+			if err := validateTplFn(r.Labels); err != nil {
+				return fmt.Errorf("invalid labels for rule %q.%q: %w", g.Name, ruleName, err)
+			}
+		}
+	}
+	return checkOverflow(g.XXX, fmt.Sprintf("group %q", g.Name))
+}
+
+// Rule describes entity that represent either
+// recording rule or alerting rule.
+type Rule struct {
+	ID          uint64
+	Record      string              `yaml:"record,omitempty"`
+	Alert       string              `yaml:"alert,omitempty"`
+	Expr        string              `yaml:"expr"`
+	For         *promutils.Duration `yaml:"for,omitempty"`
+	Labels      map[string]string   `yaml:"labels,omitempty"`
+	Annotations map[string]string   `yaml:"annotations,omitempty"`
+	Debug       bool                `yaml:"debug,omitempty"`
+	// UpdateEntriesLimit defines max number of rule's state updates stored in memory.
+	// Overrides `-rule.updateEntriesLimit`.
+	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rule Rule
+	if err := unmarshal((*rule)(r)); err != nil {
+		return err
+	}
+	r.ID = HashRule(*r)
+	return nil
+}
+
+// Name returns Rule name according to its type
+func (r *Rule) Name() string {
+	if r.Record != "" {
+		return r.Record
+	}
+	return r.Alert
+}
+
+// String implements Stringer interface
+func (r *Rule) String() string {
+	ruleType := "recording"
+	if r.Alert != "" {
+		ruleType = "alerting"
+	}
+	b := strings.Builder{}
+	b.WriteString(fmt.Sprintf("%s rule %q", ruleType, r.Name()))
+	b.WriteString(fmt.Sprintf("; expr: %q", r.Expr))
+
+	kv := sortMap(r.Labels)
+	for i := range kv {
+		if i == 0 {
+			b.WriteString("; labels:")
+		}
+		b.WriteString(" ")
+		b.WriteString(kv[i].key)
+		b.WriteString("=")
+		b.WriteString(kv[i].value)
+		if i < len(kv)-1 {
+			b.WriteString(",")
+		}
+	}
+	return b.String()
+}
+
+// HashRule hashes significant Rule fields into
+// unique hash that supposed to define Rule uniqueness
+func HashRule(r Rule) uint64 {
+	h := fnv.New64a()
+	h.Write([]byte(r.Expr))
+	if r.Record != "" {
+		h.Write([]byte("recording"))
+		h.Write([]byte(r.Record))
+	} else {
+		h.Write([]byte("alerting"))
+		h.Write([]byte(r.Alert))
+	}
+	kv := sortMap(r.Labels)
+	for _, i := range kv {
+		h.Write([]byte(i.key))
+		h.Write([]byte(i.value))
+		h.Write([]byte("\xff"))
+	}
+	return h.Sum64()
+}
+
+// Validate check for Rule configuration errors
+func (r *Rule) Validate() error {
+	if (r.Record == "" && r.Alert == "") || (r.Record != "" && r.Alert != "") {
+		return fmt.Errorf("either `record` or `alert` must be set")
+	}
+	if r.Expr == "" {
+		return fmt.Errorf("expression can't be empty")
+	}
+	return checkOverflow(r.XXX, "rule")
+}
+
+// ValidateTplFn must validate the given annotations
+type ValidateTplFn func(annotations map[string]string) error
+
+// ParseSilent parses rule configs from given file patterns without emitting logs
+func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
+	files, err := readFromFS(pathPatterns, true)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read from the config: %s", err)
+	}
+	return parse(files, validateTplFn, validateExpressions)
+}
+
+// Parse parses rule configs from given file patterns
+func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
+	files, err := readFromFS(pathPatterns, false)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read from the config: %s", err)
+	}
+	groups, err := parse(files, validateTplFn, validateExpressions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse %s: %s", pathPatterns, err)
+	}
+	if len(groups) < 1 {
+		logger.Warnf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
+	errGroup := new(utils.ErrGroup)
+	var groups []Group
+	for file, data := range files {
+		uniqueGroups := map[string]struct{}{}
+		gr, err := parseConfig(data)
+		if err != nil {
+			errGroup.Add(fmt.Errorf("failed to parse file %q: %w", file, err))
+			continue
+		}
+		for _, g := range gr {
+			if err := g.Validate(validateTplFn, validateExpressions); err != nil {
+				errGroup.Add(fmt.Errorf("invalid group %q in file %q: %w", g.Name, file, err))
+				continue
+			}
+			if _, ok := uniqueGroups[g.Name]; ok {
+				errGroup.Add(fmt.Errorf("group name %q duplicate in file %q", g.Name, file))
+				continue
+			}
+			uniqueGroups[g.Name] = struct{}{}
+			g.File = file
+			groups = append(groups, g)
+		}
+	}
+	if err := errGroup.Err(); err != nil {
+		return nil, err
+	}
+	sort.SliceStable(groups, func(i, j int) bool {
+		if groups[i].File != groups[j].File {
+			return groups[i].File < groups[j].File
+		}
+		return groups[i].Name < groups[j].Name
+	})
+	return groups, nil
+}
+
+func parseConfig(data []byte) ([]Group, error) {
+	data, err := envtemplate.ReplaceBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
+	}
+	g := struct {
+		Groups []Group `yaml:"groups"`
+		// Catches all undefined fields and must be empty after parsing.
+		XXX map[string]interface{} `yaml:",inline"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	if err != nil {
+		return nil, err
+	}
+	return g.Groups, checkOverflow(g.XXX, "config")
+}
+
+func checkOverflow(m map[string]interface{}, ctx string) error {
+	if len(m) > 0 {
+		var keys []string
+		for k := range m {
+			keys = append(keys, k)
+		}
+		return fmt.Errorf("unknown fields in %s: %s", ctx, strings.Join(keys, ", "))
+	}
+	return nil
+}
+
+type item struct {
+	key, value string
+}
+
+func sortMap(m map[string]string) []item {
+	var kv []item
+	for k, v := range m {
+		kv = append(kv, item{key: k, value: v})
+	}
+	sort.Slice(kv, func(i, j int) bool {
+		return kv[i].key < kv[j].key
+	})
+	return kv
+}

app/vmalert/config/config_test.go

@@ -0,0 +1,604 @@
+package config
+
+import (
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func TestMain(m *testing.M) {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
+		os.Exit(1)
+	}
+	os.Exit(m.Run())
+}
+
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
+func TestParseBad(t *testing.T) {
+	testCases := []struct {
+		path   []string
+		expErr string
+	}{
+		{
+			[]string{"testdata/rules/rules0-bad.rules"},
+			"unexpected token",
+		},
+		{
+			[]string{"testdata/dir/rules0-bad.rules"},
+			"error parsing annotation",
+		},
+		{
+			[]string{"testdata/dir/rules1-bad.rules"},
+			"duplicate in file",
+		},
+		{
+			[]string{"testdata/dir/rules2-bad.rules"},
+			"function \"unknown\" not defined",
+		},
+		{
+			[]string{"testdata/dir/rules3-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/dir/rules4-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/rules/rules1-bad.rules"},
+			"bad graphite expr",
+		},
+		{
+			[]string{"testdata/dir/rules6-bad.rules"},
+			"missing ':' in header",
+		},
+	}
+	for _, tc := range testCases {
+		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
+		if err == nil {
+			t.Errorf("expected to get error")
+			return
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		}
+	}
+}
+
+func TestRule_Validate(t *testing.T) {
+	if err := (&Rule{}).Validate(); err == nil {
+		t.Errorf("expected empty name error")
+	}
+	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
+		t.Errorf("expected empty expr error")
+	}
+	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
+		t.Errorf("expected valid rule; got %s", err)
+	}
+}
+
+func TestGroup_Validate(t *testing.T) {
+	testCases := []struct {
+		group               *Group
+		rules               []Rule
+		validateAnnotations bool
+		validateExpressions bool
+		expErr              string
+	}{
+		{
+			group:  &Group{},
+			expErr: "group name must be set",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr:              "invalid expression",
+			validateExpressions: true,
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": `
+{{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
+  {{ . | first | value | humanize1024 }}B
+{{ end }}`,
+						},
+					},
+				},
+			},
+			validateAnnotations: true,
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test",
+				Rules: []Rule{
+					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{Name: "test thanos",
+				Type: NewRawType("thanos"),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "unknown datasource type",
+		},
+		{
+			group: &Group{Name: "test graphite",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "some-description",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{Name: "test prometheus",
+				Type: NewPrometheusType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{
+				Name: "test graphite inherit",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sum(up == 0 ) by (host)",
+					},
+				},
+			},
+		},
+		{
+			group: &Group{
+				Name: "test graphite prometheus bad expr",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sum(up == 0 ) by (host)",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+					},
+				},
+			},
+			expErr: "invalid rule",
+		},
+	}
+
+	for _, tc := range testCases {
+		var validateTplFn ValidateTplFn
+		if tc.validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
+		if err == nil {
+			if tc.expErr != "" {
+				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
+			}
+			continue
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		}
+	}
+}
+
+func TestHashRule(t *testing.T) {
+	testCases := []struct {
+		a, b  Rule
+		equal bool
+	}{
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1"},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "bar",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Alert: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			false,
+		},
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 2"},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "baz",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			false,
+		},
+	}
+	for i, tc := range testCases {
+		aID, bID := HashRule(tc.a), HashRule(tc.b)
+		if tc.equal != (aID == bID) {
+			t.Fatalf("missmatch for rule %d", i)
+		}
+	}
+}
+
+func TestGroupChecksum(t *testing.T) {
+	f := func(t *testing.T, data, newData string) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		if g.Checksum == "" {
+			t.Fatalf("expected to get non-empty checksum")
+		}
+
+		var ng Group
+		if err := yaml.Unmarshal([]byte(newData), &ng); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		if g.Checksum == ng.Checksum {
+			t.Fatalf("expected to get different checksums")
+		}
+	}
+	t.Run("Ok", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+  - record: handler:requests:rate5m
+    expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+`, `
+name: TestGroup
+rules:
+  - record: handler:requests:rate5m
+    expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`for` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+    for: 5m
+`, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`interval` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+interval: 2s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+interval: 4s
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+	t.Run("`concurrency` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+concurrency: 2
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+concurrency: 16
+rules:
+  - alert: ExampleAlertWithFor
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`params` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+    nocache: ["1"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+params:
+    nocache: ["0"]
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`limit` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+limit: 5
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+limit: 10
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`headers` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+headers:
+  - "TenantID: foo"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+headers:
+  - "TenantID: bar"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
+	t.Run("`debug` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+    debug: true
+`)
+	})
+	t.Run("`update_entries_limit` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+    update_entries_limit: 33
+`)
+	})
+}
+
+func TestGroupParams(t *testing.T) {
+	f := func(t *testing.T, data string, expParams url.Values) {
+		t.Helper()
+		var g Group
+		if err := yaml.Unmarshal([]byte(data), &g); err != nil {
+			t.Fatalf("failed to unmarshal: %s", err)
+		}
+		got, exp := g.Params.Encode(), expParams.Encode()
+		if got != exp {
+			t.Fatalf("expected to have %q; got %q", exp, got)
+		}
+	}
+
+	t.Run("no params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{})
+	})
+
+	t.Run("params", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+params:
+  nocache: ["1"]
+  denyPartialResponse: ["true"]
+rules:
+  - alert: ExampleAlertAlwaysFiring
+    expr: sum by(job) (up == 1)
+`, url.Values{"nocache": {"1"}, "denyPartialResponse": {"true"}})
+	})
+}

app/vmalert/config/fs.go

@@ -0,0 +1,108 @@
+package config
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// FS represent a file system abstract for reading files.
+type FS interface {
+	// Init initializes FS.
+	Init() error
+
+	// String must return human-readable representation of FS.
+	String() string
+
+	// List returns the list of file names which will be read via Read fn
+	List() ([]string, error)
+
+	// Read returns a list of read files in form of a map
+	// where key is a file name and value is a content of read file.
+	// Read must be called only after the successful Init call.
+	Read(files []string) (map[string][]byte, error)
+}
+
+var (
+	fsRegistryMu sync.Mutex
+	fsRegistry   = make(map[string]FS)
+)
+
+// readFromFS parses the given path list and inits FS for each item.
+// Once inited, readFromFS will try to read and return files from each FS.
+// readFromFS returns an error if at least one FS failed to init.
+// The function can be called multiple times but each unique path
+// will be inited only once.
+// If silent == true, readFromFS will not emit any logs.
+//
+// It is allowed to mix different FS types in path list.
+func readFromFS(paths []string, silent bool) (map[string][]byte, error) {
+	var err error
+	result := make(map[string][]byte)
+	for _, path := range paths {
+
+		fsRegistryMu.Lock()
+		fs, ok := fsRegistry[path]
+		if !ok {
+			fs, err = newFS(path)
+			if err != nil {
+				fsRegistryMu.Unlock()
+				return nil, fmt.Errorf("error while parsing path %q: %w", path, err)
+			}
+			if err := fs.Init(); err != nil {
+				fsRegistryMu.Unlock()
+				return nil, fmt.Errorf("error while initializing path %q: %w", path, err)
+			}
+			fsRegistry[path] = fs
+		}
+		fsRegistryMu.Unlock()
+
+		list, err := fs.List()
+		if err != nil {
+			return nil, fmt.Errorf("failed to list files from %q", fs)
+		}
+
+		if !silent {
+			logger.Infof("found %d files to read from %q", len(list), fs)
+		}
+
+		if len(list) < 1 {
+			continue
+		}
+
+		files, err := fs.Read(list)
+		if err != nil {
+			return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
+		}
+		for k, v := range files {
+			if _, ok := result[k]; ok {
+				return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k)
+			}
+			result[k] = v
+		}
+	}
+	return result, nil
+}
+
+// newFS creates FS based on the give path.
+// Supported file systems are: fs
+func newFS(path string) (FS, error) {
+	scheme := "fs"
+	n := strings.Index(path, "://")
+	if n >= 0 {
+		scheme = path[:n]
+		path = path[n+len("://"):]
+	}
+	if len(path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	switch scheme {
+	case "fs":
+		return &fslocal.FS{Pattern: path}, nil
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q", scheme)
+	}
+}

app/vmalert/config/fs_test.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestNewFS(t *testing.T) {
+	f := func(path, expStr string) {
+		t.Helper()
+		fs, err := newFS(path)
+		if err != nil {
+			t.Fatalf("unexpected err: %s", err)
+		}
+		if fs.String() != expStr {
+			t.Fatalf("expected FS %q; got %q", expStr, fs.String())
+		}
+	}
+
+	f("/foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
+	f("fs:///foo/bar", "Local FS{MatchPattern: \"/foo/bar\"}")
+}
+
+func TestNewFSNegative(t *testing.T) {
+	f := func(path, expErr string) {
+		t.Helper()
+		_, err := newFS(path)
+		if err == nil {
+			t.Fatalf("expected to have err: %s", expErr)
+		}
+		if !strings.Contains(err.Error(), expErr) {
+			t.Fatalf("expected to have err %q; got %q instead", expErr, err)
+		}
+	}
+
+	f("", "path cannot be empty")
+	f("fs://", "path cannot be empty")
+	f("foobar://baz", `unsupported scheme "foobar"`)
+}

app/vmalert/config/fslocal/fslocal.go

@@ -0,0 +1,49 @@
+package fslocal
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// FS represents a local file system
+type FS struct {
+	// Pattern is used for matching one or multiple files.
+	// The pattern may describe hierarchical names such as
+	// /usr/*/bin/ed (assuming the Separator is '/').
+	Pattern string
+}
+
+// Init verifies that configured Pattern is correct
+func (fs *FS) Init() error {
+	_, err := filepath.Glob(fs.Pattern)
+	return err
+}
+
+// String implements Stringer interface
+func (fs *FS) String() string {
+	return fmt.Sprintf("Local FS{MatchPattern: %q}", fs.Pattern)
+}
+
+// List returns the list of file names which will be read via Read fn
+func (fs *FS) List() ([]string, error) {
+	matches, err := filepath.Glob(fs.Pattern)
+	if err != nil {
+		return nil, fmt.Errorf("error while matching files via pattern %s: %w", fs.Pattern, err)
+	}
+	return matches, nil
+}
+
+// Read returns a map of read files where
+// key is the file name and value is file's content.
+func (fs *FS) Read(files []string) (map[string][]byte, error) {
+	result := make(map[string][]byte)
+	for _, path := range files {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil, fmt.Errorf("error while reading file %q: %w", path, err)
+		}
+		result[path] = data
+	}
+	return result, nil
+}

app/vmalert/config/testdata/dir/rules0-bad.rules

@@ -0,0 +1,19 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidAnnotations
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }"
+          description: "{{$labels}}"
+      - alert: UnkownAnnotationsFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ value|query }}"
+          description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules0-good.rules

@@ -0,0 +1,14 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          expr: "{{ $expr|queryEscape }}"
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+

app/vmalert/config/testdata/dir/rules1-bad.rules

@@ -0,0 +1,22 @@
+groups:
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    labels:
+       dc: gcp
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules2-bad.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: group
+    rules:
+      - alert: UnkownLabelFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          summary: "{{ unknown|query }}"
+        annotations:
+          description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules3-bad.rules

@@ -0,0 +1,5 @@
+groups:
+  - name: group
+    rules:
+      - for: 5m
+        expr: vm_rows > 0

app/vmalert/config/testdata/dir/rules4-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    rules:
+      - alert: rows
+        record: record
+        for: 5m
+        expr: vm_rows > 0

app/vmalert/config/testdata/dir/rules5-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    rules:
+      - alert: rows
+        expr: vm_rows > 0
+      - record: rows
+        expr: sum(vm_rows)

app/vmalert/config/testdata/dir/rules6-bad.rules

@@ -0,0 +1,7 @@
+groups:
+  - name: group
+    headers:
+    - 'foobar'
+    rules:
+      - alert: rows
+        expr: vm_rows > 0

app/vmalert/config/testdata/rules/rules-query-good.rules

@@ -0,0 +1,15 @@
+groups:
+  - name: alertmanager.rules
+    rules:
+      - alert: AlertmanagerConfigInconsistent
+        annotations:
+          message: |
+            The configuration of the instances of the Alertmanager cluster `{{ $labels.namespace }}/{{ $labels.service }}` are out of sync.
+            {{ range printf "alertmanager_config_hash{namespace=\"%s\",service=\"%s\"}" $labels.namespace $labels.service | query }}
+            Configuration hash for pod {{ .Labels.pod }} is "{{ printf "%.f" .Value }}"
+            {{ end }}
+        expr: |
+          count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="openshift-monitoring"})) != 1
+        for: 5m
+        labels:
+          severity: critical

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -0,0 +1,40 @@
+groups:
+  - name: ReplayGroup
+    interval: 1m
+    concurrency: 1
+    limit: 1000
+    rules:
+      - record: type:vm_cache_entries:rate5m
+        expr: sum(rate(vm_cache_entries[5m])) by (type)
+        labels:
+          recording: true
+      - record: go_cgo_calls_count:rate5m
+        expr: rate(go_cgo_calls_count{job="vmdb"}[5m])
+        labels:
+          recording: true
+
+  - name: vmsingleReplay
+    interval: 30s
+    concurrency: 2
+    rules:
+      - alert: RequestErrorsToAPI
+        expr: increase(vm_http_request_errors_total[5m]) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=35&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
+          description: "Requests to path {{ $labels.path }} are receiving errors.
+            Please verify if clients are sending correct requests."
+
+      - alert: TooManyLogs
+        expr: sum(increase(vm_log_messages_total{level!="info"}[5m])) by (job, instance) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=67&var-instance={{ $labels.instance }}"
+          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+          description: "Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.\n
+           Worth to check logs for specific error messages."

app/vmalert/config/testdata/rules/rules0-bad.rules

@@ -0,0 +1,28 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidExpr
+        for: 5m
+        expr: vm_rows{ > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: EmptyExpr
+        for: 5m
+        expr: ""
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: ""
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: foo
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/config/testdata/rules/rules0-good.rules

@@ -0,0 +1,26 @@
+groups:
+  - name: groupGorSingleAlert
+    params:
+      nocache: ["1"]
+      denyPartialResponse: ["true"]
+    rules:
+      - alert: VMRows
+        for: 10s
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+  - name: TestGroup
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        annotations:
+          summary: "Too high connection number for {{$labels.instance}}"
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job)
+          (up == 1)

app/vmalert/config/testdata/rules/rules1-bad.rules

@@ -0,0 +1,12 @@
+groups:
+  - name: TestGraphiteBadGroup
+    interval: 2s
+    concurrency: 2
+    type: graphite
+    rules:
+      - alert: Conns
+        expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500) by instance
+        for: 3m
+        annotations:
+          summary: Too high connection number for {{$labels.instance}}
+          description: "It is {{ $value }} connections for {{$labels.instance}}"

app/vmalert/config/testdata/rules/rules1-good.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: groupTest
+    rules:
+      - alert: VMRows
+        for: 1ms
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -0,0 +1,64 @@
+groups:
+  - name: TestGroup
+    interval: 5s
+    concurrency: 2
+    limit: 1000
+    headers:
+      - "MyHeader: foo"
+    params:
+      denyPartialResponse: ["true"]
+    rules:
+      - alert: Conns
+        expr: vm_tcplistener_conns > 0
+        for: 3m
+        debug: true
+        update_entries_limit: 40
+        annotations:
+          labels: "Available labels: {{ $labels }}"
+          summary: Too high connection number for {{ $labels.instance }}
+            {{ with printf "sum(vm_tcplistener_conns{instance=%q})" .Labels.instance | query }}
+              {{ . | first | value }}
+            {{ end }}
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        update_entries_limit: -1
+        expr: sum by(job)
+          (up == 1)
+        labels:
+          job: '{{ $labels.job }}'
+          dynamic: '{{ $x := query "up" | first | value }}{{ if eq 1.0 $x }}one{{ else }}unknown{{ end }}'
+        annotations:
+          description: Job {{ $labels.job }} is up!
+          external: cluster-{{ $externalLabels.cluster }}; replica-{{ $externalLabels.replica }}
+          summary: All instances up {{ range query "up" }}
+            {{ . | label "instance" }}
+            {{ end }}
+      - record: handler:requests:rate5m
+        expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+        labels:
+          recording: true
+      - record: code:requests:rate5m
+        expr: sum(rate(promhttp_metric_handler_requests_total[5m])) by (code)
+        labels:
+          env: dev
+          recording: true
+      - record: code:requests:rate5m
+        expr: sum(rate(promhttp_metric_handler_requests_total[5m])) by (code)
+        labels:
+          env: staging
+          recording: true
+      - record: successful_requests:ratio_rate5m
+        labels:
+          recording: true
+        expr: |2
+            sum(code:requests:rate5m{code="200"})
+          /
+            sum(code:requests:rate5m)
+      - record: code:requests:slo
+        labels:
+           recording: true
+        expr: 0.95
+      - record: time:current
+        labels:
+           recording: true
+        expr: time()

app/vmalert/config/testdata/rules/rules3-good.rules

@@ -0,0 +1,24 @@
+groups:
+  - name: TestGroup
+    interval: 2s
+    concurrency: 2
+    type: graphite
+    rules:
+      - alert: Conns
+        expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500)
+        for: 3m
+
+        annotations:
+          summary: Too high connection number for {{$labels.instance}}
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+  - name: TestGroupPromMixed
+    interval: 2s
+    concurrency: 2
+    type: prometheus
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by (instance) > 1
+        for: 3m
+        annotations:
+          summary: Too high connection number for {{$labels.instance}}
+          description: "It is {{ $value }} connections for {{$labels.instance}}"

app/vmalert/config/testdata/rules/rules4-good.rules

@@ -0,0 +1,8 @@
+groups:
+  - name: TestEmptyRules
+    interval: 2s
+    concurrency: 2
+    rules:
+
+  - name: TestNoRules
+    type: prometheus

app/vmalert/config/testdata/rules/rules_interval_good.rules

@@ -0,0 +1,12 @@
+groups:
+  - name: groupTest
+    interval: 1s
+    rules:
+      - alert: VMRows
+        for: 2s
+        expr: sum(rate(vm_http_request_errors_total[2s])) > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/config/testdata/templates/templates0-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template0" }}
+Visit {{ externalURL }}
+{{ end }}

app/vmalert/config/testdata/templates/templates1-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template1" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates2-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template2" }}
+{{ 1048576 | humanize1024 }}
+{{ end }}

app/vmalert/config/testdata/templates/templates3-good.tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ printf "%s to %s!" "welcome" "hell" | toUpper }}
+{{ end }}

app/vmalert/config/testdata/templates/templates4-good-tmpl

@@ -0,0 +1,3 @@
+{{ define "template3" }}
+{{ 1230912039102391023.0 | humanizeDuration }}
+{{ end }}

app/vmalert/config/types.go

@@ -0,0 +1,116 @@
+package config
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql"
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+// Type represents data source type
+type Type struct {
+	Name string
+}
+
+// NewPrometheusType returns prometheus datasource type
+func NewPrometheusType() Type {
+	return Type{
+		Name: "prometheus",
+	}
+}
+
+// NewGraphiteType returns graphite datasource type
+func NewGraphiteType() Type {
+	return Type{
+		Name: "graphite",
+	}
+}
+
+// NewRawType returns datasource type from raw string
+// without validation.
+func NewRawType(d string) Type {
+	return Type{Name: d}
+}
+
+// Get returns datasource type
+func (t *Type) Get() string {
+	return t.Name
+}
+
+// Set changes datasource type
+func (t *Type) Set(d Type) {
+	t.Name = d.Name
+}
+
+// String implements String interface with default value.
+func (t Type) String() string {
+	if t.Name == "" {
+		return "prometheus"
+	}
+	return t.Name
+}
+
+// ValidateExpr validates query expression with datasource ql.
+func (t *Type) ValidateExpr(expr string) error {
+	switch t.String() {
+	case "graphite":
+		if _, err := graphiteql.Parse(expr); err != nil {
+			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
+		}
+	case "prometheus":
+		if _, err := metricsql.Parse(expr); err != nil {
+			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
+		}
+	default:
+		return fmt.Errorf("unknown datasource type=%q", t.Name)
+	}
+	return nil
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	if s == "" {
+		s = "prometheus"
+	}
+	switch s {
+	case "graphite", "prometheus":
+	default:
+		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
+	}
+	t.Name = s
+	return nil
+}
+
+// MarshalYAML implements the yaml.Unmarshaler interface.
+func (t Type) MarshalYAML() (interface{}, error) {
+	return t.Name, nil
+}
+
+// Header is a Key - Value struct for holding an HTTP header.
+type Header struct {
+	Key   string
+	Value string
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	if s == "" {
+		return nil
+	}
+	n := strings.IndexByte(s, ':')
+	if n < 0 {
+		return fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, s)
+	}
+	h.Key = strings.TrimSpace(s[:n])
+	h.Value = strings.TrimSpace(s[n+1:])
+	return nil
+}

app/vmalert/datasource/datasource.go

@@ -0,0 +1,99 @@
+package datasource
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// Querier interface wraps Query and QueryRange methods
+type Querier interface {
+	// Query executes instant request with the given query at the given ts.
+	// It returns list of Metric in response, the http.Request used for sending query
+	// and error if any. Returned http.Request can't be reused and its body is already read.
+	// Query should stop once ctx is cancelled.
+	Query(ctx context.Context, query string, ts time.Time) ([]Metric, *http.Request, error)
+	// QueryRange executes range request with the given query on the given time range.
+	// It returns list of Metric in response and error if any.
+	// QueryRange should stop once ctx is cancelled.
+	QueryRange(ctx context.Context, query string, from, to time.Time) ([]Metric, error)
+}
+
+// QuerierBuilder builds Querier with given params.
+type QuerierBuilder interface {
+	// BuildWithParams creates a new Querier object with the given params
+	BuildWithParams(params QuerierParams) Querier
+}
+
+// QuerierParams params for Querier.
+type QuerierParams struct {
+	DataSourceType     string
+	EvaluationInterval time.Duration
+	QueryParams        url.Values
+	Headers            map[string]string
+	Debug              bool
+}
+
+// Metric is the basic entity which should be return by datasource
+type Metric struct {
+	Labels     []Label
+	Timestamps []int64
+	Values     []float64
+}
+
+// SetLabel adds or updates existing one label
+// by the given key and label
+func (m *Metric) SetLabel(key, value string) {
+	for i, l := range m.Labels {
+		if l.Name == key {
+			m.Labels[i].Value = value
+			return
+		}
+	}
+	m.AddLabel(key, value)
+}
+
+// SetLabels sets the given map as Metric labels
+func (m *Metric) SetLabels(ls map[string]string) {
+	var i int
+	m.Labels = make([]Label, len(ls))
+	for k, v := range ls {
+		m.Labels[i] = Label{
+			Name:  k,
+			Value: v,
+		}
+		i++
+	}
+}
+
+// AddLabel appends the given label to the label set
+func (m *Metric) AddLabel(key, value string) {
+	m.Labels = append(m.Labels, Label{Name: key, Value: value})
+}
+
+// DelLabel deletes the given label from the label set
+func (m *Metric) DelLabel(key string) {
+	for i, l := range m.Labels {
+		if l.Name == key {
+			m.Labels = append(m.Labels[:i], m.Labels[i+1:]...)
+		}
+	}
+}
+
+// Label returns the given label value.
+// If label is missing empty string will be returned
+func (m *Metric) Label(key string) string {
+	for _, l := range m.Labels {
+		if l.Name == key {
+			return l.Value
+		}
+	}
+	return ""
+}
+
+// Label represents metric's label
+type Label struct {
+	Name  string
+	Value string
+}

app/vmalert/datasource/datasource_test.go

@@ -0,0 +1,18 @@
+package datasource
+
+import "testing"
+
+func TestMetric_Label(t *testing.T) {
+	m := &Metric{}
+
+	m.AddLabel("foo", "bar")
+	checkEqualString(t, "bar", m.Label("foo"))
+
+	m.SetLabel("foo", "baz")
+	checkEqualString(t, "baz", m.Label("foo"))
+
+	m.SetLabel("qux", "quux")
+	checkEqualString(t, "quux", m.Label("qux"))
+
+	checkEqualString(t, "", m.Label("non-existing"))
+}

app/vmalert/datasource/init.go

@@ -0,0 +1,114 @@
+package datasource
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+)
+
+var (
+	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
+		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend and -datasource.showURL")
+	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to show -datasource.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
+
+	headers = flag.String("datasource.headers", "", "Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. "+
+		"For example, -datasource.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -datasource.url. "+
+		"Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'")
+
+	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
+
+	bearerToken     = flag.String("datasource.bearerToken", "", "Optional bearer auth token to use for -datasource.url.")
+	bearerTokenFile = flag.String("datasource.bearerTokenFile", "", "Optional path to bearer token file to use for -datasource.url.")
+
+	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
+	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
+	tlsKeyFile            = flag.String("datasource.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -datasource.url")
+	tlsCAFile             = flag.String("datasource.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used`)
+	tlsServerName         = flag.String("datasource.tlsServerName", "", `Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used`)
+
+	oauth2ClientID         = flag.String("datasource.oauth2.clientID", "", "Optional OAuth2 clientID to use for -datasource.url. ")
+	oauth2ClientSecret     = flag.String("datasource.oauth2.clientSecret", "", "Optional OAuth2 clientSecret to use for -datasource.url.")
+	oauth2ClientSecretFile = flag.String("datasource.oauth2.clientSecretFile", "", "Optional OAuth2 clientSecretFile to use for -datasource.url. ")
+	oauth2TokenURL         = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url.")
+	oauth2Scopes           = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
+
+	lookBack  = flag.Duration("datasource.lookback", 0, `Lookback defines how far into the past to look when evaluating queries. For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
+	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
+		"For example, if -datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query. "+
+		"If set to 0, rule's evaluation interval will be used instead.")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Whether to align "time" parameter with evaluation interval.`+
+		"Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
+	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
+	disableKeepAlive   = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
+		`If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.`)
+	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
+		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
+)
+
+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showDatasourceURL {
+		flagutil.RegisterSecretFlag("datasource.url")
+	}
+}
+
+// Param represents an HTTP GET param
+type Param struct {
+	Key, Value string
+}
+
+// Init creates a Querier from provided flag values.
+// Provided extraParams will be added as GET params for
+// each request.
+func Init(extraParams url.Values) (QuerierBuilder, error) {
+	if *addr == "" {
+		return nil, fmt.Errorf("datasource.url is empty")
+	}
+
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %w", err)
+	}
+	tr.DisableKeepAlives = *disableKeepAlive
+	tr.MaxIdleConnsPerHost = *maxIdleConnections
+	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+	}
+
+	if extraParams == nil {
+		extraParams = url.Values{}
+	}
+	if *roundDigits > 0 {
+		extraParams.Set("round_digits", fmt.Sprintf("%d", *roundDigits))
+	}
+
+	authCfg, err := utils.AuthConfig(
+		utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		utils.WithBearer(*bearerToken, *bearerTokenFile),
+		utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes),
+		utils.WithHeaders(*headers))
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure auth: %w", err)
+	}
+
+	return &VMStorage{
+		c:                &http.Client{Transport: tr},
+		authCfg:          authCfg,
+		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		appendTypePrefix: *appendTypePrefix,
+		lookBack:         *lookBack,
+		queryStep:        *queryStep,
+		dataSourceType:   datasourcePrometheus,
+		extraParams:      extraParams,
+	}, nil
+}

app/vmalert/datasource/vm.go

@@ -0,0 +1,189 @@
+package datasource
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
+
+type datasourceType string
+
+const (
+	datasourcePrometheus datasourceType = "prometheus"
+	datasourceGraphite   datasourceType = "graphite"
+)
+
+func toDatasourceType(s string) datasourceType {
+	if s == string(datasourceGraphite) {
+		return datasourceGraphite
+	}
+	return datasourcePrometheus
+}
+
+// VMStorage represents vmstorage entity with ability to read and write metrics
+type VMStorage struct {
+	c                *http.Client
+	authCfg          *promauth.Config
+	datasourceURL    string
+	appendTypePrefix bool
+	lookBack         time.Duration
+	queryStep        time.Duration
+
+	dataSourceType     datasourceType
+	evaluationInterval time.Duration
+	extraParams        url.Values
+	extraHeaders       []keyValue
+
+	// whether to print additional log messages
+	// for each sent request
+	debug bool
+}
+
+type keyValue struct {
+	key   string
+	value string
+}
+
+// Clone makes clone of VMStorage, shares http client.
+func (s *VMStorage) Clone() *VMStorage {
+	return &VMStorage{
+		c:                s.c,
+		authCfg:          s.authCfg,
+		datasourceURL:    s.datasourceURL,
+		lookBack:         s.lookBack,
+		queryStep:        s.queryStep,
+		appendTypePrefix: s.appendTypePrefix,
+		dataSourceType:   s.dataSourceType,
+	}
+}
+
+// ApplyParams - changes given querier params.
+func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
+	s.dataSourceType = toDatasourceType(params.DataSourceType)
+	s.evaluationInterval = params.EvaluationInterval
+	s.extraParams = params.QueryParams
+	s.debug = params.Debug
+	if params.Headers != nil {
+		for key, value := range params.Headers {
+			kv := keyValue{key: key, value: value}
+			s.extraHeaders = append(s.extraHeaders, kv)
+		}
+	}
+	return s
+}
+
+// BuildWithParams - implements interface.
+func (s *VMStorage) BuildWithParams(params QuerierParams) Querier {
+	return s.Clone().ApplyParams(params)
+}
+
+// NewVMStorage is a constructor for VMStorage
+func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Duration, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage {
+	return &VMStorage{
+		c:                c,
+		authCfg:          authCfg,
+		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
+		appendTypePrefix: appendTypePrefix,
+		lookBack:         lookBack,
+		queryStep:        queryStep,
+		dataSourceType:   datasourcePrometheus,
+	}
+}
+
+// Query executes the given query and returns parsed response
+func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) ([]Metric, *http.Request, error) {
+	req, err := s.newRequestPOST()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	switch s.dataSourceType {
+	case "", datasourcePrometheus:
+		s.setPrometheusInstantReqParams(req, query, ts)
+	case datasourceGraphite:
+		s.setGraphiteReqParams(req, query, ts)
+	default:
+		return nil, nil, fmt.Errorf("engine not found: %q", s.dataSourceType)
+	}
+
+	resp, err := s.do(ctx, req)
+	if err != nil {
+		return nil, req, err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	parseFn := parsePrometheusResponse
+	if s.dataSourceType != datasourcePrometheus {
+		parseFn = parseGraphiteResponse
+	}
+	result, err := parseFn(req, resp)
+	return result, req, err
+}
+
+// QueryRange executes the given query on the given time range.
+// For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
+// Graphite type isn't supported.
+func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) ([]Metric, error) {
+	if s.dataSourceType != datasourcePrometheus {
+		return nil, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType)
+	}
+	req, err := s.newRequestPOST()
+	if err != nil {
+		return nil, err
+	}
+	if start.IsZero() {
+		return nil, fmt.Errorf("start param is missing")
+	}
+	if end.IsZero() {
+		return nil, fmt.Errorf("end param is missing")
+	}
+	s.setPrometheusRangeReqParams(req, query, start, end)
+	resp, err := s.do(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	return parsePrometheusResponse(req, resp)
+}
+
+func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+	if s.debug {
+		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, req.URL.RawQuery)
+	}
+	resp, err := s.c.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL.Redacted(), body)
+	}
+	return resp, nil
+}
+
+func (s *VMStorage) newRequestPOST() (*http.Request, error) {
+	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if s.authCfg != nil {
+		s.authCfg.SetHeaders(req, true)
+	}
+	for _, h := range s.extraHeaders {
+		req.Header.Set(h.key, h.value)
+	}
+	return req, nil
+}

app/vmalert/datasource/vm_graphite_api.go

@@ -0,0 +1,75 @@
+package datasource
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+type graphiteResponse []graphiteResponseTarget
+
+type graphiteResponseTarget struct {
+	Target     string            `json:"target"`
+	Tags       map[string]string `json:"tags"`
+	DataPoints [][2]float64      `json:"datapoints"`
+}
+
+func (r graphiteResponse) metrics() []Metric {
+	var ms []Metric
+	for _, res := range r {
+		if len(res.DataPoints) < 1 {
+			continue
+		}
+		var m Metric
+		// add only last value to the result.
+		last := res.DataPoints[len(res.DataPoints)-1]
+		m.Values = append(m.Values, last[0])
+		m.Timestamps = append(m.Timestamps, int64(last[1]))
+		for k, v := range res.Tags {
+			m.AddLabel(k, v)
+		}
+		ms = append(ms, m)
+	}
+	return ms
+}
+
+func parseGraphiteResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
+	r := &graphiteResponse{}
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL.Redacted(), err)
+	}
+	return r.metrics(), nil
+}
+
+const (
+	graphitePath   = "/render"
+	graphitePrefix = "/graphite"
+)
+
+func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestamp time.Time) {
+	if s.appendTypePrefix {
+		r.URL.Path += graphitePrefix
+	}
+	r.URL.Path += graphitePath
+	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
+	q.Set("format", "json")
+	q.Set("target", query)
+	from := "-5min"
+	if s.lookBack > 0 {
+		lookBack := timestamp.Add(-s.lookBack)
+		from = strconv.FormatInt(lookBack.Unix(), 10)
+	}
+	q.Set("from", from)
+	q.Set("until", "now")
+	r.URL.RawQuery = q.Encode()
+}

app/vmalert/datasource/vm_prom_api.go

@@ -0,0 +1,195 @@
+package datasource
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+var (
+	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path "+
+		"to the configured -datasource.url and -remoteRead.url")
+)
+
+type promResponse struct {
+	Status    string `json:"status"`
+	ErrorType string `json:"errorType"`
+	Error     string `json:"error"`
+	Data      struct {
+		ResultType string          `json:"resultType"`
+		Result     json.RawMessage `json:"result"`
+	} `json:"data"`
+}
+
+type promInstant struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TV     [2]interface{}    `json:"value"`
+	} `json:"result"`
+}
+
+func (r promInstant) metrics() ([]Metric, error) {
+	result := make([]Metric, len(r.Result))
+	for i, res := range r.Result {
+		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
+		}
+		var m Metric
+		m.SetLabels(res.Labels)
+		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
+		m.Values = append(m.Values, f)
+		result[i] = m
+	}
+	return result, nil
+}
+
+type promRange struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TVs    [][2]interface{}  `json:"values"`
+	} `json:"result"`
+}
+
+func (r promRange) metrics() ([]Metric, error) {
+	var result []Metric
+	for i, res := range r.Result {
+		var m Metric
+		for _, tv := range res.TVs {
+			f, err := strconv.ParseFloat(tv[1].(string), 64)
+			if err != nil {
+				return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, tv[1], err)
+			}
+			m.Values = append(m.Values, f)
+			m.Timestamps = append(m.Timestamps, int64(tv[0].(float64)))
+		}
+		if len(m.Values) < 1 || len(m.Timestamps) < 1 {
+			return nil, fmt.Errorf("metric %v contains no values", res)
+		}
+		m.Labels = nil
+		for k, v := range r.Result[i].Labels {
+			m.AddLabel(k, v)
+		}
+		result = append(result, m)
+	}
+	return result, nil
+}
+
+type promScalar [2]interface{}
+
+func (r promScalar) metrics() ([]Metric, error) {
+	var m Metric
+	f, err := strconv.ParseFloat(r[1].(string), 64)
+	if err != nil {
+		return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", r, r[1], err)
+	}
+	m.Values = append(m.Values, f)
+	m.Timestamps = append(m.Timestamps, int64(r[0].(float64)))
+	return []Metric{m}, nil
+}
+
+const (
+	statusSuccess, statusError  = "success", "error"
+	rtVector, rtMatrix, rScalar = "vector", "matrix", "scalar"
+)
+
+func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) {
+	r := &promResponse{}
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL.Redacted(), err)
+	}
+	if r.Status == statusError {
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error)
+	}
+	if r.Status != statusSuccess {
+		return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
+	}
+	switch r.Data.ResultType {
+	case rtVector:
+		var pi promInstant
+		if err := json.Unmarshal(r.Data.Result, &pi.Result); err != nil {
+			return nil, fmt.Errorf("umarshal err %s; \n %#v", err, string(r.Data.Result))
+		}
+		return pi.metrics()
+	case rtMatrix:
+		var pr promRange
+		if err := json.Unmarshal(r.Data.Result, &pr.Result); err != nil {
+			return nil, err
+		}
+		return pr.metrics()
+	case rScalar:
+		var ps promScalar
+		if err := json.Unmarshal(r.Data.Result, &ps); err != nil {
+			return nil, err
+		}
+		return ps.metrics()
+	default:
+		return nil, fmt.Errorf("unknown result type %q", r.Data.ResultType)
+	}
+}
+
+func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+	if s.appendTypePrefix {
+		r.URL.Path += "/prometheus"
+	}
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query"
+	}
+	q := r.URL.Query()
+	if s.lookBack > 0 {
+		timestamp = timestamp.Add(-s.lookBack)
+	}
+	if *queryTimeAlignment && s.evaluationInterval > 0 {
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
+		timestamp = timestamp.Truncate(s.evaluationInterval)
+	}
+	q.Set("time", fmt.Sprintf("%d", timestamp.Unix()))
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+	}
+	if s.queryStep > 0 { // override step with user-specified value
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
+	}
+	r.URL.RawQuery = q.Encode()
+	s.setPrometheusReqParams(r, query)
+}
+
+func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
+	if s.appendTypePrefix {
+		r.URL.Path += "/prometheus"
+	}
+	if !*disablePathAppend {
+		r.URL.Path += "/api/v1/query_range"
+	}
+	q := r.URL.Query()
+	q.Add("start", fmt.Sprintf("%d", start.Unix()))
+	q.Add("end", fmt.Sprintf("%d", end.Unix()))
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+		// always convert to seconds to keep compatibility with older
+		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+	}
+	r.URL.RawQuery = q.Encode()
+	s.setPrometheusReqParams(r, query)
+}
+
+func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
+	q := r.URL.Query()
+	for k, vs := range s.extraParams {
+		if q.Has(k) { // extraParams are prior to params in URL
+			q.Del(k)
+		}
+		for _, v := range vs {
+			q.Add(k, v)
+		}
+	}
+	q.Set("query", query)
+	r.URL.RawQuery = q.Encode()
+}

app/vmalert/datasource/vm_prom_api_test.go

@@ -0,0 +1,20 @@
+package datasource
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func BenchmarkMetrics(b *testing.B) {
+	payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`)
+
+	var pi promInstant
+	if err := json.Unmarshal(payload, &pi.Result); err != nil {
+		b.Fatalf(err.Error())
+	}
+	b.Run("Instant", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, _ = pi.metrics()
+		}
+	})
+}

app/vmalert/datasource/vm_test.go

@@ -0,0 +1,618 @@
+package datasource
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+)
+
+var (
+	ctx           = context.Background()
+	basicAuthName = "foo"
+	basicAuthPass = "bar"
+	baCfg         = &promauth.BasicAuthConfig{
+		Username: basicAuthName,
+		Password: promauth.NewSecret(basicAuthPass),
+	}
+	query       = "vm_rows"
+	queryRender = "constantLine(10)"
+)
+
+func TestVMInstantQuery(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc("/render", func(w http.ResponseWriter, request *http.Request) {
+		c++
+		switch c {
+		case 8:
+			w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
+		}
+	})
+	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+		}
+		if r.URL.Query().Get("query") != query {
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+		}
+		timeParam := r.URL.Query().Get("time")
+		if timeParam == "" {
+			t.Errorf("expected 'time' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(timeParam, 10, 64); err != nil {
+			t.Errorf("failed to parse 'time' query param: %s", err)
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			w.Write([]byte("[]"))
+		case 3:
+			w.Write([]byte(`{"status":"error", "errorType":"type:", "error":"some error msg"}`))
+		case 4:
+			w.Write([]byte(`{"status":"unknown"}`))
+		case 5:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
+		case 6:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
+		case 7:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
+		}
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	authCfg, err := baCfg.NewConfig(".")
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, 0, false, srv.Client())
+
+	p := datasourcePrometheus
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(p), EvaluationInterval: 15 * time.Second})
+	ts := time.Now()
+
+	expErr := func(err string) {
+		if _, _, err := pq.Query(ctx, query, ts); err == nil {
+			t.Fatalf("expected %q got nil", err)
+		}
+	}
+
+	expErr("connection error")              // 0
+	expErr("invalid response status error") // 1
+	expErr("response body error")           // 2
+	expErr("error status")                  // 3
+	expErr("unknown status")                // 4
+	expErr("non-vector resultType error")   // 5
+
+	m, _, err := pq.Query(ctx, query, ts) // 6 - vector
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 2 {
+		t.Fatalf("expected 2 metrics got %d in %+v", len(m), m)
+	}
+	expected := []Metric{
+		{
+			Labels:     []Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
+			Timestamps: []int64{1583786142},
+			Values:     []float64{13763},
+		},
+		{
+			Labels:     []Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
+			Timestamps: []int64{1583786140},
+			Values:     []float64{2000},
+		},
+	}
+	metricsEqual(t, m, expected)
+
+	m, req, err := pq.Query(ctx, query, ts) // 7 - scalar
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if req == nil {
+		t.Fatalf("expected request to be non-nil")
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 metrics got %d in %+v", len(m), m)
+	}
+	expected = []Metric{
+		{
+			Timestamps: []int64{1583786142},
+			Values:     []float64{1},
+		},
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("unexpected metric %+v want %+v", m, expected)
+	}
+
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
+
+	m, _, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
+	}
+	exp := []Metric{
+		{
+			Labels:     []Label{{Value: "constantLine(10)", Name: "name"}},
+			Timestamps: []int64{1611758403},
+			Values:     []float64{10},
+		},
+	}
+	metricsEqual(t, m, exp)
+}
+
+func metricsEqual(t *testing.T, gotM, expectedM []Metric) {
+	for i, exp := range expectedM {
+		got := gotM[i]
+		gotTS, expTS := got.Timestamps, exp.Timestamps
+		if !reflect.DeepEqual(gotTS, expTS) {
+			t.Fatalf("unexpected timestamps %+v want %+v", gotTS, expTS)
+		}
+		gotV, expV := got.Values, exp.Values
+		if !reflect.DeepEqual(gotV, expV) {
+			t.Fatalf("unexpected values %+v want %+v", gotV, expV)
+		}
+		sort.Slice(got.Labels, func(i, j int) bool {
+			return got.Labels[i].Name < got.Labels[j].Name
+		})
+		sort.Slice(exp.Labels, func(i, j int) bool {
+			return exp.Labels[i].Name < exp.Labels[j].Name
+		})
+		if !reflect.DeepEqual(exp.Labels, got.Labels) {
+			t.Fatalf("unexpected labels %+v want %+v", got.Labels, exp.Labels)
+		}
+	}
+}
+
+func TestVMRangeQuery(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc("/api/v1/query_range", func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+		}
+		if r.URL.Query().Get("query") != query {
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+		}
+		startTS := r.URL.Query().Get("start")
+		if startTS == "" {
+			t.Errorf("expected 'start' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(startTS, 10, 64); err != nil {
+			t.Errorf("failed to parse 'start' query param: %s", err)
+		}
+		endTS := r.URL.Query().Get("end")
+		if endTS == "" {
+			t.Errorf("expected 'end' in query param, got nil instead")
+		}
+		if _, err := strconv.ParseInt(endTS, 10, 64); err != nil {
+			t.Errorf("failed to parse 'end' query param: %s", err)
+		}
+		step := r.URL.Query().Get("step")
+		if step != "15s" {
+			t.Errorf("expected 'step' query param to be 15s; got %q instead", step)
+		}
+		switch c {
+		case 0:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"vm_rows"},"values":[[1583786142,"13763"]]}]}}`))
+		}
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	authCfg, err := baCfg.NewConfig(".")
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	s := NewVMStorage(srv.URL, authCfg, time.Minute, *queryStep, false, srv.Client())
+
+	pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus), EvaluationInterval: 15 * time.Second})
+
+	_, err = pq.QueryRange(ctx, query, time.Now(), time.Time{})
+	expectError(t, err, "is missing")
+
+	_, err = pq.QueryRange(ctx, query, time.Time{}, time.Now())
+	expectError(t, err, "is missing")
+
+	start, end := time.Now().Add(-time.Minute), time.Now()
+
+	m, err := pq.QueryRange(ctx, query, start, end)
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
+	}
+	expected := Metric{
+		Labels:     []Label{{Value: "vm_rows", Name: "__name__"}},
+		Timestamps: []int64{1583786142},
+		Values:     []float64{13763},
+	}
+	if !reflect.DeepEqual(m[0], expected) {
+		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
+	}
+
+	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
+
+	_, err = gq.QueryRange(ctx, queryRender, start, end)
+	expectError(t, err, "is not supported")
+}
+
+func TestRequestParams(t *testing.T) {
+	authCfg, err := baCfg.NewConfig(".")
+	if err != nil {
+		t.Fatalf("unexpected: %s", err)
+	}
+	query := "up"
+	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
+	testCases := []struct {
+		name       string
+		queryRange bool
+		vm         *VMStorage
+		checkFn    func(t *testing.T, r *http.Request)
+	}{
+		{
+			"prometheus path",
+			false,
+			&VMStorage{
+				dataSourceType: datasourcePrometheus,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/api/v1/query", r.URL.Path)
+			},
+		},
+		{
+			"prometheus prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   datasourcePrometheus,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
+			},
+		},
+		{
+			"prometheus range path",
+			true,
+			&VMStorage{
+				dataSourceType: datasourcePrometheus,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
+			},
+		},
+		{
+			"prometheus range prefix",
+			true,
+			&VMStorage{
+				dataSourceType:   datasourcePrometheus,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
+			},
+		},
+		{
+			"graphite path",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"graphite prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   datasourceGraphite,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"default params",
+			false,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&time=%d", query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"default range params",
+			true,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("end=%d&query=%s&start=%d", timestamp.Unix(), query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"basic auth",
+			false,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"basic auth range",
+			true,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"lookback",
+			false,
+			&VMStorage{
+				lookBack: time.Minute,
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&time=%d", query, timestamp.Add(-time.Minute).Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"evaluation interval",
+			false,
+			&VMStorage{
+				evaluationInterval: 15 * time.Second,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 15 * time.Second
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, evalInterval, tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"lookback + evaluation interval",
+			false,
+			&VMStorage{
+				lookBack:           time.Minute,
+				evaluationInterval: 15 * time.Second,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 15 * time.Second
+				tt := timestamp.Add(-time.Minute)
+				tt = tt.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, evalInterval, tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"step override",
+			false,
+			&VMStorage{
+				queryStep: time.Minute,
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(time.Minute.Seconds()), timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"step to seconds",
+			false,
+			&VMStorage{
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				tt := timestamp.Truncate(evalInterval)
+				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(evalInterval.Seconds()), tt.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params range",
+			true,
+			&VMStorage{
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("end=%d&max_lookback=1h&nocache=1&query=%s&start=%d",
+					timestamp.Unix(), query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"graphite extra params",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := tc.vm.newRequestPOST()
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			switch tc.vm.dataSourceType {
+			case "", datasourcePrometheus:
+				if tc.queryRange {
+					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
+				} else {
+					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
+				}
+			case datasourceGraphite:
+				tc.vm.setGraphiteReqParams(req, query, timestamp)
+			}
+			tc.checkFn(t, req)
+		})
+	}
+}
+
+func TestHeaders(t *testing.T) {
+	var testCases = []struct {
+		name    string
+		vmFn    func() *VMStorage
+		checkFn func(t *testing.T, r *http.Request)
+	}{
+		{
+			name: "basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			name: "bearer auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				reqToken := r.Header.Get("Authorization")
+				splitToken := strings.Split(reqToken, "Bearer ")
+				if len(splitToken) != 2 {
+					t.Errorf("expected two items got %d", len(splitToken))
+				}
+				token := splitToken[1]
+				checkEqualString(t, "foo", token)
+			},
+		},
+		{
+			name: "custom extraHeaders",
+			vmFn: func() *VMStorage {
+				return &VMStorage{extraHeaders: []keyValue{
+					{key: "Foo", value: "bar"},
+					{key: "Baz", value: "qux"},
+				}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				h1 := r.Header.Get("Foo")
+				checkEqualString(t, "bar", h1)
+				h2 := r.Header.Get("Baz")
+				checkEqualString(t, "qux", h2)
+			},
+		},
+		{
+			name: "custom header overrides basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{
+					authCfg: cfg,
+					extraHeaders: []keyValue{
+						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+					}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "Aladdin", u)
+				checkEqualString(t, "open sesame", p)
+			},
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			vm := tt.vmFn()
+			req, err := vm.newRequestPOST()
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			tt.checkFn(t, req)
+		})
+	}
+}
+
+func checkEqualString(t *testing.T, exp, got string) {
+	t.Helper()
+	if got != exp {
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
+	}
+}
+
+func expectError(t *testing.T, err error, exp string) {
+	t.Helper()
+	if err == nil {
+		t.Errorf("expected non-nil error")
+	}
+	if !strings.Contains(err.Error(), exp) {
+		t.Errorf("expected error %q to contain %q", err, exp)
+	}
+}

app/vmalert/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8880
+
+ENTRYPOINT ["/vmalert-prod"]
+ARG src_binary
+COPY $src_binary ./vmalert-prod

app/vmalert/group.go

@@ -0,0 +1,586 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"hash/fnv"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// Group is an entity for grouping rules
+type Group struct {
+	mu             sync.RWMutex
+	Name           string
+	File           string
+	Rules          []Rule
+	Type           config.Type
+	Interval       time.Duration
+	Limit          int
+	Concurrency    int
+	Checksum       string
+	LastEvaluation time.Time
+
+	Labels  map[string]string
+	Params  url.Values
+	Headers map[string]string
+
+	doneCh     chan struct{}
+	finishedCh chan struct{}
+	// channel accepts new Group obj
+	// which supposed to update current group
+	updateCh chan *Group
+	// evalCancel stores the cancel fn for interrupting
+	// rules evaluation. Used on groups update() and close().
+	evalCancel context.CancelFunc
+
+	metrics *groupMetrics
+}
+
+type groupMetrics struct {
+	iterationTotal    *utils.Counter
+	iterationDuration *utils.Summary
+	iterationMissed   *utils.Counter
+	iterationInterval *utils.Gauge
+}
+
+func newGroupMetrics(g *Group) *groupMetrics {
+	m := &groupMetrics{}
+	labels := fmt.Sprintf(`group=%q, file=%q`, g.Name, g.File)
+	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationMissed = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	m.iterationInterval = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
+		g.mu.RLock()
+		i := g.Interval.Seconds()
+		g.mu.RUnlock()
+		return i
+	})
+	return m
+}
+
+// merges group rule labels into result map
+// set2 has priority over set1.
+func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
+	r := map[string]string{}
+	for k, v := range set1 {
+		r[k] = v
+	}
+	for k, v := range set2 {
+		if prevV, ok := r[k]; ok {
+			logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
+				k, prevV, groupName, ruleName, k, v)
+		}
+		r[k] = v
+	}
+	return r
+}
+
+func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group {
+	g := &Group{
+		Type:        cfg.Type,
+		Name:        cfg.Name,
+		File:        cfg.File,
+		Interval:    cfg.Interval.Duration(),
+		Limit:       cfg.Limit,
+		Concurrency: cfg.Concurrency,
+		Checksum:    cfg.Checksum,
+		Params:      cfg.Params,
+		Headers:     make(map[string]string),
+		Labels:      cfg.Labels,
+
+		doneCh:     make(chan struct{}),
+		finishedCh: make(chan struct{}),
+		updateCh:   make(chan *Group),
+	}
+	if g.Interval == 0 {
+		g.Interval = defaultInterval
+	}
+	if g.Concurrency < 1 {
+		g.Concurrency = 1
+	}
+	for _, h := range cfg.Headers {
+		g.Headers[h.Key] = h.Value
+	}
+	g.metrics = newGroupMetrics(g)
+	rules := make([]Rule, len(cfg.Rules))
+	for i, r := range cfg.Rules {
+		var extraLabels map[string]string
+		// apply external labels
+		if len(labels) > 0 {
+			extraLabels = labels
+		}
+		// apply group labels, it has priority on external labels
+		if len(cfg.Labels) > 0 {
+			extraLabels = mergeLabels(g.Name, r.Name(), extraLabels, g.Labels)
+		}
+		// apply rules labels, it has priority on other labels
+		if len(extraLabels) > 0 {
+			r.Labels = mergeLabels(g.Name, r.Name(), extraLabels, r.Labels)
+		}
+
+		rules[i] = g.newRule(qb, r)
+	}
+	g.Rules = rules
+	return g
+}
+
+func (g *Group) newRule(qb datasource.QuerierBuilder, rule config.Rule) Rule {
+	if rule.Alert != "" {
+		return newAlertingRule(qb, g, rule)
+	}
+	return newRecordingRule(qb, g, rule)
+}
+
+// ID return unique group ID that consists of
+// rules file and group Name
+func (g *Group) ID() uint64 {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	hash := fnv.New64a()
+	hash.Write([]byte(g.File))
+	hash.Write([]byte("\xff"))
+	hash.Write([]byte(g.Name))
+	hash.Write([]byte(g.Type.Get()))
+	return hash.Sum64()
+}
+
+// Restore restores alerts state for group rules
+func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, ts time.Time, lookback time.Duration) error {
+	for _, rule := range g.Rules {
+		ar, ok := rule.(*AlertingRule)
+		if !ok {
+			continue
+		}
+		if ar.For < 1 {
+			continue
+		}
+		q := qb.BuildWithParams(datasource.QuerierParams{
+			DataSourceType:     g.Type.String(),
+			EvaluationInterval: g.Interval,
+			QueryParams:        g.Params,
+			Headers:            g.Headers,
+			Debug:              ar.Debug,
+		})
+		if err := ar.Restore(ctx, q, ts, lookback); err != nil {
+			return fmt.Errorf("error while restoring rule %q: %w", rule, err)
+		}
+	}
+	return nil
+}
+
+// updateWith updates existing group with
+// passed group object. This function ignores group
+// evaluation interval change. It supposed to be updated
+// in group.start function.
+// Not thread-safe.
+func (g *Group) updateWith(newGroup *Group) error {
+	rulesRegistry := make(map[uint64]Rule)
+	for _, nr := range newGroup.Rules {
+		rulesRegistry[nr.ID()] = nr
+	}
+
+	for i, or := range g.Rules {
+		nr, ok := rulesRegistry[or.ID()]
+		if !ok {
+			// old rule is not present in the new list
+			// so we mark it for removing
+			g.Rules[i].Close()
+			g.Rules[i] = nil
+			continue
+		}
+		if err := or.UpdateWith(nr); err != nil {
+			return err
+		}
+		delete(rulesRegistry, nr.ID())
+	}
+
+	var newRules []Rule
+	for _, r := range g.Rules {
+		if r == nil {
+			// skip nil rules
+			continue
+		}
+		newRules = append(newRules, r)
+	}
+	// add the rest of rules from registry
+	for _, nr := range rulesRegistry {
+		newRules = append(newRules, nr)
+	}
+	// note that g.Interval is not updated here
+	// so the value can be compared later in
+	// group.Start function
+	g.Type = newGroup.Type
+	g.Concurrency = newGroup.Concurrency
+	g.Params = newGroup.Params
+	g.Headers = newGroup.Headers
+	g.Labels = newGroup.Labels
+	g.Limit = newGroup.Limit
+	g.Checksum = newGroup.Checksum
+	g.Rules = newRules
+	return nil
+}
+
+// interruptEval interrupts in-flight rules evaluations
+// within the group. It is expected that g.evalCancel
+// will be repopulated after the call.
+func (g *Group) interruptEval() {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	if g.evalCancel != nil {
+		g.evalCancel()
+	}
+}
+
+func (g *Group) close() {
+	if g.doneCh == nil {
+		return
+	}
+	close(g.doneCh)
+	g.interruptEval()
+	<-g.finishedCh
+
+	g.metrics.iterationDuration.Unregister()
+	g.metrics.iterationTotal.Unregister()
+	g.metrics.iterationMissed.Unregister()
+	g.metrics.iterationInterval.Unregister()
+	for _, rule := range g.Rules {
+		rule.Close()
+	}
+}
+
+var skipRandSleepOnGroupStart bool
+
+func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client, rr datasource.QuerierBuilder) {
+	defer func() { close(g.finishedCh) }()
+
+	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+	if !skipRandSleepOnGroupStart {
+		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
+		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
+		if randSleep < sleepOffset {
+			randSleep += uint64(g.Interval)
+		}
+		randSleep -= sleepOffset
+		sleepTimer := time.NewTimer(time.Duration(randSleep))
+		select {
+		case <-ctx.Done():
+			sleepTimer.Stop()
+			return
+		case <-g.doneCh:
+			sleepTimer.Stop()
+			return
+		case <-sleepTimer.C:
+		}
+	}
+
+	e := &executor{
+		rw:                       rw,
+		notifiers:                nts,
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)}
+
+	evalTS := time.Now()
+
+	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
+
+	eval := func(ctx context.Context, ts time.Time) {
+		g.metrics.iterationTotal.Inc()
+
+		start := time.Now()
+
+		if len(g.Rules) < 1 {
+			g.metrics.iterationDuration.UpdateDuration(start)
+			g.LastEvaluation = start
+			return
+		}
+
+		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
+		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration, g.Limit)
+		for err := range errs {
+			if err != nil {
+				logger.Errorf("group %q: %s", g.Name, err)
+			}
+		}
+		g.metrics.iterationDuration.UpdateDuration(start)
+		g.LastEvaluation = start
+	}
+
+	evalCtx, cancel := context.WithCancel(ctx)
+	g.mu.Lock()
+	g.evalCancel = cancel
+	g.mu.Unlock()
+	defer g.evalCancel()
+
+	eval(evalCtx, evalTS)
+
+	t := time.NewTicker(g.Interval)
+	defer t.Stop()
+
+	// restore the rules state after the first evaluation
+	// so only active alerts can be restored.
+	if rr != nil {
+		err := g.Restore(ctx, rr, evalTS, *remoteReadLookBack)
+		if err != nil {
+			logger.Errorf("error while restoring ruleState for group %q: %s", g.Name, err)
+		}
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			logger.Infof("group %q: context cancelled", g.Name)
+			return
+		case <-g.doneCh:
+			logger.Infof("group %q: received stop signal", g.Name)
+			return
+		case ng := <-g.updateCh:
+			g.mu.Lock()
+
+			// it is expected that g.evalCancel will be evoked
+			// somewhere else to unblock group from the rules evaluation.
+			// we recreate the evalCtx and g.evalCancel, so it can
+			// be called again.
+			evalCtx, cancel = context.WithCancel(ctx)
+			g.evalCancel = cancel
+
+			err := g.updateWith(ng)
+			if err != nil {
+				logger.Errorf("group %q: failed to update: %s", g.Name, err)
+				g.mu.Unlock()
+				continue
+			}
+
+			// ensure that staleness is tracked or existing rules only
+			e.purgeStaleSeries(g.Rules)
+
+			if g.Interval != ng.Interval {
+				g.Interval = ng.Interval
+				t.Stop()
+				t = time.NewTicker(g.Interval)
+			}
+			g.mu.Unlock()
+			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
+		case <-t.C:
+			missed := (time.Since(evalTS) / g.Interval) - 1
+			if missed > 0 {
+				g.metrics.iterationMissed.Inc()
+			}
+			evalTS = evalTS.Add((missed + 1) * g.Interval)
+
+			eval(evalCtx, evalTS)
+		}
+	}
+}
+
+// getResolveDuration returns the duration after which firing alert
+// can be considered as resolved.
+func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Duration {
+	if groupInterval > delta {
+		delta = groupInterval
+	}
+	resolveDuration := delta * 4
+	if maxDuration > 0 && resolveDuration > maxDuration {
+		resolveDuration = maxDuration
+	}
+	return resolveDuration
+}
+
+type executor struct {
+	notifiers func() []notifier.Notifier
+	rw        *remotewrite.Client
+
+	previouslySentSeriesToRWMu sync.Mutex
+	// previouslySentSeriesToRW stores series sent to RW on previous iteration
+	// map[ruleID]map[ruleLabels][]prompb.Label
+	// where `ruleID` is ID of the Rule within a Group
+	// and `ruleLabels` is []prompb.Label marshalled to a string
+	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
+}
+
+func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {
+	res := make(chan error, len(rules))
+	if concurrency == 1 {
+		// fast path
+		for _, rule := range rules {
+			res <- e.exec(ctx, rule, ts, resolveDuration, limit)
+		}
+		close(res)
+		return res
+	}
+
+	sem := make(chan struct{}, concurrency)
+	go func() {
+		wg := sync.WaitGroup{}
+		for _, rule := range rules {
+			sem <- struct{}{}
+			wg.Add(1)
+			go func(r Rule) {
+				res <- e.exec(ctx, r, ts, resolveDuration, limit)
+				<-sem
+				wg.Done()
+			}(rule)
+		}
+		wg.Wait()
+		close(res)
+	}()
+	return res
+}
+
+var (
+	alertsFired = metrics.NewCounter(`vmalert_alerts_fired_total`)
+
+	execTotal  = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)
+
+	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+	remoteWriteTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
+)
+
+func (e *executor) exec(ctx context.Context, rule Rule, ts time.Time, resolveDuration time.Duration, limit int) error {
+	execTotal.Inc()
+
+	tss, err := rule.Exec(ctx, ts, limit)
+	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			// the context can be cancelled on graceful shutdown
+			// or on group update. So no need to handle the error as usual.
+			return nil
+		}
+		execErrors.Inc()
+		return fmt.Errorf("rule %q: failed to execute: %w", rule, err)
+	}
+
+	if e.rw != nil {
+		pushToRW := func(tss []prompbmarshal.TimeSeries) error {
+			var lastErr error
+			for _, ts := range tss {
+				remoteWriteTotal.Inc()
+				if err := e.rw.Push(ts); err != nil {
+					remoteWriteErrors.Inc()
+					lastErr = fmt.Errorf("rule %q: remote write failure: %w", rule, err)
+				}
+			}
+			return lastErr
+		}
+		if err := pushToRW(tss); err != nil {
+			return err
+		}
+
+		staleSeries := e.getStaleSeries(rule, tss, ts)
+		if err := pushToRW(staleSeries); err != nil {
+			return err
+		}
+	}
+
+	ar, ok := rule.(*AlertingRule)
+	if !ok {
+		return nil
+	}
+
+	alerts := ar.alertsToSend(ts, resolveDuration, *resendDelay)
+	if len(alerts) < 1 {
+		return nil
+	}
+
+	wg := sync.WaitGroup{}
+	errGr := new(utils.ErrGroup)
+	for _, nt := range e.notifiers() {
+		wg.Add(1)
+		go func(nt notifier.Notifier) {
+			if err := nt.Send(ctx, alerts); err != nil {
+				errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", rule, nt.Addr(), err))
+			}
+			wg.Done()
+		}(nt)
+	}
+	wg.Wait()
+	return errGr.Err()
+}
+
+// getStaledSeries checks whether there are stale series from previously sent ones.
+func (e *executor) getStaleSeries(rule Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
+	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
+	for _, ts := range tss {
+		// convert labels to strings so we can compare with previously sent series
+		key := labelsToString(ts.Labels)
+		ruleLabels[key] = ts.Labels
+	}
+
+	rID := rule.ID()
+	var staleS []prompbmarshal.TimeSeries
+	// check whether there are series which disappeared and need to be marked as stale
+	e.previouslySentSeriesToRWMu.Lock()
+	for key, labels := range e.previouslySentSeriesToRW[rID] {
+		if _, ok := ruleLabels[key]; ok {
+			continue
+		}
+		// previously sent series are missing in current series, so we mark them as stale
+		ss := newTimeSeriesPB([]float64{decimal.StaleNaN}, []int64{timestamp.Unix()}, labels)
+		staleS = append(staleS, ss)
+	}
+	// set previous series to current
+	e.previouslySentSeriesToRW[rID] = ruleLabels
+	e.previouslySentSeriesToRWMu.Unlock()
+
+	return staleS
+}
+
+// purgeStaleSeries deletes references in tracked
+// previouslySentSeriesToRW list to Rules which aren't present
+// in the given activeRules list. The method is used when the list
+// of loaded rules has changed and executor has to remove
+// references to non-existing rules.
+func (e *executor) purgeStaleSeries(activeRules []Rule) {
+	newPreviouslySentSeriesToRW := make(map[uint64]map[string][]prompbmarshal.Label)
+
+	e.previouslySentSeriesToRWMu.Lock()
+
+	for _, rule := range activeRules {
+		id := rule.ID()
+		prev, ok := e.previouslySentSeriesToRW[id]
+		if ok {
+			// keep previous series for staleness detection
+			newPreviouslySentSeriesToRW[id] = prev
+		}
+	}
+	e.previouslySentSeriesToRW = nil
+	e.previouslySentSeriesToRW = newPreviouslySentSeriesToRW
+
+	e.previouslySentSeriesToRWMu.Unlock()
+}
+
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b strings.Builder
+	b.WriteRune('{')
+	for i, label := range labels {
+		if len(label.Name) == 0 {
+			b.WriteString("__name__")
+		} else {
+			b.WriteString(label.Name)
+		}
+		b.WriteRune('=')
+		b.WriteString(strconv.Quote(label.Value))
+		if i < len(labels)-1 {
+			b.WriteRune(',')
+		}
+	}
+	b.WriteRune('}')
+	return b.String()
+}

app/vmalert/group_test.go

@@ -0,0 +1,504 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func init() {
+	// Disable rand sleep on group start during tests in order to speed up test execution.
+	// Rand sleep is needed only in prod code.
+	skipRandSleepOnGroupStart = true
+}
+
+func TestUpdateWith(t *testing.T) {
+	testCases := []struct {
+		name         string
+		currentRules []config.Rule
+		newRules     []config.Rule
+	}{
+		{
+			"new rule",
+			nil,
+			[]config.Rule{{Alert: "bar"}},
+		},
+		{
+			"update alerting rule",
+			[]config.Rule{{
+				Alert: "foo",
+				Expr:  "up > 0",
+				For:   promutils.NewDuration(time.Second),
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+				Annotations: map[string]string{
+					"summary":     "{{ $value|humanize }}",
+					"description": "{{$labels}}",
+				},
+			}},
+			[]config.Rule{{
+				Alert: "foo",
+				Expr:  "up > 10",
+				For:   promutils.NewDuration(time.Second),
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+				Annotations: map[string]string{
+					"summary": "none",
+				},
+			}},
+		},
+		{
+			"update recording rule",
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "max(up)",
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+			}},
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "min(up)",
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+			}},
+		},
+		{
+			"empty rule",
+			[]config.Rule{{Alert: "foo"}, {Record: "bar"}},
+			nil,
+		},
+		{
+			"multiple rules",
+			[]config.Rule{
+				{Alert: "bar"},
+				{Alert: "baz"},
+				{Alert: "foo"},
+			},
+			[]config.Rule{
+				{Alert: "baz"},
+				{Record: "foo"},
+			},
+		},
+		{
+			"replace rule",
+			[]config.Rule{{Alert: "foo1"}},
+			[]config.Rule{{Alert: "foo2"}},
+		},
+		{
+			"replace multiple rules",
+			[]config.Rule{
+				{Alert: "foo1"},
+				{Record: "foo2"},
+				{Alert: "foo3"},
+			},
+			[]config.Rule{
+				{Alert: "foo3"},
+				{Alert: "foo4"},
+				{Record: "foo5"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := &Group{Name: "test"}
+			qb := &fakeQuerier{}
+			for _, r := range tc.currentRules {
+				r.ID = config.HashRule(r)
+				g.Rules = append(g.Rules, g.newRule(qb, r))
+			}
+
+			ng := &Group{Name: "test"}
+			for _, r := range tc.newRules {
+				r.ID = config.HashRule(r)
+				ng.Rules = append(ng.Rules, ng.newRule(qb, r))
+			}
+
+			err := g.updateWith(ng)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if len(g.Rules) != len(tc.newRules) {
+				t.Fatalf("expected to have %d rules; got: %d",
+					len(g.Rules), len(tc.newRules))
+			}
+			sort.Slice(g.Rules, func(i, j int) bool {
+				return g.Rules[i].ID() < g.Rules[j].ID()
+			})
+			sort.Slice(ng.Rules, func(i, j int) bool {
+				return ng.Rules[i].ID() < ng.Rules[j].ID()
+			})
+			for i, r := range g.Rules {
+				got, want := r, ng.Rules[i]
+				if got.ID() != want.ID() {
+					t.Fatalf("expected to have rule %q; got %q", want, got)
+				}
+				if err := compareRules(t, got, want); err != nil {
+					t.Fatalf("comparison error: %s", err)
+				}
+			}
+		})
+	}
+}
+
+func TestGroupStart(t *testing.T) {
+	// TODO: make parsing from string instead of file
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, notifier.ValidateTemplates, true)
+	if err != nil {
+		t.Fatalf("failed to parse rules: %s", err)
+	}
+
+	fs := &fakeQuerier{}
+	fn := &fakeNotifier{}
+
+	const evalInterval = time.Millisecond
+	g := newGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
+	g.Concurrency = 2
+
+	const inst1, inst2, job = "foo", "bar", "baz"
+	m1 := metricWithLabels(t, "instance", inst1, "job", job)
+	m2 := metricWithLabels(t, "instance", inst2, "job", job)
+
+	r := g.Rules[0].(*AlertingRule)
+	alert1, err := r.newAlert(m1, nil, time.Now(), nil)
+	if err != nil {
+		t.Fatalf("faield to create alert: %s", err)
+	}
+	alert1.State = notifier.StateFiring
+	// add external label
+	alert1.Labels["cluster"] = "east-1"
+	// add rule labels - see config/testdata/rules1-good.rules
+	alert1.Labels["label"] = "bar"
+	alert1.Labels["host"] = inst1
+	// add service labels
+	alert1.Labels[alertNameLabel] = alert1.Name
+	alert1.Labels[alertGroupNameLabel] = g.Name
+	alert1.ID = hash(alert1.Labels)
+
+	alert2, err := r.newAlert(m2, nil, time.Now(), nil)
+	if err != nil {
+		t.Fatalf("faield to create alert: %s", err)
+	}
+	alert2.State = notifier.StateFiring
+	// add external label
+	alert2.Labels["cluster"] = "east-1"
+	// add rule labels - see config/testdata/rules1-good.rules
+	alert2.Labels["label"] = "bar"
+	alert2.Labels["host"] = inst2
+	// add service labels
+	alert2.Labels[alertNameLabel] = alert2.Name
+	alert2.Labels[alertGroupNameLabel] = g.Name
+	alert2.ID = hash(alert2.Labels)
+
+	finished := make(chan struct{})
+	fs.add(m1)
+	fs.add(m2)
+	go func() {
+		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil, fs)
+		close(finished)
+	}()
+
+	// wait for multiple evals
+	time.Sleep(20 * evalInterval)
+
+	gotAlerts := fn.getAlerts()
+	expectedAlerts := []notifier.Alert{*alert1, *alert2}
+	compareAlerts(t, expectedAlerts, gotAlerts)
+
+	gotAlertsNum := fn.getCounter()
+	if gotAlertsNum < len(expectedAlerts)*2 {
+		t.Fatalf("expected to receive at least %d alerts; got %d instead",
+			len(expectedAlerts)*2, gotAlertsNum)
+	}
+
+	// reset previous data
+	fs.reset()
+	// and set only one datapoint for response
+	fs.add(m1)
+
+	// wait for multiple evals
+	time.Sleep(20 * evalInterval)
+
+	gotAlerts = fn.getAlerts()
+	alert2.State = notifier.StateInactive
+	expectedAlerts = []notifier.Alert{*alert1, *alert2}
+	compareAlerts(t, expectedAlerts, gotAlerts)
+
+	g.close()
+	<-finished
+}
+
+func TestResolveDuration(t *testing.T) {
+	testCases := []struct {
+		groupInterval time.Duration
+		maxDuration   time.Duration
+		resendDelay   time.Duration
+		expected      time.Duration
+	}{
+		{time.Minute, 0, 0, 4 * time.Minute},
+		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
+		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
+		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
+		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
+		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
+		{0, 0, 0, 0},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
+			got := getResolveDuration(tc.groupInterval, tc.resendDelay, tc.maxDuration)
+			if got != tc.expected {
+				t.Errorf("expected to have %v; got %v", tc.expected, got)
+			}
+		})
+	}
+}
+
+func TestGetStaleSeries(t *testing.T) {
+	ts := time.Now()
+	e := &executor{
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+	f := func(rule Rule, labels, expLabels [][]prompbmarshal.Label) {
+		t.Helper()
+		var tss []prompbmarshal.TimeSeries
+		for _, l := range labels {
+			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
+		}
+		staleS := e.getStaleSeries(rule, tss, ts)
+		if staleS == nil && expLabels == nil {
+			return
+		}
+		if len(staleS) != len(expLabels) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expLabels), len(staleS))
+		}
+		for i, exp := range expLabels {
+			got := staleS[i]
+			if !reflect.DeepEqual(exp, got.Labels) {
+				t.Fatalf("expected to get labels: \n%v;\ngot instead: \n%v",
+					exp, got.Labels)
+			}
+			if len(got.Samples) != 1 {
+				t.Fatalf("expected to have 1 sample; got %d", len(got.Samples))
+			}
+			if !decimal.IsStaleNaN(got.Samples[0].Value) {
+				t.Fatalf("expected sample value to be %v; got %v", decimal.StaleNaN, got.Samples[0].Value)
+			}
+		}
+	}
+
+	// warn: keep in mind, that executor holds the state, so sequence of f calls matters
+
+	// single series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		nil)
+
+	// multiple series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")})
+
+	// multiple rules and series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 2},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+}
+
+func TestPurgeStaleSeries(t *testing.T) {
+	ts := time.Now()
+	labels := toPromLabels(t, "__name__", "job:foo", "job", "foo")
+	tss := []prompbmarshal.TimeSeries{newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels)}
+
+	f := func(curRules, newRules, expStaleRules []Rule) {
+		t.Helper()
+		e := &executor{
+			previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		}
+		// seed executor with series for
+		// current rules
+		for _, rule := range curRules {
+			e.getStaleSeries(rule, tss, ts)
+		}
+
+		e.purgeStaleSeries(newRules)
+
+		if len(e.previouslySentSeriesToRW) != len(expStaleRules) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expStaleRules), len(e.previouslySentSeriesToRW))
+		}
+
+		for _, exp := range expStaleRules {
+			if _, ok := e.previouslySentSeriesToRW[exp.ID()]; !ok {
+				t.Fatalf("expected to have rule %d; got nil instead", exp.ID())
+			}
+		}
+	}
+
+	f(nil, nil, nil)
+	f(
+		nil,
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+	)
+}
+
+func TestFaultyNotifier(t *testing.T) {
+	fq := &fakeQuerier{}
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+
+	r := newTestAlertingRule("instant", 0)
+	r.q = fq
+
+	fn := &fakeNotifier{}
+	e := &executor{
+		notifiers: func() []notifier.Notifier {
+			return []notifier.Notifier{
+				&faultyNotifier{},
+				fn,
+			}
+		},
+	}
+	delay := 5 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), delay)
+	defer cancel()
+
+	go func() {
+		_ = e.exec(ctx, r, time.Now(), 0, 10)
+	}()
+
+	tn := time.Now()
+	deadline := tn.Add(delay / 2)
+	for {
+		if fn.getCounter() > 0 {
+			return
+		}
+		if tn.After(deadline) {
+			break
+		}
+		tn = time.Now()
+		time.Sleep(time.Millisecond * 100)
+	}
+	t.Fatalf("alive notifier didn't receive notification by %v", deadline)
+}
+
+func TestFaultyRW(t *testing.T) {
+	fq := &fakeQuerier{}
+	fq.add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+
+	r := &RecordingRule{
+		Name:  "test",
+		state: newRuleState(10),
+		q:     fq,
+	}
+
+	e := &executor{
+		rw:                       &remotewrite.Client{},
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+
+	err := e.exec(context.Background(), r, time.Now(), 0, 10)
+	if err == nil {
+		t.Fatalf("expected to get an error from faulty RW client, got nil instead")
+	}
+}
+
+func TestCloseWithEvalInterruption(t *testing.T) {
+	groups, err := config.Parse([]string{"config/testdata/rules/rules1-good.rules"}, notifier.ValidateTemplates, true)
+	if err != nil {
+		t.Fatalf("failed to parse rules: %s", err)
+	}
+
+	const delay = time.Second * 2
+	fq := &fakeQuerierWithDelay{delay: delay}
+
+	const evalInterval = time.Millisecond
+	g := newGroup(groups[0], fq, evalInterval, nil)
+
+	go g.start(context.Background(), nil, nil, nil)
+
+	time.Sleep(evalInterval * 20)
+
+	go func() {
+		g.close()
+	}()
+
+	deadline := time.Tick(delay / 2)
+	select {
+	case <-deadline:
+		t.Fatalf("deadline for close exceeded")
+	case <-g.finishedCh:
+	}
+}

app/vmalert/helpers_test.go

@@ -0,0 +1,354 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"reflect"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+type fakeQuerier struct {
+	sync.Mutex
+	metrics []datasource.Metric
+	err     error
+}
+
+func (fq *fakeQuerier) setErr(err error) {
+	fq.Lock()
+	fq.err = err
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) reset() {
+	fq.Lock()
+	fq.err = nil
+	fq.metrics = fq.metrics[:0]
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) add(metrics ...datasource.Metric) {
+	fq.Lock()
+	fq.metrics = append(fq.metrics, metrics...)
+	fq.Unlock()
+}
+
+func (fq *fakeQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fq
+}
+
+func (fq *fakeQuerier) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
+	req, _, err := fq.Query(ctx, q, time.Now())
+	return req, err
+}
+
+func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]datasource.Metric, *http.Request, error) {
+	fq.Lock()
+	defer fq.Unlock()
+	if fq.err != nil {
+		return nil, nil, fq.err
+	}
+	cp := make([]datasource.Metric, len(fq.metrics))
+	copy(cp, fq.metrics)
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	return cp, req, nil
+}
+
+type fakeQuerierWithRegistry struct {
+	sync.Mutex
+	registry map[string][]datasource.Metric
+}
+
+func (fqr *fakeQuerierWithRegistry) set(key string, metrics ...datasource.Metric) {
+	fqr.Lock()
+	if fqr.registry == nil {
+		fqr.registry = make(map[string][]datasource.Metric)
+	}
+	fqr.registry[key] = metrics
+	fqr.Unlock()
+}
+
+func (fqr *fakeQuerierWithRegistry) reset() {
+	fqr.Lock()
+	fqr.registry = nil
+	fqr.Unlock()
+}
+
+func (fqr *fakeQuerierWithRegistry) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fqr
+}
+
+func (fqr *fakeQuerierWithRegistry) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
+	req, _, err := fqr.Query(ctx, q, time.Now())
+	return req, err
+}
+
+func (fqr *fakeQuerierWithRegistry) Query(_ context.Context, expr string, _ time.Time) ([]datasource.Metric, *http.Request, error) {
+	fqr.Lock()
+	defer fqr.Unlock()
+
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	metrics, ok := fqr.registry[expr]
+	if !ok {
+		return nil, req, nil
+	}
+	cp := make([]datasource.Metric, len(metrics))
+	copy(cp, metrics)
+	return cp, req, nil
+}
+
+type fakeQuerierWithDelay struct {
+	fakeQuerier
+	delay time.Duration
+}
+
+func (fqd *fakeQuerierWithDelay) Query(ctx context.Context, expr string, ts time.Time) ([]datasource.Metric, *http.Request, error) {
+	timer := time.NewTimer(fqd.delay)
+	select {
+	case <-ctx.Done():
+	case <-timer.C:
+	}
+	return fqd.fakeQuerier.Query(ctx, expr, ts)
+}
+
+func (fqd *fakeQuerierWithDelay) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fqd
+}
+
+type fakeNotifier struct {
+	sync.Mutex
+	alerts []notifier.Alert
+	// records number of received alerts in total
+	counter int
+}
+
+func (*fakeNotifier) Close()       {}
+func (*fakeNotifier) Addr() string { return "" }
+func (fn *fakeNotifier) Send(_ context.Context, alerts []notifier.Alert) error {
+	fn.Lock()
+	defer fn.Unlock()
+	fn.counter += len(alerts)
+	fn.alerts = alerts
+	return nil
+}
+
+func (fn *fakeNotifier) getCounter() int {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.counter
+}
+
+func (fn *fakeNotifier) getAlerts() []notifier.Alert {
+	fn.Lock()
+	defer fn.Unlock()
+	return fn.alerts
+}
+
+type faultyNotifier struct {
+	fakeNotifier
+}
+
+func (fn *faultyNotifier) Send(ctx context.Context, _ []notifier.Alert) error {
+	d, ok := ctx.Deadline()
+	if ok {
+		time.Sleep(time.Until(d))
+	}
+	return fmt.Errorf("send failed")
+}
+
+func metricWithValueAndLabels(t *testing.T, value float64, labels ...string) datasource.Metric {
+	return metricWithValuesAndLabels(t, []float64{value}, labels...)
+}
+
+func metricWithValuesAndLabels(t *testing.T, values []float64, labels ...string) datasource.Metric {
+	t.Helper()
+	m := metricWithLabels(t, labels...)
+	m.Values = values
+	for i := range values {
+		m.Timestamps = append(m.Timestamps, int64(i))
+	}
+	return m
+}
+
+func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	m := datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}
+	for i := 0; i < len(labels); i += 2 {
+		m.Labels = append(m.Labels, datasource.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return m
+}
+
+func toPromLabels(t *testing.T, labels ...string) []prompbmarshal.Label {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	var ls []prompbmarshal.Label
+	for i := 0; i < len(labels); i += 2 {
+		ls = append(ls, prompbmarshal.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return ls
+}
+
+func compareGroups(t *testing.T, a, b *Group) {
+	t.Helper()
+	if a.Name != b.Name {
+		t.Fatalf("expected group name %q; got %q", a.Name, b.Name)
+	}
+	if a.File != b.File {
+		t.Fatalf("expected group %q file name %q; got %q", a.Name, a.File, b.File)
+	}
+	if a.Interval != b.Interval {
+		t.Fatalf("expected group %q interval %v; got %v", a.Name, a.Interval, b.Interval)
+	}
+	if len(a.Rules) != len(b.Rules) {
+		t.Fatalf("expected group %s to have %d rules; got: %d",
+			a.Name, len(a.Rules), len(b.Rules))
+	}
+	for i, r := range a.Rules {
+		got, want := r, b.Rules[i]
+		if a.ID() != b.ID() {
+			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
+		}
+		if err := compareRules(t, want, got); err != nil {
+			t.Fatalf("comparison error: %s", err)
+		}
+	}
+}
+
+func compareRules(t *testing.T, a, b Rule) error {
+	t.Helper()
+	switch v := a.(type) {
+	case *AlertingRule:
+		br, ok := b.(*AlertingRule)
+		if !ok {
+			return fmt.Errorf("rule %q supposed to be of type AlertingRule", b.ID())
+		}
+		return compareAlertingRules(t, v, br)
+	case *RecordingRule:
+		br, ok := b.(*RecordingRule)
+		if !ok {
+			return fmt.Errorf("rule %q supposed to be of type RecordingRule", b.ID())
+		}
+		return compareRecordingRules(t, v, br)
+	default:
+		return fmt.Errorf("unexpected rule type received %T", a)
+	}
+}
+
+func compareRecordingRules(t *testing.T, a, b *RecordingRule) error {
+	t.Helper()
+	if a.Expr != b.Expr {
+		return fmt.Errorf("expected to have expression %q; got %q", a.Expr, b.Expr)
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) {
+		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+	}
+	return nil
+}
+
+func compareAlertingRules(t *testing.T, a, b *AlertingRule) error {
+	t.Helper()
+	if a.Expr != b.Expr {
+		return fmt.Errorf("expected to have expression %q; got %q", a.Expr, b.Expr)
+	}
+	if a.For != b.For {
+		return fmt.Errorf("expected to have for %q; got %q", a.For, b.For)
+	}
+	if !reflect.DeepEqual(a.Annotations, b.Annotations) {
+		return fmt.Errorf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) {
+		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+	}
+	if a.Type.String() != b.Type.String() {
+		return fmt.Errorf("expected to have Type %#v; got %#v", a.Type.String(), b.Type.String())
+	}
+	return nil
+}
+
+func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
+	t.Helper()
+	if len(a) != len(b) {
+		return fmt.Errorf("expected number of timeseries %d; got %d", len(a), len(b))
+	}
+	for i := range a {
+		expTS, gotTS := a[i], b[i]
+		if len(expTS.Samples) != len(gotTS.Samples) {
+			return fmt.Errorf("expected number of samples %d; got %d", len(expTS.Samples), len(gotTS.Samples))
+		}
+		for i, exp := range expTS.Samples {
+			got := gotTS.Samples[i]
+			if got.Value != exp.Value {
+				return fmt.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
+			}
+			// timestamp validation isn't always correct for now.
+			// this must be improved with time mock.
+			/*if got.Timestamp != exp.Timestamp {
+				return fmt.Errorf("expected timestamp %d; got %d", exp.Timestamp, got.Timestamp)
+			}*/
+		}
+		if len(expTS.Labels) != len(gotTS.Labels) {
+			return fmt.Errorf("expected number of labels %d (%v); got %d (%v)",
+				len(expTS.Labels), expTS.Labels, len(gotTS.Labels), gotTS.Labels)
+		}
+		for i, exp := range expTS.Labels {
+			got := gotTS.Labels[i]
+			if got.Name != exp.Name {
+				return fmt.Errorf("expected label name %q; got %q", exp.Name, got.Name)
+			}
+			if got.Value != exp.Value {
+				return fmt.Errorf("expected label value %q; got %q", exp.Value, got.Value)
+			}
+		}
+	}
+	return nil
+}
+
+func compareAlerts(t *testing.T, as, bs []notifier.Alert) {
+	t.Helper()
+	if len(as) != len(bs) {
+		t.Fatalf("expected to have length %d; got %d", len(as), len(bs))
+	}
+	sort.Slice(as, func(i, j int) bool {
+		return as[i].ID < as[j].ID
+	})
+	sort.Slice(bs, func(i, j int) bool {
+		return bs[i].ID < bs[j].ID
+	})
+	for i := range as {
+		a, b := as[i], bs[i]
+		if a.Name != b.Name {
+			t.Fatalf("expected t have Name %q; got %q", a.Name, b.Name)
+		}
+		if a.State != b.State {
+			t.Fatalf("expected t have State %q; got %q", a.State, b.State)
+		}
+		if a.Value != b.Value {
+			t.Fatalf("expected t have Value %f; got %f", a.Value, b.Value)
+		}
+		if !reflect.DeepEqual(a.Annotations, b.Annotations) {
+			t.Fatalf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
+		}
+		if !reflect.DeepEqual(a.Labels, b.Labels) {
+			t.Fatalf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
+		}
+	}
+}

app/vmalert/main.go

@@ -0,0 +1,387 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rulePath = flagutil.NewArrayString("rule", `Path to the files with alerting and/or recording rules.
+Supports hierarchical patterns and regexpes.
+Examples:
+ -rule="/path/to/file". Path to a single file with alerting rules
+ -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". 
+Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
+
+Enterprise version of vmalert supports S3 and GCS paths to rules.
+For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
+S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
+all files with prefix rule_ in folder dir.
+See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
+`)
+
+	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions
+	for rules annotations templating. Flag can be specified multiple times.
+Examples:
+ -rule.templates="/path/to/file". Path to a single file with go templates
+ -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
+absolute path to all .tpl files in root.`)
+
+	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
+
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
+
+	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
+
+	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
+	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
+	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
+		"which by default is 4 times evaluationInterval of the parent group.")
+	resendDelay            = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
+	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
+		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overriden per rule via update_entries_limit param.")
+
+	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
+	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
+		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
+		`Supports templating - see https://docs.victoriametrics.com/vmalert.html#templating . `+
+		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' . `+
+		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
+	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
+		"Pass multiple -label flags in order to add multiple label sets.")
+
+	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
+		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases.")
+
+	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
+
+	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
+)
+
+var alertURLGeneratorFn notifier.AlertURLGenerator
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	remoteread.InitSecretFlags()
+	remotewrite.InitSecretFlags()
+	datasource.InitSecretFlags()
+	buildinfo.Init()
+	logger.Init()
+	pushmetrics.Init()
+
+	if !*remoteReadIgnoreRestoreErrors {
+		logger.Warnf("flag `remoteRead.ignoreRestoreErrors` is deprecated and will be removed in next releases.")
+	}
+
+	err := templates.Load(*ruleTemplatesPath, true)
+	if err != nil {
+		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)
+	}
+
+	if *dryRun {
+		groups, err := config.Parse(*rulePath, notifier.ValidateTemplates, true)
+		if err != nil {
+			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
+		}
+		if len(groups) == 0 {
+			logger.Fatalf("No rules for validation. Please specify path to file(s) with alerting and/or recording rules using `-rule` flag")
+		}
+		return
+	}
+
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		logger.Fatalf("failed to init `external.url`: %s", err)
+	}
+
+	alertURLGeneratorFn, err = getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
+	if err != nil {
+		logger.Fatalf("failed to init `external.alert.source`: %s", err)
+	}
+
+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
+	if *replayFrom != "" || *replayTo != "" {
+		rw, err := remotewrite.Init(context.Background())
+		if err != nil {
+			logger.Fatalf("failed to init remoteWrite: %s", err)
+		}
+		if rw == nil {
+			logger.Fatalf("remoteWrite.url can't be empty in replay mode")
+		}
+		groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
+		if err != nil {
+			logger.Fatalf("cannot parse configuration file: %s", err)
+		}
+		// prevent queries from caching and boundaries aligning
+		// when querying VictoriaMetrics datasource.
+		q, err := datasource.Init(url.Values{"nocache": {"1"}})
+		if err != nil {
+			logger.Fatalf("failed to init datasource: %s", err)
+		}
+		if err := replay(groupsCfg, q, rw); err != nil {
+			logger.Fatalf("replay failed: %s", err)
+		}
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	manager, err := newManager(ctx)
+	if err != nil {
+		logger.Fatalf("failed to init: %s", err)
+	}
+	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
+	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)
+	if err != nil {
+		logger.Fatalf("cannot parse configuration file: %s", err)
+	}
+
+	// Register SIGHUP handler for config re-read just before manager.start call.
+	// This guarantees that the config will be re-read if the signal arrives during manager.start call.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
+	sighupCh := procutil.NewSighupChan()
+
+	if err := manager.start(ctx, groupsCfg); err != nil {
+		logger.Fatalf("failed to start: %s", err)
+	}
+
+	go configReload(ctx, manager, groupsCfg, sighupCh)
+
+	rh := &requestHandler{m: manager}
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	cancel()
+	manager.close()
+}
+
+var (
+	configReloads      = metrics.NewCounter(`vmalert_config_last_reload_total`)
+	configReloadErrors = metrics.NewCounter(`vmalert_config_last_reload_errors_total`)
+	configSuccess      = metrics.NewCounter(`vmalert_config_last_reload_successful`)
+	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
+)
+
+func newManager(ctx context.Context) (*manager, error) {
+	q, err := datasource.Init(nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init datasource: %w", err)
+	}
+
+	labels := make(map[string]string)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init notifier: %w", err)
+	}
+	manager := &manager{
+		groups:         make(map[uint64]*Group),
+		querierBuilder: q,
+		notifiers:      nts,
+		labels:         labels,
+	}
+	rw, err := remotewrite.Init(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init remoteWrite: %w", err)
+	}
+	manager.rw = rw
+
+	rr, err := remoteread.Init()
+	if err != nil {
+		return nil, fmt.Errorf("failed to init remoteRead: %w", err)
+	}
+	manager.rr = rr
+
+	return manager, nil
+}
+
+func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
+	if externalURL != "" {
+		return url.Parse(externalURL)
+	}
+	hname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+	port := ""
+	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
+		port = ":" + ipport[1]
+	}
+	schema := "http://"
+	if isSecure {
+		schema = "https://"
+	}
+	return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port))
+}
+
+func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, validateTemplate bool) (notifier.AlertURLGenerator, error) {
+	if externalAlertSource == "" {
+		return func(a notifier.Alert) string {
+			gID, aID := strconv.FormatUint(a.GroupID, 10), strconv.FormatUint(a.ID, 10)
+			return fmt.Sprintf("%s/vmalert/alert?%s=%s&%s=%s", externalURL, paramGroupID, gID, paramAlertID, aID)
+		}, nil
+	}
+	if validateTemplate {
+		if err := notifier.ValidateTemplates(map[string]string{
+			"tpl": externalAlertSource,
+		}); err != nil {
+			return nil, fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
+		}
+	}
+	m := map[string]string{
+		"tpl": externalAlertSource,
+	}
+	return func(alert notifier.Alert) string {
+		templated, err := alert.ExecTemplate(nil, alert.Labels, m)
+		if err != nil {
+			logger.Errorf("can not exec source template %s", err)
+		}
+		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
+	}, nil
+}
+
+func usage() {
+	const s = `
+vmalert processes alerts and recording rules.
+
+See the docs at https://docs.victoriametrics.com/vmalert.html .
+`
+	flagutil.Usage(s)
+}
+
+func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
+	var configCheckCh <-chan time.Time
+	checkInterval := *configCheckInterval
+	if checkInterval == 0 && *rulesCheckInterval > 0 {
+		logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead")
+		checkInterval = *rulesCheckInterval
+	}
+	if checkInterval > 0 {
+		ticker := time.NewTicker(checkInterval)
+		configCheckCh = ticker.C
+		defer ticker.Stop()
+	}
+
+	var validateTplFn config.ValidateTplFn
+	if *validateTemplates {
+		validateTplFn = notifier.ValidateTemplates
+	}
+
+	// init reload metrics with positive values to improve alerting conditions
+	configSuccess.Set(1)
+	configTimestamp.Set(fasttime.UnixTimestamp())
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-sighupCh:
+			tmplMsg := ""
+			if len(*ruleTemplatesPath) > 0 {
+				tmplMsg = fmt.Sprintf("and templates %q ", *ruleTemplatesPath)
+			}
+			logger.Infof("SIGHUP received. Going to reload rules %q %s...", *rulePath, tmplMsg)
+			configReloads.Inc()
+		case <-configCheckCh:
+		}
+		if err := notifier.Reload(); err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to reload notifier config: %s", err)
+			continue
+		}
+		err := templates.Load(*ruleTemplatesPath, false)
+		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("failed to load new templates: %s", err)
+			continue
+		}
+		newGroupsCfg, err := config.ParseSilent(*rulePath, validateTplFn, *validateExpressions)
+		if err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("cannot parse configuration file: %s", err)
+			continue
+		}
+		if configsEqual(newGroupsCfg, groupsCfg) {
+			templates.Reload()
+			// set success to 1 since previous reload
+			// could have been unsuccessful
+			configSuccess.Set(1)
+			// config didn't change - skip it
+			continue
+		}
+		if err := m.update(ctx, newGroupsCfg, false); err != nil {
+			configReloadErrors.Inc()
+			configSuccess.Set(0)
+			logger.Errorf("error while reloading rules: %s", err)
+			continue
+		}
+		templates.Reload()
+		groupsCfg = newGroupsCfg
+		configSuccess.Set(1)
+		configTimestamp.Set(fasttime.UnixTimestamp())
+		logger.Infof("Rules reloaded successfully from %q", *rulePath)
+	}
+}
+
+func configsEqual(a, b []config.Group) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i].Checksum != b[i].Checksum {
+			return false
+		}
+	}
+	return true
+}

app/vmalert/main_test.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+)
+
+func TestGetExternalURL(t *testing.T) {
+	expURL := "https://vicotriametrics.com/path"
+	u, err := getExternalURL(expURL, "", false)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if u.String() != expURL {
+		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+	}
+	h, _ := os.Hostname()
+	expURL = fmt.Sprintf("https://%s:4242", h)
+	u, err = getExternalURL("", "0.0.0.0:4242", true)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if u.String() != expURL {
+		t.Errorf("unexpected url want %s, got %s", expURL, u.String())
+	}
+}
+
+func TestGetAlertURLGenerator(t *testing.T) {
+	testAlert := notifier.Alert{GroupID: 42, ID: 2, Value: 4, Labels: map[string]string{"tenant": "baz"}}
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	fn, err := getAlertURLGenerator(u, "", false)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
+	if exp != fn(testAlert) {
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+	}
+	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
+	if err == nil {
+		t.Errorf("expected template validation error got nil")
+	}
+	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+	}
+}
+
+func TestConfigReload(t *testing.T) {
+	originalRulePath := *rulePath
+	defer func() {
+		*rulePath = originalRulePath
+	}()
+
+	const (
+		rules1 = `
+groups:
+  - name: group-1
+    rules:
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job) (up == 1)
+      - record: handler:requests:rate5m 
+        expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+`
+		rules2 = `
+groups:
+  - name: group-1
+    rules:
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job) (up == 1)
+  - name: group-2
+    rules:
+      - record: handler:requests:rate5m 
+        expr: sum(rate(prometheus_http_requests_total[5m])) by (handler)
+`
+	)
+
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	writeToFile(t, f.Name(), rules1)
+
+	*rulesCheckInterval = 200 * time.Millisecond
+	*rulePath = []string{f.Name()}
+	ctx, cancel := context.WithCancel(context.Background())
+
+	m := &manager{
+		querierBuilder: &fakeQuerier{},
+		groups:         make(map[uint64]*Group),
+		labels:         map[string]string{},
+		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} },
+		rw:             &remotewrite.Client{},
+	}
+
+	syncCh := make(chan struct{})
+	sighupCh := procutil.NewSighupChan()
+	go func() {
+		configReload(ctx, m, nil, sighupCh)
+		close(syncCh)
+	}()
+
+	lenLocked := func(m *manager) int {
+		m.groupsMu.RLock()
+		defer m.groupsMu.RUnlock()
+		return len(m.groups)
+	}
+
+	time.Sleep(*rulesCheckInterval * 2)
+	groupsLen := lenLocked(m)
+	if groupsLen != 1 {
+		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
+	}
+
+	writeToFile(t, f.Name(), rules2)
+	time.Sleep(*rulesCheckInterval * 2)
+	groupsLen = lenLocked(m)
+	if groupsLen != 2 {
+		fmt.Println(m.groups)
+		t.Fatalf("expected to have exactly 2 groups loaded; got %d", groupsLen)
+	}
+
+	writeToFile(t, f.Name(), rules1)
+	procutil.SelfSIGHUP()
+	time.Sleep(*rulesCheckInterval / 2)
+	groupsLen = lenLocked(m)
+	if groupsLen != 1 {
+		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
+	}
+
+	writeToFile(t, f.Name(), `corrupted`)
+	procutil.SelfSIGHUP()
+	time.Sleep(*rulesCheckInterval / 2)
+	groupsLen = lenLocked(m)
+	if groupsLen != 1 { // should remain unchanged
+		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
+	}
+
+	cancel()
+	<-syncCh
+}
+
+func writeToFile(t *testing.T, file, b string) {
+	t.Helper()
+	err := os.WriteFile(file, []byte(b), 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+}