docs: add HA options

.github/workflows/build.yml

@@ -1,54 +0,0 @@
-name: build
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
-      - '**/Makefile'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
-      - '**/Makefile'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
-      - name: Run crossbuild
-        run: make crossbuild

.github/workflows/codeql-analysis-go.yml

@@ -1,62 +0,0 @@
-name: 'CodeQL Go'
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: go
-
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: 'language:go'

.github/workflows/codeql-analysis-js.yml

@@ -1,27 +1,22 @@
-name: 'CodeQL JS/TS'
+name: "CodeQL - JS"
 
 on:
   push:
-    branches:
-      - cluster
-      - master
+    branches: [master, cluster]
     paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
+      - "**.js"
   pull_request:
-    branches:
-      - cluster
-      - master
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
     paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
-
+  
 jobs:
   analyze:
     name: Analyze
@@ -31,6 +26,11 @@ jobs:
       contents: read
       security-events: write
 
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["javascript"]
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -38,9 +38,9 @@ jobs:
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with:
-          languages: javascript-typescript
+          languages: ${{ matrix.language }}
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v3
         with:
-          category: 'language:js/ts'
+          category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,103 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["go"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+        if: ${{ matrix.language == 'go' }}
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
+        if: ${{ matrix.language == 'go' }}
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/main.yml

@@ -1,25 +1,29 @@
 name: main
-
 on:
   push:
     branches:
-      - cluster
       - master
-    paths:
-      - '**.go'
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
   pull_request:
     branches:
-      - cluster
       - master
-    paths:
-      - '**.go'
-
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
 permissions:
   contents: read
 
 concurrency:
-  cancel-in-progress: true
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   lint:
@@ -33,16 +37,16 @@ jobs:
         id: go
         uses: actions/setup-go@v5
         with:
-          cache: false
           go-version: stable
+          cache: false
 
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
-            ~/go/bin
             ~/go/pkg/mod
+            ~/go/bin
           key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-check-all-
 
@@ -51,18 +55,10 @@ jobs:
           make check-all
           git diff --exit-code
 
-  test:
-    name: test
+  build:
     needs: lint
+    name: build
     runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        scenario:
-          - 'test-full'
-          - 'test-full-386'
-          - 'test-pure'
-
     steps:
       - name: Code checkout
         uses: actions/checkout@v4
@@ -71,20 +67,51 @@ jobs:
         id: go
         uses: actions/setup-go@v5
         with:
-          cache: false
           go-version: stable
+          cache: false
 
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
-            ~/go/bin
             ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
+
+      - name: Build
+        run: make crossbuild
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        scenario: ["test-full", "test-pure", "test-full-386"]
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
           key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
 
-      - name: Run tests
+      - name: run tests
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage

.github/workflows/sync-docs.yml

@@ -25,7 +25,7 @@ jobs:
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: docs
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v6
+        uses: crazy-max/ghaction-import-gpg@v5
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}

.gitignore

@@ -7,7 +7,6 @@
 .vscode
 *.test
 *.swp
-/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
@@ -23,6 +22,4 @@ Gemfile.lock
 /_site
 _site
 *.tmp
-/docs/.jekyll-metadata
-coverage.txt
-cspell.json
+/docs/.jekyll-metadata

CODE_OF_CONDUCT_RU.md

@@ -0,0 +1,120 @@
+
+# Кодекс Поведения участника
+
+## Наши обязательства
+
+Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
+свободным от притеснений для всех, независимо от возраста, телосложения,
+видимых или невидимых ограничений способности, этнической принадлежности,
+половых признаков, гендерной идентичности и выражения, уровня опыта,
+образования, социо-экономического статуса, национальности, внешности,
+расы, религии, или сексуальной идентичности и ориентации.
+
+Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
+дружелюбное, многообразное, инклюзивное и здоровое сообщество.
+
+## Наши стандарты
+
+Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
+
+* Проявление доброты и эмпатии к другим участникам проекта
+* Уважение к чужой точке зрения и опыту
+* Конструктивная критика и принятие конструктивной критики
+* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
+  и извлечение уроков из опыта
+* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
+
+Примеры неприемлемого поведения участников включают в себя:
+
+* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
+* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
+* Публичное или приватное домогательство
+* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
+* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
+
+## Обязанности
+
+Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
+поведения и будут предпринимать соответствующие и честные меры по исправлению положения
+в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
+
+Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
+комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
+с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
+
+## Область применения
+
+Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
+а также когда человек официально представляет сообщество в публичных местах.
+Примеры представления проекта или сообщества включают использование официальной электронной почты,
+публикации в официальном аккаунте в социальных сетях,
+или упоминания как представителя в онлайн или оффлайн мероприятии.
+
+## Приведение в исполнение
+
+О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
+поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
+Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
+
+Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
+неприкосновенность автора сообщения.
+
+## Руководство по исполнению
+
+Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
+чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
+
+### 1. Исправление
+
+**Общественное влияние**: Использование недопустимой лексики или другое поведение,
+считающиеся непрофессиональным или нежелательным в сообществе.
+
+**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
+объясняющее суть нарушения и почему такое поведение
+было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
+
+### 2. Предупреждение
+
+**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
+
+**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
+На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
+включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
+в публичных пространствах, а так же во внешних каналах,
+таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
+
+### 3. Временный бан
+
+**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
+включая продолжительное неуместное поведение.
+
+**Последствия**: Временный запрет (бан) на любое взаимодействие
+или публичное общение с сообществом на определенный период времени.
+На этот период не допускается публичное или личное взаимодействие с людьми,
+вовлеченными в инцидент, включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса.
+Нарушение этих правил влечет за собой вечный бан.
+
+### 4. Вечный бан
+
+**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
+включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
+или проявление агрессии либо пренебрежительного отношения к категориям лиц.
+
+**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
+
+## Атрибуция
+
+Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
+версии 2.0, доступной по адресу
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
+
+Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

Makefile

@@ -17,7 +17,6 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
@@ -180,8 +179,6 @@ vet:
 
 check-all: fmt vet golangci-lint govulncheck
 
-clean-checkers: remove-golangci-lint remove-govulncheck
-
 test:
 	go test ./lib/... ./app/...
 
@@ -234,10 +231,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
-
-remove-golangci-lint:
-	rm -rf `which golangci-lint`
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
 
 govulncheck: install-govulncheck
 	govulncheck ./...
@@ -245,11 +239,37 @@ govulncheck: install-govulncheck
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 
-remove-govulncheck:
-	rm -rf `which govulncheck`
-
 install-wwhrd:
 	which wwhrd || go install github.com/frapposelli/wwhrd@latest
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
+
+copy-docs:
+# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
+# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
+	echo "---" > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "sort: ${ORDER}" >> ${DST}; \
+		echo "weight: ${ORDER}" >> ${DST}; \
+		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
+	fi
+
+	echo "title: ${TITLE}" >> ${DST}
+	@if [ ${OLD_URL} ]; then \
+		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
+	fi
+	echo "---" >> ${DST}
+	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	rm -rf docs/*.tmp
+
+# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
+# For ORDER=0 it adds no order tag/weight tag.
+# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
+# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
+# Images starting with <img src="docs/ are replaced with <img src="
+# Cluster docs are supposed to be ordered as 2nd.
+# The rest of docs is ordered manually.
+docs-sync:
+	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md OLD_URL='/Cluster-VictoriaMetrics.html' ORDER=2 TITLE='Cluster version' $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to <security@victoriametrics.com>
+Please report any security issues to security@victoriametrics.com

app/victoria-logs/Makefile

@@ -81,9 +81,6 @@ victoria-logs-linux-ppc64le:
 victoria-logs-linux-s390x:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-victoria-logs-linux-loong64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 victoria-logs-linux-386:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/victoria-logs/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image AS certs
+FROM $certs_image as certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vlinsert/elasticsearch/elasticsearch.go

@@ -57,12 +57,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
-		// Return fake response for Logstash APIs requests.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
 	switch path {
 	case "/":
 		switch r.Method {
@@ -103,10 +97,12 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		lmp := cp.NewLogMessageProcessor()
+		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+		processLogMessage := cp.GetProcessLogMessageFunc(lr)
 		isGzip := r.Header.Get("Content-Encoding") == "gzip"
-		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, lmp)
-		lmp.MustClose()
+		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
+		vlstorage.MustAddRows(lr)
+		logstorage.PutLogRows(lr)
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
 			return true
@@ -135,7 +131,9 @@ var (
 	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp insertutils.LogMessageProcessor) (int, error) {
+func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
+	processLogMessage func(timestamp int64, fields []logstorage.Field),
+) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
 	if isGzip {
@@ -160,7 +158,7 @@ func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp i
 	n := 0
 	nCheckpoint := 0
 	for {
-		ok, err := readBulkLine(sc, timeField, msgField, lmp)
+		ok, err := readBulkLine(sc, timeField, msgField, processLogMessage)
 		wcr.DecConcurrency()
 		if err != nil || !ok {
 			rowsIngestedTotal.Add(n - nCheckpoint)
@@ -176,7 +174,9 @@ func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp i
 
 var lineBufferPool bytesutil.ByteBufferPool
 
-func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
+func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
+	processLogMessage func(timestamp int64, fields []logstorage.Field),
+) (bool, error) {
 	var line []byte
 
 	// Read the command, must be "create" or "index"
@@ -222,7 +222,7 @@ func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils
 		ts = time.Now().UnixNano()
 	}
 	logstorage.RenameField(p.Fields, msgField, "_msg")
-	lmp.AddRow(ts, p.Fields)
+	processLogMessage(ts, p.Fields)
 	logstorage.PutJSONParser(p)
 
 	return true, nil

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -4,18 +4,22 @@ import (
 	"bytes"
 	"compress/gzip"
 	"fmt"
+	"reflect"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-func TestReadBulkRequest_Failure(t *testing.T) {
+func TestReadBulkRequestFailure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
 
-		tlp := &insertutils.TestLogMessageProcessor{}
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
+		}
+
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, "_time", "_msg", tlp)
+		rows, err := readBulkRequest(r, false, "_time", "_msg", processLogMessage)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -32,38 +36,52 @@ func TestReadBulkRequest_Failure(t *testing.T) {
 foobar`)
 }
 
-func TestReadBulkRequest_Success(t *testing.T) {
+func TestReadBulkRequestSuccess(t *testing.T) {
 	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		tlp := &insertutils.TestLogMessageProcessor{}
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+			result += string(logstorage.MarshalFieldsToJSON(nil, fields)) + "\n"
+		}
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, timeField, msgField, tlp)
+		rows, err := readBulkRequest(r, false, timeField, msgField, processLogMessage)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if rows != rowsExpected {
 			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
 
 		// Read the request with compression
-		tlp = &insertutils.TestLogMessageProcessor{}
+		timestamps = nil
+		result = ""
 		compressedData := compressData(data)
 		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest(r, true, timeField, msgField, tlp)
+		rows, err = readBulkRequest(r, true, timeField, msgField, processLogMessage)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if rows != rowsExpected {
 			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("verification failure after compression: %s", err)
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
 	}
 
@@ -86,7 +104,8 @@ func TestReadBulkRequest_Success(t *testing.T) {
 	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
 	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}`
+{"_msg":"xyz","@timestamp":"","x":"y"}
+`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
 func BenchmarkReadBulkRequest(b *testing.B) {
@@ -33,7 +33,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 
 	timeField := "@timestamp"
 	msgField := "message"
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
+	processLogMessage := func(_ int64, _ []logstorage.Field) {}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))
@@ -41,7 +41,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest(r, isGzip, timeField, msgField, blp)
+			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutils/common_params.go

@@ -2,8 +2,6 @@ package insertutils
 
 import (
 	"net/http"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
@@ -12,7 +10,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 // CommonParams contains common HTTP parameters used by log ingestion APIs.
@@ -91,109 +88,29 @@ func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
 	return cp
 }
 
-// LogMessageProcessor is an interface for log message processors.
-type LogMessageProcessor interface {
-	// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
-	//
-	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
-	AddRow(timestamp int64, fields []logstorage.Field)
-
-	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
-	MustClose()
-}
-
-type logMessageProcessor struct {
-	mu            sync.Mutex
-	wg            sync.WaitGroup
-	stopCh        chan struct{}
-	lastFlushTime time.Time
-
-	cp *CommonParams
-	lr *logstorage.LogRows
-}
-
-func (lmp *logMessageProcessor) initPeriodicFlush() {
-	lmp.lastFlushTime = time.Now()
-
-	lmp.wg.Add(1)
-	go func() {
-		defer lmp.wg.Done()
-
-		d := timeutil.AddJitterToDuration(time.Second)
-		ticker := time.NewTicker(d)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-lmp.stopCh:
-				return
-			case <-ticker.C:
-				lmp.mu.Lock()
-				if time.Since(lmp.lastFlushTime) >= d {
-					lmp.flushLocked()
-				}
-				lmp.mu.Unlock()
-			}
+// GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
+func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
+	return func(timestamp int64, fields []logstorage.Field) {
+		if len(fields) > *MaxFieldsPerLine {
+			rf := logstorage.RowFormatter(fields)
+			logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+			rowsDroppedTotalTooManyFields.Inc()
+			return
 		}
-	}()
-}
 
-// AddRow adds new log message to lmp with the given timestamp and fields.
-func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	if len(fields) > *MaxFieldsPerLine {
-		rf := logstorage.RowFormatter(fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
+		lr.MustAdd(cp.TenantID, timestamp, fields)
+		if cp.Debug {
+			s := lr.GetRowString(0)
+			lr.ResetKeepSettings()
+			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
+			rowsDroppedTotalDebug.Inc()
+			return
+		}
+		if lr.NeedFlush() {
+			vlstorage.MustAddRows(lr)
+			lr.ResetKeepSettings()
+		}
 	}
-
-	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// flushLocked must be called under locked lmp.mu.
-func (lmp *logMessageProcessor) flushLocked() {
-	lmp.lastFlushTime = time.Now()
-	vlstorage.MustAddRows(lmp.lr)
-	lmp.lr.ResetKeepSettings()
-}
-
-// MustClose flushes the remaining data to the underlying storage and closes lmp.
-func (lmp *logMessageProcessor) MustClose() {
-	close(lmp.stopCh)
-	lmp.wg.Wait()
-
-	lmp.flushLocked()
-	logstorage.PutLogRows(lmp.lr)
-	lmp.lr = nil
-}
-
-// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
-//
-// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
-func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	lmp := &logMessageProcessor{
-		cp: cp,
-		lr: lr,
-
-		stopCh: make(chan struct{}),
-	}
-	lmp.initPeriodicFlush()
-
-	return lmp
 }
 
 var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)

app/vlinsert/insertutils/testutils.go

@@ -1,53 +0,0 @@
-package insertutils
-
-import (
-	"fmt"
-	"reflect"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// TestLogMessageProcessor implements LogMessageProcessor for testing.
-type TestLogMessageProcessor struct {
-	timestamps []int64
-	rows       []string
-}
-
-// AddRow adds row with the given timestamp and fields to tlp
-func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
-	tlp.timestamps = append(tlp.timestamps, timestamp)
-	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
-}
-
-// MustClose closes tlp.
-func (tlp *TestLogMessageProcessor) MustClose() {
-}
-
-// Verify verifies the number of rows, timestamps and results after AddRow calls.
-func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error {
-	result := strings.Join(tlp.rows, "\n")
-	if len(tlp.rows) != rowsExpected {
-		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), rowsExpected, result, resultExpected)
-	}
-
-	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
-		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
-	}
-	if result != resultExpected {
-		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-	}
-
-	return nil
-}
-
-// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
-type BenchmarkLogMessageProcessor struct{}
-
-// AddRow implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field) {
-}
-
-// MustClose implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) MustClose() {
-}

app/vlinsert/insertutils/timestamp.go

@@ -7,24 +7,24 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+// ExtractTimestampISO8601FromFields extracts ISO8601 timestamp in nanoseconds from the field with the name timeField at fields.
 //
 // The value for the timeField is set to empty string after returning from the function,
 // so it could be ignored during data ingestion.
 //
 // The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+func ExtractTimestampISO8601FromFields(timeField string, fields []logstorage.Field) (int64, error) {
 	for i := range fields {
 		f := &fields[i]
 		if f.Name != timeField {
 			continue
 		}
-		if f.Value == "" || f.Value == "0" {
-			return time.Now().UnixNano(), nil
-		}
-		nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
+		nsecs, ok := logstorage.TryParseTimestampISO8601(f.Value)
 		if !ok {
-			return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
+			if f.Value == "0" || f.Value == "" {
+				return time.Now().UnixNano(), nil
+			}
+			return time.Now().UnixNano(), fmt.Errorf("cannot unmarshal iso8601 timestamp from %s=%q", timeField, f.Value)
 		}
 		f.Value = ""
 		return nsecs, nil

app/vlinsert/insertutils/timestamp_test.go

@@ -1,75 +0,0 @@
-package insertutils
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs != nsecsExpected {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
-		}
-
-		for _, f := range fields {
-			if f.Name == timeField {
-				if f.Value != "" {
-					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
-				}
-			}
-		}
-	}
-
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20Z"},
-	}, 1718753840000000000)
-
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
-	}, 1718725040000000000)
-
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
-	}, 1718773640123000000)
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
-		{Name: "foo", Value: "bar"},
-	}, 1718773640123456789)
-}
-
-func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		fields := []logstorage.Field{
-			{Name: "time", Value: s},
-		}
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if nsecs != 0 {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
-		}
-	}
-
-	f("foobar")
-
-	// no Z at the end
-	f("2024-06-18T23:37:20")
-
-	// incomplete time
-	f("2024-06-18")
-	f("2024-06-18T23:37")
-}

app/vlinsert/jsonline/jsonline.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"time"
 
@@ -40,6 +39,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
 
 	reader := r.Body
 	if r.Header.Get("Content-Encoding") == "gzip" {
@@ -52,22 +53,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		reader = zr
 	}
 
-	lmp := cp.NewLogMessageProcessor()
-	err = processStreamInternal(reader, cp.TimeField, cp.MsgField, lmp)
-	lmp.MustClose()
-
-	if err != nil {
-		logger.Errorf("jsonline: %s", err)
-	} else {
-		// update requestDuration only for successfully parsed requests.
-		// There is no need in updating requestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		requestDuration.UpdateDuration(startTime)
-	}
-}
-
-func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertutils.LogMessageProcessor) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
+	wcr := writeconcurrencylimiter.GetReader(reader)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
 	lb := lineBufferPool.Get()
@@ -79,21 +65,30 @@ func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertut
 
 	n := 0
 	for {
-		ok, err := readLine(sc, timeField, msgField, lmp)
+		ok, err := readLine(sc, cp.TimeField, cp.MsgField, processLogMessage)
 		wcr.DecConcurrency()
 		if err != nil {
 			errorsTotal.Inc()
-			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
+			logger.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
+			break
 		}
 		if !ok {
-			return nil
+			break
 		}
 		n++
 		rowsIngestedTotal.Inc()
 	}
+
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+
+	// update requestDuration only for successfully parsed requests.
+	// There is no need in updating requestDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	requestDuration.UpdateDuration(startTime)
 }
 
-func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
+func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
 	var line []byte
 	for len(line) == 0 {
 		if !sc.Scan() {
@@ -112,12 +107,12 @@ func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.Log
 	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
-	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
+	ts, err := insertutils.ExtractTimestampISO8601FromFields(timeField, p.Fields)
 	if err != nil {
 		return false, fmt.Errorf("cannot get timestamp: %w", err)
 	}
 	logstorage.RenameField(p.Fields, msgField, "_msg")
-	lmp.AddRow(ts, p.Fields)
+	processLogMessage(ts, p.Fields)
 	logstorage.PutJSONParser(p)
 
 	return true, nil

app/vlinsert/jsonline/jsonline_test.go

@@ -1,24 +1,47 @@
 package jsonline
 
 import (
+	"bufio"
 	"bytes"
+	"reflect"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-func TestProcessStreamInternal_Success(t *testing.T) {
+func TestReadLine_Success(t *testing.T) {
 	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, timeField, msgField, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+			result += string(logstorage.MarshalFieldsToJSON(nil, fields)) + "\n"
 		}
 
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
+		r := bytes.NewBufferString(data)
+		sc := bufio.NewScanner(r)
+		rows := 0
+		for {
+			ok, err := readLine(sc, timeField, msgField, processLogMessage)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if !ok {
+				break
+			}
+			rows++
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
 	}
 
@@ -32,24 +55,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
 	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}`
+{"_msg":"xyz","@timestamp":"","x":"y"}
+`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "time", "", tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid json
-	f("foobar")
-
-	// invalid timestamp field
-	f(`{"time":"foobar"}`)
-}

app/vlinsert/loki/loki.go

@@ -46,7 +46,7 @@ func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
 	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
 		org := r.Header.Get("X-Scope-OrgID")
 		if org != "" {
-			tenantID, err := logstorage.ParseTenantID(org)
+			tenantID, err := logstorage.GetTenantIDFromString(org)
 			if err != nil {
 				return nil, err
 			}

app/vlinsert/loki/loki_json.go

@@ -8,7 +8,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -52,9 +51,11 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	lmp := cp.NewLogMessageProcessor()
-	n, err := parseJSONRequest(data, lmp)
-	lmp.MustClose()
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+	n, err := parseJSONRequest(data, processLogMessage)
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
 		return
@@ -74,7 +75,7 @@ var (
 	requestJSONDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
-func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
+func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
 	v, err := p.ParseBytes(data)
@@ -167,7 +168,7 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, er
 				Name:  "_msg",
 				Value: bytesutil.ToUnsafeString(msg),
 			})
-			lmp.AddRow(ts, fields)
+			processLogMessage(ts, fields)
 		}
 		rowsIngested += len(lines)
 	}

app/vlinsert/loki/loki_json_test.go

@@ -1,17 +1,19 @@
 package loki
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-func TestParseJSONRequest_Failure(t *testing.T) {
+func TestParseJSONRequestFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		n, err := parseJSONRequest([]byte(s), tlp)
+		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
+			t.Fatalf("unexpected call to parseJSONRequest callback!")
+		})
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -54,30 +56,39 @@ func TestParseJSONRequest_Failure(t *testing.T) {
 	f(`{"streams":[{"values":[["123",1234]]}]}`)
 }
 
-func TestParseJSONRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
+func TestParseJSONRequestSuccess(t *testing.T) {
+	f := func(s string, resultExpected string) {
 		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-
-		n, err := parseJSONRequest([]byte(s), tlp)
+		var lines []string
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+			var a []string
+			for _, f := range fields {
+				a = append(a, f.String())
+			}
+			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
+			lines = append(lines, line)
+		})
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
+		if n != len(lines) {
+			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
+		}
+		result := strings.Join(lines, "\n")
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
 	}
 
 	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
+	f(`{"streams":[]}`, ``)
+	f(`{"streams":[{"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
 
 	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
 
 	// Non-empty stream labels
 	f(`{"streams":[{"stream":{
@@ -87,9 +98,9 @@ func TestParseJSONRequest_Success(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
+]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
+_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
+_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
 
 	// Multiple streams
 	f(`{
@@ -113,7 +124,7 @@ func TestParseJSONRequest_Success(t *testing.T) {
 			]
 		}
 	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
+}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
+_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
+_time:1877836900005000002 "x":"y" "_msg":"yx"`)
 }

app/vlinsert/loki/loki_json_timing_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
 func BenchmarkParseJSONRequest(b *testing.B) {
@@ -22,13 +22,12 @@ func BenchmarkParseJSONRequest(b *testing.B) {
 }
 
 func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseJSONRequest(data, blp)
+			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/loki/loki_protobuf.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -44,9 +43,11 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	lmp := cp.NewLogMessageProcessor()
-	n, err := parseProtobufRequest(data, lmp)
-	lmp.MustClose()
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+	n, err := parseProtobufRequest(data, processLogMessage)
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
 		return
@@ -66,7 +67,7 @@ var (
 	requestProtobufDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
-func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
+func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
 	bb := bytesBufPool.Get()
 	defer bytesBufPool.Put(bb)
 
@@ -79,14 +80,12 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err = req.UnmarshalProtobuf(bb.B)
+	err = req.Unmarshal(bb.B)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
-	fields := getFields()
-	defer putFields(fields)
-
+	var commonFields []logstorage.Field
 	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
@@ -94,60 +93,30 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
 		// Labels are same for all entries in the stream.
-		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
+		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
 		if err != nil {
 			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
-		commonFieldsLen := len(fields.fields)
+		fields := commonFields
 
 		entries := stream.Entries
 		for j := range entries {
-			e := &entries[j]
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			for _, lp := range e.StructuredMetadata {
-				fields.fields = append(fields.fields, logstorage.Field{
-					Name:  lp.Name,
-					Value: lp.Value,
-				})
-			}
-
-			fields.fields = append(fields.fields, logstorage.Field{
+			entry := &entries[j]
+			fields = append(fields[:len(commonFields)], logstorage.Field{
 				Name:  "_msg",
-				Value: e.Line,
+				Value: entry.Line,
 			})
-
-			ts := e.Timestamp.UnixNano()
+			ts := entry.Timestamp.UnixNano()
 			if ts == 0 {
 				ts = currentTimestamp
 			}
-
-			lmp.AddRow(ts, fields.fields)
+			processLogMessage(ts, fields)
 		}
 		rowsIngested += len(stream.Entries)
 	}
 	return rowsIngested, nil
 }
 
-func getFields() *fields {
-	v := fieldsPool.Get()
-	if v == nil {
-		return &fields{}
-	}
-	return v.(*fields)
-}
-
-func putFields(f *fields) {
-	f.fields = f.fields[:0]
-	fieldsPool.Put(f)
-}
-
-var fieldsPool sync.Pool
-
-type fields struct {
-	fields []logstorage.Field
-}
-
 // parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
 //
 // See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
@@ -213,6 +182,6 @@ func getPushRequest() *PushRequest {
 }
 
 func putPushRequest(req *PushRequest) {
-	req.reset()
+	req.Reset()
 	pushReqsPool.Put(req)
 }

app/vlinsert/loki/loki_protobuf_test.go

@@ -6,80 +6,83 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/golang/snappy"
 )
 
-type testLogMessageProcessor struct {
-	pr PushRequest
-}
-
-func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
-	msg := ""
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			msg = f.Value
-		}
-	}
-	var a []string
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			continue
-		}
-		item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-		a = append(a, item)
-	}
-	labels := "{" + strings.Join(a, ", ") + "}"
-	tlp.pr.Streams = append(tlp.pr.Streams, Stream{
-		Labels: labels,
-		Entries: []Entry{
-			{
-				Timestamp: time.Unix(0, timestamp),
-				Line:      strings.Clone(msg),
-			},
-		},
-	})
-}
-
-func (tlp *testLogMessageProcessor) MustClose() {
-}
-
-func TestParseProtobufRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
+func TestParseProtobufRequestSuccess(t *testing.T) {
+	f := func(s string, resultExpected string) {
 		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		n, err := parseJSONRequest([]byte(s), tlp)
+		var pr PushRequest
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+			msg := ""
+			for _, f := range fields {
+				if f.Name == "_msg" {
+					msg = f.Value
+				}
+			}
+			var a []string
+			for _, f := range fields {
+				if f.Name == "_msg" {
+					continue
+				}
+				item := fmt.Sprintf("%s=%q", f.Name, f.Value)
+				a = append(a, item)
+			}
+			labels := "{" + strings.Join(a, ", ") + "}"
+			pr.Streams = append(pr.Streams, Stream{
+				Labels: labels,
+				Entries: []Entry{
+					{
+						Timestamp: time.Unix(0, timestamp),
+						Line:      msg,
+					},
+				},
+			})
+		})
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if n != len(tlp.pr.Streams) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
+		if n != len(pr.Streams) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(pr.Streams), n)
 		}
 
-		data := tlp.pr.MarshalProtobuf(nil)
+		data, err := pr.Marshal()
+		if err != nil {
+			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
+		}
 		encodedData := snappy.Encode(nil, data)
 
-		tlp2 := &insertutils.TestLogMessageProcessor{}
-		n, err = parseProtobufRequest(encodedData, tlp2)
+		var lines []string
+		n, err = parseProtobufRequest(encodedData, func(timestamp int64, fields []logstorage.Field) {
+			var a []string
+			for _, f := range fields {
+				a = append(a, f.String())
+			}
+			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
+			lines = append(lines, line)
+		})
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp2.Verify(n, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
+		if n != len(lines) {
+			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
+		}
+		result := strings.Join(lines, "\n")
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
 	}
 
 	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
+	f(`{"streams":[]}`, ``)
+	f(`{"streams":[{"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
 
 	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
 
 	// Non-empty stream labels
 	f(`{"streams":[{"stream":{
@@ -89,9 +92,9 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
+]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
+_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
+_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
 
 	// Multiple streams
 	f(`{
@@ -115,12 +118,12 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 			]
 		}
 	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
+}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
+_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
+_time:1877836900005000002 "x":"y" "_msg":"yx"`)
 }
 
-func TestParsePromLabels_Success(t *testing.T) {
+func TestParsePromLabelsSuccess(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		fields, err := parsePromLabels(nil, s)
@@ -144,7 +147,7 @@ func TestParsePromLabels_Success(t *testing.T) {
 	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
 }
 
-func TestParsePromLabels_Failure(t *testing.T) {
+func TestParsePromLabelsFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		fields, err := parsePromLabels(nil, s)

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -8,8 +8,7 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
 func BenchmarkParseProtobufRequest(b *testing.B) {
@@ -25,13 +24,12 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
 }
 
 func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseProtobufRequest(body, blp)
+			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
@@ -39,47 +37,29 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	})
 }
 
-func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
-	var b []byte
-	var entries []Entry
-	streams := make([]Stream, streamsCount)
-	for i := range streams {
-		b = b[:0]
-		b = append(b, '{')
-		for j := 0; j < labelsCount; j++ {
-			b = append(b, "label_"...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, `="value_`...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, '"')
-			if j < labelsCount-1 {
-				b = append(b, ',')
+func getProtobufBody(streams, rows, labels int) []byte {
+	var pr PushRequest
+
+	for i := 0; i < streams; i++ {
+		var st Stream
+
+		st.Labels = `{`
+		for j := 0; j < labels; j++ {
+			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
+			if j < labels-1 {
+				st.Labels += `,`
 			}
 		}
-		b = append(b, '}')
-		labels := string(b)
+		st.Labels += `}`
 
-		var rowsBuf []byte
-		entriesLen := len(entries)
-		for j := 0; j < rowsCount; j++ {
-			rowsBufLen := len(rowsBuf)
-			rowsBuf = append(rowsBuf, "value_"...)
-			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
-			entries = append(entries, Entry{
-				Timestamp: time.Now(),
-				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
-			})
+		for j := 0; j < rows; j++ {
+			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
 		}
 
-		st := &streams[i]
-		st.Labels = labels
-		st.Entries = entries[entriesLen:]
-	}
-	pr := PushRequest{
-		Streams: streams,
+		pr.Streams = append(pr.Streams, st)
 	}
 
-	body := pr.MarshalProtobuf(nil)
+	body, _ := pr.Marshal()
 	encodedBody := snappy.Encode(nil, body)
 
 	return encodedBody

app/vlinsert/loki/pb.go

@@ -1,302 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: push_request.proto
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package loki
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// PushRequest represents Loki PushRequest
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
-type PushRequest struct {
-	Streams []Stream
-
-	entriesBuf   []Entry
-	labelPairBuf []LabelPair
-}
-
-func (pr *PushRequest) reset() {
-	pr.Streams = pr.Streams[:0]
-
-	pr.entriesBuf = pr.entriesBuf[:0]
-	pr.labelPairBuf = pr.labelPairBuf[:0]
-}
-
-// UnmarshalProtobuf unmarshals pr from protobuf message at src.
-//
-// pr remains valid until src is modified.
-func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
-	pr.reset()
-	var err error
-	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
-	return err
-}
-
-// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
-func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	pr.marshalProtobuf(m.MessageMarshaler())
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, s := range pr.Streams {
-		s.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message PushRequest {
-	//   repeated Stream streams = 1;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
-			}
-			pr.Streams = append(pr.Streams, Stream{})
-			s := &pr.Streams[len(pr.Streams)-1]
-			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
-			}
-		}
-	}
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Stream represents Loki stream.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
-type Stream struct {
-	Labels  string
-	Entries []Entry
-}
-
-func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, s.Labels)
-	for _, e := range s.Entries {
-		e.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message Stream {
-	//   string labels = 1;
-	//   repeated Entry entries = 2;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	entriesBufLen := len(entriesBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			labels, ok := fc.String()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
-			}
-			s.Labels = labels
-		case 2:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
-			}
-			entriesBuf = append(entriesBuf, Entry{})
-			e := &entriesBuf[len(entriesBuf)-1]
-			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
-			}
-		}
-	}
-	s.Entries = entriesBuf[entriesBufLen:]
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Entry represents Loki entry.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
-type Entry struct {
-	Timestamp          time.Time
-	Line               string
-	StructuredMetadata []LabelPair
-}
-
-func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	marshalTime(mm, 1, e.Timestamp)
-	mm.AppendString(2, e.Line)
-	for _, lp := range e.StructuredMetadata {
-		lp.marshalProtobuf(mm.AppendMessage(3))
-	}
-}
-
-func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
-	// message Entry {
-	//   Timestamp timestamp = 1;
-	//   string line = 2;
-	//   repeated LabelPair structuredMetadata = 3;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	labelPairBufLen := len(labelPairBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
-			}
-			timestamp, err := unmarshalTime(data)
-			if err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
-			}
-			e.Timestamp = timestamp
-		case 2:
-			line, ok := fc.String()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Line")
-			}
-			e.Line = line
-		case 3:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
-			}
-			labelPairBuf = append(labelPairBuf, LabelPair{})
-			lp := &labelPairBuf[len(labelPairBuf)-1]
-			if err := lp.unmarshalProtobuf(data); err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
-			}
-		}
-	}
-	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
-	return labelPairBuf, nil
-}
-
-// LabelPair represents Loki label pair.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
-type LabelPair struct {
-	Name  string
-	Value string
-}
-
-func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, lp.Name)
-	mm.AppendString(2, lp.Value)
-}
-
-func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
-	// message LabelPair {
-	//   string name = 1;
-	//   string value = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			name, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot read name")
-			}
-			lp.Name = name
-		case 2:
-			value, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot unmarshal value")
-			}
-			lp.Value = value
-		}
-	}
-	return nil
-}
-
-func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
-	nsecs := timestamp.UnixNano()
-	ts := Timestamp{
-		Seconds: nsecs / 1e9,
-		Nanos:   int32(nsecs % 1e9),
-	}
-	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
-}
-
-func unmarshalTime(src []byte) (time.Time, error) {
-	var ts Timestamp
-	if err := ts.unmarshalProtobuf(src); err != nil {
-		return time.Time{}, err
-	}
-	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	return timestamp, nil
-}
-
-// Timestamp is protobuf well-known timestamp type.
-type Timestamp struct {
-	Seconds int64
-	Nanos   int32
-}
-
-func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendInt64(1, ts.Seconds)
-	mm.AppendInt32(2, ts.Nanos)
-}
-
-func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
-	// message Timestamp {
-	//   int64 seconds = 1;
-	//   int32 nanos = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			seconds, ok := fc.Int64()
-			if !ok {
-				return fmt.Errorf("cannot read Seconds")
-			}
-			ts.Seconds = seconds
-		case 2:
-			nanos, ok := fc.Int32()
-			if !ok {
-				return fmt.Errorf("cannot read Nanos")
-			}
-			ts.Nanos = nanos
-		}
-	}
-	return nil
-}

app/vlinsert/loki/push_request.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package logproto;
+
+import "gogoproto/gogo.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
+
+message PushRequest {
+  repeated StreamAdapter streams = 1 [
+    (gogoproto.jsontag) = "streams",
+    (gogoproto.customtype) = "Stream"
+  ];
+}
+
+message StreamAdapter {
+  string labels = 1 [(gogoproto.jsontag) = "labels"];
+  repeated EntryAdapter entries = 2 [
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "entries"
+  ];
+  // hash contains the original hash of the stream.
+  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
+}
+
+message EntryAdapter {
+  google.protobuf.Timestamp timestamp = 1 [
+    (gogoproto.stdtime) = true,
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "ts"
+  ];
+  string line = 2 [(gogoproto.jsontag) = "line"];
+}

app/vlinsert/loki/timestamp.go

@@ -0,0 +1,110 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+)
+
+const (
+	// Seconds field of the earliest valid Timestamp.
+	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	minValidSeconds = -62135596800
+	// Seconds field just after the latest valid Timestamp.
+	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	maxValidSeconds = 253402300800
+)
+
+// validateTimestamp determines whether a Timestamp is valid.
+// A valid timestamp represents a time in the range
+// [0001-01-01, 10000-01-01) and has a Nanos field
+// in the range [0, 1e9).
+//
+// If the Timestamp is valid, validateTimestamp returns nil.
+// Otherwise, it returns an error that describes
+// the problem.
+//
+// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
+func validateTimestamp(ts *types.Timestamp) error {
+	if ts == nil {
+		return errors.New("timestamp: nil Timestamp")
+	}
+	if ts.Seconds < minValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
+	}
+	if ts.Seconds >= maxValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
+	}
+	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
+		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
+	}
+	return nil
+}
+
+// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
+// but avoids the escape incurred by using fmt.Sprintf, eliminating
+// unnecessary heap allocations.
+func formatTimestamp(ts *types.Timestamp) string {
+	if ts == nil {
+		return "nil"
+	}
+
+	seconds := strconv.FormatInt(ts.Seconds, 10)
+	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
+	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
+}
+
+func sizeOfStdTime(t time.Time) int {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0
+	}
+	return ts.Size()
+}
+
+func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0, err
+	}
+	return ts.MarshalTo(data)
+}
+
+func stdTimeUnmarshal(t *time.Time, data []byte) error {
+	ts := &types.Timestamp{}
+	if err := ts.Unmarshal(data); err != nil {
+		return err
+	}
+	tt, err := timestampFromProto(ts)
+	if err != nil {
+		return err
+	}
+	*t = tt
+	return nil
+}
+
+func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
+	// Don't return the zero value on error, because corresponds to a valid
+	// timestamp. Instead return whatever time.Unix gives us.
+	var t time.Time
+	if ts == nil {
+		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
+	} else {
+		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	}
+	return t, validateTimestamp(ts)
+}
+
+func timestampProto(t time.Time) (types.Timestamp, error) {
+	ts := types.Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.Nanosecond()),
+	}
+	return ts, validateTimestamp(&ts)
+}

app/vlinsert/loki/types.go

@@ -0,0 +1,481 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"fmt"
+	"io"
+	"time"
+)
+
+// Stream contains a unique labels set as a string and a set of entries for it.
+// We are not using the proto generated version but this custom one so that we
+// can improve serialization see benchmark.
+type Stream struct {
+	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
+	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
+	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
+}
+
+// Entry is a log entry with a timestamp.
+type Entry struct {
+	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
+	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Stream) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Hash != 0 {
+		i = encodeVarintPush(dAtA, i, m.Hash)
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Entries) > 0 {
+		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintPush(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		i -= len(m.Labels)
+		copy(dAtA[i:], m.Labels)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Entry) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Line) > 0 {
+		i -= len(m.Line)
+		copy(dAtA[i:], m.Line)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
+		i--
+		dAtA[i] = 0x12
+	}
+	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
+	if err7 != nil {
+		return 0, err7
+	}
+	i -= n7
+	i = encodeVarintPush(dAtA, i, uint64(n7))
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Stream) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Labels = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Entries = append(m.Entries, Entry{})
+			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
+			}
+			m.Hash = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Hash |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Entry) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Line = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Size returns the size of the serialized Stream.
+func (m *Stream) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Labels)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	if len(m.Entries) > 0 {
+		for _, e := range m.Entries {
+			l = e.Size()
+			n += 1 + l + sovPush(uint64(l))
+		}
+	}
+	if m.Hash != 0 {
+		n += 1 + sovPush(m.Hash)
+	}
+	return n
+}
+
+// Size returns the size of the serialized Entry
+func (m *Entry) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = sizeOfStdTime(m.Timestamp)
+	n += 1 + l + sovPush(uint64(l))
+	l = len(m.Line)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	return n
+}
+
+// Equal returns true if the two Streams are equal.
+func (m *Stream) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Stream)
+	if !ok {
+		that2, ok := that.(Stream)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if m.Labels != that1.Labels {
+		return false
+	}
+	if len(m.Entries) != len(that1.Entries) {
+		return false
+	}
+	for i := range m.Entries {
+		if !m.Entries[i].Equal(that1.Entries[i]) {
+			return false
+		}
+	}
+	return m.Hash == that1.Hash
+}
+
+// Equal returns true if the two Entries are equal.
+func (m *Entry) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Entry)
+	if !ok {
+		that2, ok := that.(Entry)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if !m.Timestamp.Equal(that1.Timestamp) {
+		return false
+	}
+	if m.Line != that1.Line {
+		return false
+	}
+	return true
+}

app/vlinsert/syslog/syslog.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -26,49 +25,34 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
+	syslogTenantID = flag.String("syslog.tenantID", "0:0", "TenantID for logs ingested via Syslog protocol. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
 		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
-	syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-	syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	listenAddrTCP = flag.String("syslog.listenAddr.tcp", "", "Optional TCP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	listenAddrUDP = flag.String("syslog.listenAddr.udp", "", "Optional UDP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
-	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-	listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+
-		"The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
+	tlsEnable = flag.Bool("syslog.tls", false, "Whether to use TLS for receiving syslog messages at -syslog.listenAddr.tcp. -syslog.tlsCertFile and -syslog.tlsKeyFile must be set "+
+		"if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	tlsCertFile = flag.String("syslog.tlsCertFile", "", "Path to file with TLS certificate for -syslog.listenAddr.tcp if -syslog.tls is set. "+
 		"Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
-		"The provided key file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	tlsKeyFile = flag.String("syslog.tlsKeyFile", "", "Path to file with TLS key for  -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"The provided key file is automatically re-read every second, so it can be dynamically updated")
 	tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+
 		"See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+
-		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 	tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+
 		"Supported values: TLS10, TLS11, TLS12, TLS13. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
-	compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-	compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-
-	useLocalTimestampTCP = flagutil.NewArrayBool("syslog.useLocalTimestamp.tcp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
-	useLocalTimestampUDP = flagutil.NewArrayBool("syslog.useLocalTimestamp.udp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
+	compressMethod = flag.String("syslog.compressMethod", "", "Compression method for syslog messages received at -syslog.listenAddr.tcp and -syslog.listenAddr.udp. "+
+		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 )
 
 // MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports
@@ -82,20 +66,32 @@ func MustInit() {
 	}
 	workersStopCh = make(chan struct{})
 
-	for argIdx, addr := range *listenAddrTCP {
-		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runTCPListener(addr, argIdx)
-			workersWG.Done()
-		}(addr, argIdx)
+	tenantID, err := logstorage.GetTenantIDFromString(*syslogTenantID)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.tenantID=%q: %s", *syslogTenantID, err)
+	}
+	globalTenantID = tenantID
+
+	switch *compressMethod {
+	case "", "none", "gzip", "deflate":
+	default:
+		logger.Fatalf("unexpected -syslog.compressLevel=%q; supported values: none, gzip, deflate", *compressMethod)
 	}
 
-	for argIdx, addr := range *listenAddrUDP {
+	if *listenAddrTCP != "" {
 		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runUDPListener(addr, argIdx)
+		go func() {
+			runTCPListener(*listenAddrTCP)
 			workersWG.Done()
-		}(addr, argIdx)
+		}()
+	}
+
+	if *listenAddrUDP != "" {
+		workersWG.Add(1)
+		go func() {
+			runUDPListener(*listenAddrUDP)
+			workersWG.Done()
+		}()
 	}
 
 	currentYear := time.Now().Year()
@@ -128,6 +124,7 @@ func MustInit() {
 }
 
 var (
+	globalTenantID    logstorage.TenantID
 	globalCurrentYear atomic.Int64
 	globalTimezone    *time.Location
 )
@@ -144,47 +141,32 @@ func MustStop() {
 	workersStopCh = nil
 }
 
-func runUDPListener(addr string, argIdx int) {
+func runUDPListener(addr string) {
 	ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr)
 	if err != nil {
 		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
 	}
 
-	tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodUDP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "udp")
-
-	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
-
 	doneCh := make(chan struct{})
 	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
+		serveUDP(ln)
 		close(doneCh)
 	}()
 
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
 	<-workersStopCh
 	if err := ln.Close(); err != nil {
 		logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err)
 	}
 	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
 }
 
-func runTCPListener(addr string, argIdx int) {
+func runTCPListener(addr string) {
 	var tlsConfig *tls.Config
-	if tlsEnable.GetOptionalArg(argIdx) {
-		certFile := tlsCertFile.GetOptionalArg(argIdx)
-		keyFile := tlsKeyFile.GetOptionalArg(argIdx)
-		tc, err := netutil.GetServerTLSConfig(certFile, keyFile, *tlsMinVersion, *tlsCipherSuites)
+	if *tlsEnable {
+		tc, err := netutil.GetServerTLSConfig(*tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites)
 		if err != nil {
 			logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s",
-				certFile, keyFile, *tlsMinVersion, *tlsCipherSuites, err)
+				*tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites, err)
 		}
 		tlsConfig = tc
 	}
@@ -193,42 +175,20 @@ func runTCPListener(addr string, argIdx int) {
 		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
 	}
 
-	tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodTCP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "tcp")
-
-	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
-
 	doneCh := make(chan struct{})
 	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
+		serveTCP(ln)
 		close(doneCh)
 	}()
 
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
 	<-workersStopCh
 	if err := ln.Close(); err != nil {
 		logger.Fatalf("syslog: cannot close TCP listener at %s: %s", addr, err)
 	}
 	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
 }
 
-func checkCompressMethod(compressMethod, addr, protocol string) {
-	switch compressMethod {
-	case "", "none", "gzip", "deflate":
-		return
-	default:
-		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
-	}
-}
-
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+func serveUDP(ln net.PacketConn) {
 	gomaxprocs := cgroup.AvailableCPUs()
 	var wg sync.WaitGroup
 	localAddr := ln.LocalAddr()
@@ -236,7 +196,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			cp := insertutils.GetCommonParamsForSyslog(globalTenantID)
 			var bb bytesutil.ByteBuffer
 			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
 			for {
@@ -261,7 +221,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 				}
 				bb.B = bb.B[:n]
 				udpRequestsTotal.Inc()
-				if err := processStream(bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
+				if err := processStream(bb.NewReader(), cp); err != nil {
 					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
 				}
 			}
@@ -270,7 +230,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 	wg.Wait()
 }
 
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+func serveTCP(ln net.Listener) {
 	var cm ingestserver.ConnsMap
 	cm.Init("syslog")
 
@@ -300,8 +260,8 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
 
 		wg.Add(1)
 		go func() {
-			cp := insertutils.GetCommonParamsForSyslog(tenantID)
-			if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
+			cp := insertutils.GetCommonParamsForSyslog(globalTenantID)
+			if err := processStream(c, cp); err != nil {
 				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
 			}
 
@@ -316,20 +276,8 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
 }
 
 // processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
-	if err := vlstorage.CanWriteData(); err != nil {
-		return err
-	}
-
-	lmp := cp.NewLogMessageProcessor()
-	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
-	lmp.MustClose()
-
-	return err
-}
-
-func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
-	switch compressMethod {
+func processStream(r io.Reader, cp *insertutils.CommonParams) error {
+	switch *compressMethod {
 	case "", "none":
 	case "gzip":
 		zr, err := common.GetGzipReader(r)
@@ -344,12 +292,12 @@ func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp
 		}
 		r = zr
 	default:
-		logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
+		logger.Panicf("BUG: compressLevel=%q; supported values: none, gzip, deflate", *compressMethod)
 	}
 
-	err := processUncompressedStream(r, useLocalTimestamp, lmp)
+	err := processUncompressedStream(r, cp)
 
-	switch compressMethod {
+	switch *compressMethod {
 	case "gzip":
 		zr := r.(*gzip.Reader)
 		common.PutGzipReader(zr)
@@ -361,166 +309,76 @@ func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp
 	return err
 }
 
-func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+func processUncompressedStream(r io.Reader, cp *insertutils.CommonParams) error {
+	if err := vlstorage.CanWriteData(); err != nil {
+		return err
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, nil)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
-	slr := getSyslogLineReader(wcr)
-	defer putSyslogLineReader(slr)
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
 
 	n := 0
 	for {
-		ok := slr.nextLine()
-		wcr.DecConcurrency()
-		if !ok {
-			break
-		}
-
 		currentYear := int(globalCurrentYear.Load())
-		err := processLine(slr.line, currentYear, globalTimezone, useLocalTimestamp, lmp)
+		ok, err := readLine(sc, currentYear, globalTimezone, processLogMessage)
+		wcr.DecConcurrency()
 		if err != nil {
 			errorsTotal.Inc()
 			return fmt.Errorf("cannot read line #%d: %s", n, err)
 		}
+		if !ok {
+			break
+		}
 		n++
 		rowsIngestedTotal.Inc()
 	}
-	return slr.Error()
-}
 
-type syslogLineReader struct {
-	line []byte
-
-	br  *bufio.Reader
-	err error
-}
-
-func (slr *syslogLineReader) reset(r io.Reader) {
-	slr.line = slr.line[:0]
-	slr.br.Reset(r)
-	slr.err = nil
-}
-
-// Error returns the last error occurred in slr.
-func (slr *syslogLineReader) Error() error {
-	if slr.err == nil || slr.err == io.EOF {
-		return nil
-	}
-	return slr.err
-}
-
-// nextLine reads the next syslog line from slr and stores it at slr.line.
-//
-// false is returned if the next line cannot be read. Error() must be called in this case
-// in order to verify whether there is an error or just slr stream has been finished.
-func (slr *syslogLineReader) nextLine() bool {
-	if slr.err != nil {
-		return false
-	}
-
-again:
-	prefix, err := slr.br.ReadSlice(' ')
-	if err != nil {
-		if err != io.EOF {
-			slr.err = fmt.Errorf("cannot read message frame prefix: %w", err)
-			return false
-		}
-		if len(prefix) == 0 {
-			slr.err = err
-			return false
-		}
-	}
-	// skip empty lines
-	for len(prefix) > 0 && prefix[0] == '\n' {
-		prefix = prefix[1:]
-	}
-	if len(prefix) == 0 {
-		// An empty prefix or a prefix with empty lines - try reading yet another prefix.
-		goto again
-	}
-
-	if prefix[0] >= '0' && prefix[0] <= '9' {
-		// This is octet-counting method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#msgxfer
-		msgLenStr := bytesutil.ToUnsafeString(prefix[:len(prefix)-1])
-		msgLen, err := strconv.ParseUint(msgLenStr, 10, 64)
-		if err != nil {
-			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
-			return false
-		}
-		if maxMsgLen := insertutils.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
-			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
-			return false
-		}
-		slr.line = slicesutil.SetLength(slr.line, int(msgLen))
-		if _, err := io.ReadFull(slr.br, slr.line); err != nil {
-			slr.err = fmt.Errorf("cannot read message with size %d bytes: %w", msgLen, err)
-			return false
-		}
-		return true
-	}
-
-	// This is octet-stuffing method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#octet-stuffing-legacy
-	slr.line = append(slr.line[:0], prefix...)
-	for {
-		line, err := slr.br.ReadSlice('\n')
-		if err == nil {
-			slr.line = append(slr.line, line[:len(line)-1]...)
-			return true
-		}
-		if err == io.EOF {
-			slr.line = append(slr.line, line...)
-			return true
-		}
-		if err == bufio.ErrBufferFull {
-			slr.line = append(slr.line, line...)
-			continue
-		}
-		slr.err = fmt.Errorf("cannot read message in octet-stuffing method: %w", err)
-		return false
-	}
-}
-
-func getSyslogLineReader(r io.Reader) *syslogLineReader {
-	v := syslogLineReaderPool.Get()
-	if v == nil {
-		br := bufio.NewReaderSize(r, 64*1024)
-		return &syslogLineReader{
-			br: br,
-		}
-	}
-	slr := v.(*syslogLineReader)
-	slr.reset(r)
-	return slr
-}
-
-func putSyslogLineReader(slr *syslogLineReader) {
-	syslogLineReaderPool.Put(slr)
-}
-
-var syslogLineReaderPool sync.Pool
-
-func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
-	p := logstorage.GetSyslogParser(currentYear, timezone)
-	lineStr := bytesutil.ToUnsafeString(line)
-	p.Parse(lineStr)
-
-	var ts int64
-	if useLocalTimestamp {
-		ts = time.Now().UnixNano()
-	} else {
-		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
-		if err != nil {
-			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
-		}
-		ts = nsecs
-	}
-	logstorage.RenameField(p.Fields, "message", "_msg")
-	lmp.AddRow(ts, p.Fields)
-	logstorage.PutSyslogParser(p)
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
 
 	return nil
 }
 
+func readLine(sc *bufio.Scanner, currentYear int, timezone *time.Location, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
+	var line []byte
+	for len(line) == 0 {
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`line size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
+		}
+		line = sc.Bytes()
+	}
+
+	p := logstorage.GetSyslogParser(currentYear, timezone)
+	lineStr := bytesutil.ToUnsafeString(line)
+	p.Parse(lineStr)
+	ts, err := insertutils.ExtractTimestampISO8601FromFields("timestamp", p.Fields)
+	if err != nil {
+		return false, fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
+	}
+	logstorage.RenameField(p.Fields, "message", "_msg")
+	processLogMessage(ts, p.Fields)
+	logstorage.PutSyslogParser(p)
+
+	return true, nil
+}
+
+var lineBufferPool bytesutil.ByteBufferPool
+
 var (
 	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
 

app/vlinsert/syslog/syslog_test.go

@@ -1,130 +1,65 @@
 package syslog
 
 import (
+	"bufio"
 	"bytes"
 	"reflect"
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-func TestSyslogLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		var lines []string
-		for slr.nextLine() {
-			lines = append(lines, string(slr.line))
-		}
-		if err := slr.Error(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines read;\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", nil)
-	f("\n\n\n", nil)
-
-	f("foobar", []string{"foobar"})
-	f("foobar\n", []string{"foobar\n"})
-	f("\n\nfoo\n\nbar\n\n", []string{"foo\n\nbar\n\n"})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...`, []string{"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches..."})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
-
-`, []string{
-		"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...",
-		"<165>Jun  4 12:08:33 abcd systemd[345]: abc defg",
-		`<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`,
-	})
-}
-
-func TestSyslogLineReader_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		if slr.nextLine() {
-			t.Fatalf("expecting failure to read the first line")
-		}
-		if err := slr.Error(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 aa")
-	f("1233423432 abc")
-}
-
-func TestProcessStreamInternal_Success(t *testing.T) {
+func TestReadLine_Success(t *testing.T) {
 	f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
 		MustInit()
 		defer MustStop()
 
-		globalTimezone = time.UTC
-		globalCurrentYear.Store(int64(currentYear))
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+			result += string(logstorage.MarshalFieldsToJSON(nil, fields)) + "\n"
 		}
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
+
+		r := bytes.NewBufferString(data)
+		sc := bufio.NewScanner(r)
+		rows := 0
+		timezone := time.UTC
+		for {
+			ok, err := readLine(sc, currentYear, timezone, processLogMessage)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if !ok {
+				break
+			}
+			rows++
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
 	}
 
 	data := `Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
+<165>Jun  4 12:08:33 abcd systemd[345]: abc defg
+<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
 `
 	currentYear := 2023
 	rowsExpected := 3
 	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
 	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
 {"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473":"iut=\"3\" eventSource=\"Application 123 = ] 56\" eventID=\"11211\"","_msg":"This is a test message with structured data."}
+`
 	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
 }
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		MustInit()
-		defer MustStop()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 foo")
-	f("123456789 bar")
-}

app/vlogsgenerator/main.go

@@ -230,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 	for i := 0; i < activeStreams; i++ {
 		ip := toIPv4(rand.Uint32())
 		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
+		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
 			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
 		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
 		for j := 0; j < *constFieldsPerLog; j++ {

app/vlselect/logsql/hits_response.qtpl

@@ -44,7 +44,7 @@
 		hs := m[k]
 		hs.sort()
 		timestamps := hs.timestamps
-		hits := hs.hits
+		values := hs.values
 	%}
 	"fields":{%s= k %},
 	"timestamps":[
@@ -56,14 +56,13 @@
 		{% endif %}
 	],
 	"values":[
-		{% if len(hits) > 0 %}
-			{%dul= hits[0] %}
-			{% for _, v := range hits[1:] %}
-				,{%dul= v %}
+		{% if len(values) > 0 %}
+			{%s= values[0] %}
+			{% for _, v := range values[1:] %}
+				,{%s= v %}
 			{% endfor %}
 		{% endif %}
-	],
-	"total":{%dul= hs.hitsTotal %}
+	]
 }
 {% endfunc %}
 

app/vlselect/logsql/hits_response.qtpl.go

@@ -149,7 +149,7 @@ func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k
 	hs := m[k]
 	hs.sort()
 	timestamps := hs.timestamps
-	hits := hs.hits
+	values := hs.values
 
 //line app/vlselect/logsql/hits_response.qtpl:48
 	qw422016.N().S(`"fields":`)
@@ -174,50 +174,46 @@ func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k
 //line app/vlselect/logsql/hits_response.qtpl:56
 	qw422016.N().S(`],"values":[`)
 //line app/vlselect/logsql/hits_response.qtpl:59
-	if len(hits) > 0 {
+	if len(values) > 0 {
 //line app/vlselect/logsql/hits_response.qtpl:60
-		qw422016.N().DUL(hits[0])
+		qw422016.N().S(values[0])
 //line app/vlselect/logsql/hits_response.qtpl:61
-		for _, v := range hits[1:] {
+		for _, v := range values[1:] {
 //line app/vlselect/logsql/hits_response.qtpl:61
 			qw422016.N().S(`,`)
 //line app/vlselect/logsql/hits_response.qtpl:62
-			qw422016.N().DUL(v)
+			qw422016.N().S(v)
 //line app/vlselect/logsql/hits_response.qtpl:63
 		}
 //line app/vlselect/logsql/hits_response.qtpl:64
 	}
 //line app/vlselect/logsql/hits_response.qtpl:64
-	qw422016.N().S(`],"total":`)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().DUL(hs.hitsTotal)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/hits_response.qtpl:68
+	qw422016.N().S(`]}`)
+//line app/vlselect/logsql/hits_response.qtpl:67
 }
 
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 func writehitsSeriesLine(qq422016 qtio422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 }
 
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 func hitsSeriesLine(m map[string]*hitsSeries, k string) string {
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	writehitsSeriesLine(qb422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:68
+//line app/vlselect/logsql/hits_response.qtpl:67
 }

app/vlselect/logsql/logsql.go

@@ -6,18 +6,14 @@ import (
 	"math"
 	"net/http"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -60,20 +56,9 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	// Obtain field entries
 	fields := r.Form["field"]
 
-	// Obtain limit on the number of top fields entries.
-	fieldsLimit, err := httputils.GetInt(r, "fields_limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if fieldsLimit < 0 {
-		fieldsLimit = 0
-	}
-
-	// Prepare the query for hits count.
-	q.Optimize()
-	q.DropAllPipes()
+	// Prepare the query
 	q.AddCountByTimePipe(int64(step), int64(offset), fields)
+	q.Optimize()
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
@@ -90,10 +75,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		for i := range timestamps {
 			timestampStr := strings.Clone(timestampValues[i])
 			hitsStr := strings.Clone(hitsValues[i])
-			hits, err := strconv.ParseUint(hitsStr, 10, 64)
-			if err != nil {
-				logger.Panicf("BUG: cannot parse hitsStr=%q: %s", hitsStr, err)
-			}
 
 			bb.Reset()
 			WriteFieldsForHits(bb, columns, i)
@@ -106,8 +87,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 				m[k] = hs
 			}
 			hs.timestamps = append(hs.timestamps, timestampStr)
-			hs.hits = append(hs.hits, hits)
-			hs.hitsTotal += hits
+			hs.values = append(hs.values, hitsStr)
 			mLock.Unlock()
 		}
 		blockResultPool.Put(bb)
@@ -119,59 +99,14 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	m = getTopHitsSeries(m, fieldsLimit)
-
 	// Write response
 	w.Header().Set("Content-Type", "application/json")
 	WriteHitsSeries(w, m)
 }
 
-func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
-	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
-		return m
-	}
-
-	type fieldsHits struct {
-		fieldsStr string
-		hs        *hitsSeries
-	}
-	a := make([]fieldsHits, 0, len(m))
-	for fieldsStr, hs := range m {
-		a = append(a, fieldsHits{
-			fieldsStr: fieldsStr,
-			hs:        hs,
-		})
-	}
-	sort.Slice(a, func(i, j int) bool {
-		return a[i].hs.hitsTotal > a[j].hs.hitsTotal
-	})
-
-	hitsOther := make(map[string]uint64)
-	for _, x := range a[fieldsLimit:] {
-		for i, timestampStr := range x.hs.timestamps {
-			hitsOther[timestampStr] += x.hs.hits[i]
-		}
-	}
-	var hsOther hitsSeries
-	for timestampStr, hits := range hitsOther {
-		hsOther.timestamps = append(hsOther.timestamps, timestampStr)
-		hsOther.hits = append(hsOther.hits, hits)
-		hsOther.hitsTotal += hits
-	}
-
-	mNew := make(map[string]*hitsSeries, fieldsLimit+1)
-	for _, x := range a[:fieldsLimit] {
-		mNew[x.fieldsStr] = x.hs
-	}
-	mNew["{}"] = &hsOther
-
-	return mNew
-}
-
 type hitsSeries struct {
-	hitsTotal  uint64
 	timestamps []string
-	hits       []uint64
+	values     []string
 }
 
 func (hs *hitsSeries) sort() {
@@ -184,7 +119,7 @@ func (hs *hitsSeries) Len() int {
 
 func (hs *hitsSeries) Swap(i, j int) {
 	hs.timestamps[i], hs.timestamps[j] = hs.timestamps[j], hs.timestamps[i]
-	hs.hits[i], hs.hits[j] = hs.hits[j], hs.hits[i]
+	hs.values[i], hs.values[j] = hs.values[j], hs.values[i]
 }
 
 func (hs *hitsSeries) Less(i, j int) bool {
@@ -315,38 +250,6 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
 	WriteValuesWithHitsJSON(w, values)
 }
 
-// ProcessStreamIDsRequest processes /select/logsql/stream_ids request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream_ids
-func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Parse limit query arg
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if limit < 0 {
-		limit = 0
-	}
-
-	// Obtain streamIDs for the given query
-	q.Optimize()
-	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain stream_ids: %s", err)
-	}
-
-	// Write results
-	w.Header().Set("Content-Type", "application/json")
-	WriteValuesWithHitsJSON(w, streamIDs)
-}
-
 // ProcessStreamsRequest processes /select/logsql/streams request.
 //
 // See https://docs.victoriametrics.com/victorialogs/querying/#querying-streams
@@ -379,187 +282,6 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 	WriteValuesWithHitsJSON(w, streams)
 }
 
-// ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
-func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	liveTailRequests.Inc()
-	defer liveTailRequests.Dec()
-
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if !q.CanLiveTail() {
-		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
-	}
-	q.Optimize()
-
-	refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
-
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	tp := newTailProcessor(cancel)
-
-	ticker := time.NewTicker(refreshInterval)
-	defer ticker.Stop()
-
-	end := time.Now().UnixNano()
-	doneCh := ctxWithCancel.Done()
-	flusher, ok := w.(http.Flusher)
-	if !ok {
-		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
-	}
-	for {
-		start := end - tailOffsetNsecs
-		end = time.Now().UnixNano()
-
-		qCopy := q.Clone()
-		qCopy.AddTimeFilter(start, end)
-		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
-			httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
-			return
-		}
-		resultRows, err := tp.getTailRows()
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot get tail results for query [%q]: %s", q, err)
-			return
-		}
-		if len(resultRows) > 0 {
-			WriteJSONRows(w, resultRows)
-			flusher.Flush()
-		}
-
-		select {
-		case <-doneCh:
-			return
-		case <-ticker.C:
-		}
-	}
-}
-
-var liveTailRequests = metrics.NewCounter(`vl_live_tailing_requests`)
-
-const tailOffsetNsecs = 5e9
-
-type logRow struct {
-	timestamp int64
-	fields    []logstorage.Field
-}
-
-func sortLogRows(rows []logRow) {
-	sort.SliceStable(rows, func(i, j int) bool {
-		return rows[i].timestamp < rows[j].timestamp
-	})
-}
-
-type tailProcessor struct {
-	cancel func()
-
-	mu sync.Mutex
-
-	perStreamRows  map[string][]logRow
-	lastTimestamps map[string]int64
-
-	err error
-}
-
-func newTailProcessor(cancel func()) *tailProcessor {
-	return &tailProcessor{
-		cancel: cancel,
-
-		perStreamRows:  make(map[string][]logRow),
-		lastTimestamps: make(map[string]int64),
-	}
-}
-
-func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-	if len(timestamps) == 0 {
-		return
-	}
-
-	tp.mu.Lock()
-	defer tp.mu.Unlock()
-
-	if tp.err != nil {
-		return
-	}
-
-	// Make sure columns contain _time field, since it is needed for proper tail work.
-	hasTime := false
-	for _, c := range columns {
-		if c.Name == "_time" {
-			hasTime = true
-			break
-		}
-	}
-	if !hasTime {
-		tp.err = fmt.Errorf("missing _time field")
-		tp.cancel()
-		return
-	}
-
-	// Copy block rows to tp.perStreamRows
-	for i, timestamp := range timestamps {
-		streamID := ""
-		fields := make([]logstorage.Field, len(columns))
-		for j, c := range columns {
-			name := strings.Clone(c.Name)
-			value := strings.Clone(c.Values[i])
-
-			fields[j] = logstorage.Field{
-				Name:  name,
-				Value: value,
-			}
-
-			if name == "_stream_id" {
-				streamID = value
-			}
-		}
-
-		tp.perStreamRows[streamID] = append(tp.perStreamRows[streamID], logRow{
-			timestamp: timestamp,
-			fields:    fields,
-		})
-	}
-}
-
-func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
-	if tp.err != nil {
-		return nil, tp.err
-	}
-
-	var resultRows []logRow
-	for streamID, rows := range tp.perStreamRows {
-		sortLogRows(rows)
-
-		lastTimestamp, ok := tp.lastTimestamps[streamID]
-		if ok {
-			// Skip already written rows
-			for len(rows) > 0 && rows[0].timestamp <= lastTimestamp {
-				rows = rows[1:]
-			}
-		}
-		if len(rows) > 0 {
-			resultRows = append(resultRows, rows...)
-			tp.lastTimestamps[streamID] = rows[len(rows)-1].timestamp
-		}
-	}
-	clear(tp.perStreamRows)
-
-	sortLogRows(resultRows)
-
-	tailRows := make([][]logstorage.Field, len(resultRows))
-	for i, row := range resultRows {
-		tailRows[i] = row.fields
-	}
-
-	return tailRows, nil
-}
-
 // ProcessQueryRequest handles /select/logsql/query request.
 //
 // See https://docs.victoriametrics.com/victorialogs/querying/#http-api
@@ -622,7 +344,6 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 
 	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
 		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
-		return
 	}
 }
 

app/vlselect/main.go

@@ -1,7 +1,6 @@
 package vlselect
 
 import (
-	"context"
 	"embed"
 	"flag"
 	"fmt"
@@ -14,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -23,7 +23,7 @@ var (
 		"See also -search.maxQueueDuration")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden on a per-query basis via 'timeout' query arg")
+	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
 )
 
 func getDefaultMaxConcurrentRequests() int {
@@ -98,83 +98,47 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
+	// Limit the number of concurrent queries, which can consume big amounts of CPU.
 	startTime := time.Now()
 	ctx := r.Context()
-	d := getMaxQueryDuration(r)
-	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
-	defer cancel()
-
-	stopCh := ctxWithTimeout.Done()
+	stopCh := ctx.Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
+		d := getMaxQueryDuration(r)
+		if d > *maxQueueDuration {
+			d = *maxQueueDuration
+		}
+		t := timerpool.Get(d)
 		select {
 		case concurrencyLimitCh <- struct{}{}:
+			timerpool.Put(t)
 			defer func() { <-concurrencyLimitCh }()
 		case <-stopCh:
-			switch ctxWithTimeout.Err() {
-			case context.Canceled:
-				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-				requestURI := httpserver.GetRequestURI(r)
-				logger.Infof("client has canceled the pending request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-					time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			case context.DeadlineExceeded:
-				concurrencyLimitTimeout.Inc()
-				err := &httpserver.ErrorWithStatusCode{
-					Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-						"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-						"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
-						"to pass bigger value to 'timeout' query arg",
-						d.Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
-					StatusCode: http.StatusServiceUnavailable,
-				}
-				httpserver.Errorf(w, r, "%s", err)
+			timerpool.Put(t)
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			requestURI := httpserver.GetRequestURI(r)
+			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+				time.Since(startTime).Seconds(), remoteAddr, requestURI)
+			return true
+		case <-t.C:
+			timerpool.Put(t)
+			concurrencyLimitTimeout.Inc()
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
+					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
+					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
+					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
+				StatusCode: http.StatusServiceUnavailable,
 			}
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 	}
 
-	if path == "/select/logsql/tail" {
-		logsqlTailRequests.Inc()
-		// Process live tailing request without timeout (e.g. use ctx instead of ctxWithTimeout),
-		// since it is OK to run live tailing requests for very long time.
-		logsql.ProcessLiveTailRequest(ctx, w, r)
-		return true
-	}
-
-	ok := processSelectRequest(ctxWithTimeout, w, r, path)
-	if !ok {
-		return false
-	}
-
-	err := ctxWithTimeout.Err()
-	switch err {
-	case nil:
-		// nothing to do
-	case context.Canceled:
-		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		requestURI := httpserver.GetRequestURI(r)
-		logger.Infof("client has canceled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-			time.Since(startTime).Seconds(), remoteAddr, requestURI)
-	case context.DeadlineExceeded:
-		err = &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
-				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", d.Seconds(), maxQueryDuration),
-			StatusCode: http.StatusServiceUnavailable,
-		}
-		httpserver.Errorf(w, r, "%s", err)
-	default:
-		httpserver.Errorf(w, r, "unexpected error: %s", err)
-	}
-
-	return true
-}
-
-func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
 	httpserver.EnableCORS(w, r)
 	switch path {
 	case "/select/logsql/field_names":
@@ -201,10 +165,6 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 		logsqlStreamFieldValuesRequests.Inc()
 		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
 		return true
-	case "/select/logsql/stream_ids":
-		logsqlStreamIDsRequests.Inc()
-		logsql.ProcessStreamIDsRequest(ctx, w, r)
-		return true
 	case "/select/logsql/streams":
 		logsqlStreamsRequests.Inc()
 		logsql.ProcessStreamsRequest(ctx, w, r)
@@ -234,7 +194,5 @@ var (
 	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
 	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
 	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
-	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
 	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
-	logsqlTailRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
 )

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.1041c3d4.css",
-    "main.js": "./static/js/main.8451d9b9.js",
+    "main.css": "./static/css/main.2fa7c03f.css",
+    "main.js": "./static/js/main.68f1bd69.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.cb83d071da309a358bc0.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.1041c3d4.css",
-    "static/js/main.8451d9b9.js"
+    "static/css/main.2fa7c03f.css",
+    "static/js/main.68f1bd69.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8451d9b9.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.68f1bd69.js"></script><link href="./static/css/main.2fa7c03f.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.68f1bd69.js.LICENSE.txt

@@ -5,7 +5,7 @@
 */
 
 /**
- * @remix-run/router v1.17.0
+ * @remix-run/router v1.15.1
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.24.0
+ * React Router DOM v6.22.1
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +27,7 @@
  */
 
 /**
- * React Router v6.24.0
+ * React Router v6.22.1
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlselect/vmui/static/media/MetricsQL.cb83d071da309a358bc0.md

@@ -10,6 +10,9 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
+
+# MetricsQL
+
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -104,7 +107,7 @@ The list of MetricsQL features on top of PromQL:
 * Trailing commas on all the lists are allowed - label filters, function args and with expressions.
   For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
   This simplifies maintenance of multi-line queries.
-* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a valid MetricsQL expression.
+* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a value MetricsQL expression.
 * Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
   It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
   Additionally, the following escape sequences are supported:
@@ -217,11 +220,9 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
+See also [median_over_time](#median_over_time).
 
 #### changes
 
@@ -261,8 +262,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -273,8 +272,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -285,8 +282,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -297,9 +292,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
-See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
+See also [count_over_time](#count_over_time).
 
 #### count_over_time
 
@@ -320,8 +313,6 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -441,8 +432,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -454,15 +443,11 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -470,8 +455,6 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -479,9 +462,8 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`.
-
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
+returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -513,14 +495,13 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -529,13 +510,12 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -544,10 +524,6 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
-
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -564,18 +540,15 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -614,8 +587,6 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -623,11 +594,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
+See also [tmax_over_time](#tmax_over_time).
 
 #### median_over_time
 
@@ -635,8 +604,6 @@ See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -644,11 +611,9 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
+See also [tmin_over_time](#tmin_over_time).
 
 #### mode_over_time
 
@@ -657,8 +622,6 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -669,8 +632,6 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -698,8 +659,6 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -711,8 +670,6 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -723,12 +680,11 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -747,8 +703,6 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -756,11 +710,10 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 #### rollup
@@ -772,10 +725,6 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
-See also [rollup_rate](#rollup_rate).
-
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -787,8 +736,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -815,8 +762,6 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
-
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -829,17 +774,12 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
-
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -847,12 +787,10 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
-
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -886,8 +824,6 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -902,8 +838,6 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -915,8 +849,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -934,8 +866,6 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -947,8 +877,6 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
@@ -960,8 +888,6 @@ values equal to `eq` on the given lookbehind window `d` per each time series ret
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
@@ -971,8 +897,6 @@ values bigger than `gt` on the given lookbehind window `d` per each time series
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
@@ -982,8 +906,6 @@ values smaller or equal to `le` on the given lookbehind window `d` per each time
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -993,8 +915,6 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -1004,8 +924,6 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1083,8 +1001,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2314,8 +2230,7 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2348,6 +2263,5 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
-  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
+  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.

app/vlstorage/main.go

@@ -20,12 +20,10 @@ import (
 var (
 	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
 		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
-	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
-		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
+		"see https://docs.victoriametrics.com/victorialogs/#retention")
 	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
 		"see https://docs.victoriametrics.com/victorialogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
+	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory with the VictoriaLogs data; "+
 		"see https://docs.victoriametrics.com/victorialogs/#storage")
 	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
 		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
@@ -51,13 +49,12 @@ func Init() {
 		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
 	}
 	cfg := &logstorage.StorageConfig{
-		Retention:              retentionPeriod.Duration(),
-		MaxDiskSpaceUsageBytes: maxDiskSpaceUsageBytes.N,
-		FlushInterval:          *inmemoryDataFlushInterval,
-		FutureRetention:        futureRetention.Duration(),
-		LogNewStreams:          *logNewStreams,
-		LogIngestedRows:        *logIngestedRows,
-		MinFreeDiskSpaceBytes:  minFreeDiskSpaceBytes.N,
+		Retention:             retentionPeriod.Duration(),
+		FlushInterval:         *inmemoryDataFlushInterval,
+		FutureRetention:       futureRetention.Duration(),
+		LogNewStreams:         *logNewStreams,
+		LogIngestedRows:       *logIngestedRows,
+		MinFreeDiskSpaceBytes: minFreeDiskSpaceBytes.N,
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
@@ -80,7 +77,7 @@ func Init() {
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics, true)
+	metrics.UnregisterSet(storageMetrics)
 	storageMetrics = nil
 
 	strg.MustClose()
@@ -145,13 +142,6 @@ func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstor
 	return strg.GetStreams(ctx, tenantIDs, q, limit)
 }
 
-// GetStreamIDs executes q and returns streamIDs seen in query results.
-//
-// If limit > 0, then up to limit unique streamIDs are returned.
-func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
-}
-
 func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	var ss logstorage.StorageStats
 	strg.UpdateStats(&ss)

app/vmagent/main.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
@@ -26,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -38,13 +37,14 @@ import (
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	statsdserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -63,6 +63,12 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	statsdListenAddr = flag.String("statsdListenAddr", "", "TCP and UDP address to listen for Statsd plaintext data. Usually :8125 must be set. Doesn't work if empty. "+
+		"See also -statsdListenAddr.useProxyProtocol")
+	statsdUseProxyProtocol = flag.Bool("statsdListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -statsdListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	statsdDisableAggregationEnforcemenet = flag.Bool(`statsd.disableAggregationEnforcement`, false, "Whether to disable streaming aggregation requirement check. "+
+		"It's recommended to run statsdServer with pre-configured streaming aggregation to decrease load at database.")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
@@ -72,8 +78,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -82,6 +88,7 @@ var (
 var (
 	influxServer       *influxserver.Server
 	graphiteServer     *graphiteserver.Server
+	statsdServer       *statsdserver.Server
 	opentsdbServer     *opentsdbserver.Server
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
@@ -116,7 +123,7 @@ func main() {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
+			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
@@ -139,6 +146,12 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
 	}
+	if len(*statsdListenAddr) > 0 {
+		if !remotewrite.HasAnyStreamAggrConfigured() && !*statsdDisableAggregationEnforcemenet {
+			logger.Fatalf("streaming aggregation must be configured with enabled statsd server. It's recommended  to aggregate metrics received at statsd listener. This check could be disabled with flag -statsd.disableAggregationEnforcement")
+		}
+		statsdServer = statsdserver.MustStart(*statsdListenAddr, *statsdUseProxyProtocol, statsd.InsertHandler)
+	}
 	if len(*opentsdbListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
 		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
@@ -174,6 +187,9 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer.MustStop()
 	}
+	if len(*statsdListenAddr) > 0 {
+		statsdServer.MustStop()
+	}
 	if len(*opentsdbListenAddr) > 0 {
 		opentsdbServer.MustStop()
 	}
@@ -318,10 +334,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -440,7 +452,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -449,17 +461,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		var bb bytesutil.ByteBuffer
 		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		promscrapeConfigReloadRequests.Inc()
@@ -568,10 +580,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -682,8 +690,7 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
-	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image AS certs
+FROM $certs_image as certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmagent/prometheusimport/request_handler_test.go

@@ -26,11 +26,11 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unxepected error %s", err)
+		t.Errorf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 

app/vmagent/remotewrite/client.go

@@ -15,8 +15,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -120,7 +120,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
-		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
+		DialContext:         httputils.GetStatDialFunc("vmagent_remotewrite"),
 		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,

app/vmagent/remotewrite/pendingseries.go

@@ -109,9 +109,10 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss     []prompbmarshal.TimeSeries
-	labels  []prompbmarshal.Label
-	samples []prompbmarshal.Sample
+	tss       []prompbmarshal.TimeSeries
+	labels    []prompbmarshal.Label
+	samples   []prompbmarshal.Sample
+	exemplars []prompbmarshal.Exemplar
 
 	// buf holds labels data
 	buf []byte
@@ -129,6 +130,7 @@ func (wr *writeRequest) reset() {
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
+	wr.exemplars = wr.exemplars[:0]
 	wr.buf = wr.buf[:0]
 }
 
@@ -200,6 +202,7 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	labelsDst := wr.labels
 	labelsLen := len(wr.labels)
 	samplesDst := wr.samples
+	exemplarsDst := wr.exemplars
 	buf := wr.buf
 	for i := range src.Labels {
 		labelsDst = append(labelsDst, prompbmarshal.Label{})
@@ -216,8 +219,12 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	samplesDst = append(samplesDst, src.Samples...)
 	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
 
+	exemplarsDst = append(exemplarsDst, src.Exemplars...)
+	dst.Exemplars = exemplarsDst[len(exemplarsDst)-len(src.Exemplars):]
+
 	wr.samples = samplesDst
 	wr.labels = labelsDst
+	wr.exemplars = exemplarsDst
 	wr.buf = buf
 }
 
@@ -229,7 +236,6 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 		// Nothing to push
 		return true
 	}
-
 	marshalConcurrencyCh <- struct{}{}
 
 	bb := writeRequestBufPool.Get()
@@ -266,6 +272,8 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 	if len(wr.Timeseries) == 1 {
 		// A single time series left. Recursively split its samples into smaller parts if possible.
 		samples := wr.Timeseries[0].Samples
+		exemplars := wr.Timeseries[0].Exemplars
+
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
 			return true
@@ -277,11 +285,16 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 			return false
 		}
 		wr.Timeseries[0].Samples = samples[n:]
+		// We do not want to send exemplars twice
+		wr.Timeseries[0].Exemplars = nil
+
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
+			wr.Timeseries[0].Exemplars = exemplars
 			return false
 		}
 		wr.Timeseries[0].Samples = samples
+		wr.Timeseries[0].Exemplars = exemplars
 		return true
 	}
 	timeseries := wr.Timeseries

app/vmagent/remotewrite/pendingseries_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestPushWriteRequest(t *testing.T) {
 	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
-	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
-	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
+	expectedBlockLensProm := []int{248, 1952, 17433, 180381, 1861994}
+	expectedBlockLensVM := []int{170, 575, 4748, 44936, 367096}
 	for i, rowsCount := range rowsCounts {
 		expectedBlockLenProm := expectedBlockLensProm[i]
 		expectedBlockLenVM := expectedBlockLensVM[i]
@@ -59,6 +59,20 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 				Value: fmt.Sprintf("value_%d_%d", i, j),
 			})
 		}
+		exemplar := prompbmarshal.Exemplar{
+			Labels: []prompbmarshal.Label{
+				{
+					Name:  "trace_id",
+					Value: "123456",
+				},
+				{
+					Name:  "log_id",
+					Value: "987654",
+				},
+			},
+			Value:     float64(i),
+			Timestamp: 1000 * int64(i),
+		}
 		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
 			Labels: labels,
 			Samples: []prompbmarshal.Sample{
@@ -67,6 +81,10 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 					Timestamp: 1000 * int64(i),
 				},
 			},
+
+			Exemplars: []prompbmarshal.Exemplar{
+				exemplar,
+			},
 		})
 	}
 	return &wr

app/vmagent/remotewrite/relabel.go

@@ -181,7 +181,7 @@ func (rctx *relabelCtx) reset() {
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		return &relabelCtx{}
 	},
 }

app/vmagent/remotewrite/remotewrite.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
-	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -89,15 +88,15 @@ var (
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
-		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence ."+
+		"See also -remoteWrite.dropSamplesOnOverload")
+	dropSamplesOnOverload = flagutil.NewArrayBool("remoteWrite.dropSamplesOnOverload", "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
 )
 
 var (
-	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
-	rwctxsGlobal []*remoteWriteCtx
+	// rwctxs contains statically populated entries when -remoteWrite.url is specified.
+	rwctxs []*remoteWriteCtx
 
 	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
 	defaultAuthToken = &auth.Token{}
@@ -110,11 +109,8 @@ var (
 		StatusCode: http.StatusTooManyRequests,
 	}
 
-	// disableOnDiskQueueAny is set to true if at least a single -remoteWrite.url is configured with -remoteWrite.disableOnDiskQueue
-	disableOnDiskQueueAny bool
-
-	// dropSamplesOnFailureGlobal is set to true if -remoteWrite.dropSamplesOnOverload is set or if multiple -remoteWrite.disableOnDiskQueue options are set.
-	dropSamplesOnFailureGlobal bool
+	// disableOnDiskQueueAll is set to true if all remoteWrite.urls were configured to disable persistent queue via disableOnDiskQueue
+	disableOnDiskQueueAll bool
 )
 
 // MultitenancyEnabled returns true if -enableMultitenantHandlers is specified.
@@ -207,18 +203,28 @@ func Init() {
 	relabelConfigSuccess.Set(1)
 	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
-	initStreamAggrConfigGlobal()
+	sasFile, sasOpts := getStreamAggrOpts(-1)
+	if sasFile != "" {
+		sas, err := newStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -streamAggr.config=%q: %s", sasFile, err)
+		}
+		sasGlobal.Store(sas)
+	} else if sasOpts.DedupInterval > 0 {
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesDropFailed, sasOpts.DedupInterval, sasOpts.DropInputLabels)
+	}
 
-	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	if len(*remoteWriteURLs) > 0 {
+		rwctxs = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	}
 
-	disableOnDiskQueues := []bool(*disableOnDiskQueue)
-	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
-
-	// Samples must be dropped if multiple -remoteWrite.disableOnDiskQueue options are configured and at least a single is set to true.
-	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
-	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
-	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
-	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
+	disableOnDiskQueueAll = true
+	for _, v := range *disableOnDiskQueue {
+		if !v {
+			disableOnDiskQueueAll = false
+			break
+		}
+	}
 
 	dropDanglingQueues()
 
@@ -228,9 +234,9 @@ func Init() {
 		defer configReloaderWG.Done()
 		for {
 			select {
+			case <-sighupCh:
 			case <-configReloaderStopCh:
 				return
-			case <-sighupCh:
 			}
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
@@ -249,8 +255,8 @@ func dropDanglingQueues() {
 	// In case if there were many persistent queues with identical *remoteWriteURLs
 	// the queue with the last index will be dropped.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
-	for _, rwctx := range rwctxsGlobal {
+	existingQueues := make(map[string]struct{}, len(rwctxs))
+	for _, rwctx := range rwctxs {
 		existingQueues[rwctx.fq.Dirname()] = struct{}{}
 	}
 
@@ -267,7 +273,7 @@ func dropDanglingQueues() {
 		}
 	}
 	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
 	}
 }
 
@@ -376,10 +382,10 @@ func Stop() {
 		deduplicatorGlobal = nil
 	}
 
-	for _, rwctx := range rwctxsGlobal {
+	for _, rwctx := range rwctxs {
 		rwctx.MustStop()
 	}
-	rwctxsGlobal = nil
+	rwctxs = nil
 
 	if sl := hourlySeriesLimiter; sl != nil {
 		sl.MustStop()
@@ -391,8 +397,6 @@ func Stop() {
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
 //
-// PushDropSamplesOnFailure drops wr samples if they cannot be sent to -remoteWrite.url by any reason.
-//
 // PushDropSamplesOnFailure can modify wr contents.
 func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
@@ -405,7 +409,7 @@ func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
 func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, dropSamplesOnFailureGlobal)
+	return tryPush(at, wr, false)
 }
 
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
@@ -422,30 +426,33 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		tenantRctx = getRelabelCtx()
 		defer putRelabelCtx(tenantRctx)
 	}
+	rowsCount := getRowsCount(tss)
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
 	// if some of remote storage systems cannot keep up with the data ingestion rate.
-	rwctxs, ok := getEligibleRemoteWriteCtxs(tss, forceDropSamplesOnFailure)
-	if !ok {
-		// At least a single remote write queue is blocked and dropSamplesOnFailure isn't set.
-		// Return false to the caller, so it could re-send samples again.
-		return false
-	}
-	if len(rwctxs) == 0 {
-		// All the remote write queues are skipped because they are blocked and dropSamplesOnFailure is set to true.
-		// Return true to the caller, so it doesn't re-send the samples again.
-		return true
+	// this shortcut is only applicable if all remote writes have disableOnDiskQueue = true
+	if disableOnDiskQueueAll {
+		for _, rwctx := range rwctxs {
+			if rwctx.fq.IsWriteBlocked() {
+				rwctx.pushFailures.Inc()
+				if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
+					// Just drop samples
+					rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+					continue
+				}
+				return false
+			}
+		}
 	}
 
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
+	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}
-	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
@@ -498,46 +505,20 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
-		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushBlockToRemoteStorages(tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
 	return true
 }
 
-func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
-	if !disableOnDiskQueueAny {
-		return rwctxsGlobal, true
-	}
-
-	// This code is applicable if at least a single remote storage has -disableOnDiskQueue
-	rwctxs := make([]*remoteWriteCtx, 0, len(rwctxsGlobal))
-	for _, rwctx := range rwctxsGlobal {
-		if !rwctx.fq.IsWriteBlocked() {
-			rwctxs = append(rwctxs, rwctx)
-		} else {
-			rwctx.pushFailures.Inc()
-			if !forceDropSamplesOnFailure {
-				return nil, false
-			}
-			rowsCount := getRowsCount(tss)
-			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-		}
-	}
-	return rwctxs, true
-}
-
-func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
-	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
-	if len(rwctxs) == 0 {
+func pushToRemoteStoragesDropFailed(tss []prompbmarshal.TimeSeries) {
+	if tryPushBlockToRemoteStorages(tss, true) {
 		return
 	}
-	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
-		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
-	}
 }
 
-func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -556,7 +537,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingBlockAmongRemoteStorages(tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
@@ -577,7 +558,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 	return !anyPushFailed.Load()
 }
 
-func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingBlockAmongRemoteStorages(tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -764,8 +745,10 @@ type remoteWriteCtx struct {
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
 
-	streamAggrKeepInput bool
-	streamAggrDropInput bool
+	streamAggrKeepInput   bool
+	streamAggrDropInput   bool
+	disableOnDiskQueue    bool
+	dropSamplesOnOverload bool
 
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
@@ -790,7 +773,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
-
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -835,20 +817,38 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		c:   c,
 		pss: pss,
 
-		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		dropSamplesOnOverload: dropSamplesOnOverload.GetOptionalArg(argIdx),
+		disableOnDiskQueue:    isPQDisabled,
 
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+
+		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+	}
+
+	// Initialize sas
+	sasFile, sasOpts := getStreamAggrOpts(argIdx)
+	if sasFile != "" {
+		sas, err := newStreamAggrConfig(argIdx, rwctx.pushInternalTrackDropped)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
+		}
+		rwctx.sas.Store(sas)
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
+	} else if sasOpts.DedupInterval > 0 {
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, sasOpts.DedupInterval, sasOpts.DropInputLabels)
 	}
-	rwctx.initStreamAggrConfig()
 
 	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
 	// sas and deduplicator must be stopped before rwctx is closed
-	// because they can write pending series to rwctx.pss if there are any
+	// because sas can write pending series to rwctx.pss if there are any
 	sas := rwctx.sas.Swap(nil)
 	sas.MustStop()
 
@@ -875,20 +875,12 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 // TryPush sends tss series to the configured remote write endpoint
 //
-// TryPush doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
+// TryPush can be called concurrently for multiple remoteWriteCtx,
+// so it shouldn't modify tss entries.
 func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
-	defer func() {
-		if rctx == nil {
-			return
-		}
-		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
-	}()
-
-	// Apply relabeling
 	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
 	if pcs.Len() > 0 {
@@ -924,22 +916,28 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 		matchIdxsPool.Put(matchIdxs)
 	} else if rwctx.deduplicator != nil {
 		rwctx.deduplicator.Push(tss)
-		return true
+		tss = tss[:0]
 	}
 
-	// Try pushing tss to remote storage
-	if rwctx.tryPushInternal(tss) {
-		return true
+	// Try pushing the data to remote storage
+	ok := rwctx.tryPushInternal(tss)
+
+	// Return back relabeling contexts to the pool
+	if rctx != nil {
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
 	}
 
-	// Couldn't push tss to remote storage
-	rwctx.pushFailures.Inc()
-	if forceDropSamplesOnFailure {
-		rowsCount := getRowsCount(tss)
-		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-		return true
+	if !ok {
+		rwctx.pushFailures.Inc()
+		if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
+			rwctx.rowsDroppedOnPushFailure.Add(len(tss))
+			return true
+		}
 	}
-	return false
+
+	return ok
 }
 
 var matchIdxsPool bytesutil.ByteBufferPool
@@ -963,26 +961,19 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSe
 	if rwctx.tryPushInternal(tss) {
 		return
 	}
-	if !rwctx.fq.IsPersistentQueueDisabled() {
+	if !rwctx.disableOnDiskQueue {
 		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
 	}
 	rwctx.pushFailures.Inc()
-	rowsCount := getRowsCount(tss)
-	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	if dropSamplesOnOverload.GetOptionalArg(rwctx.idx) {
+		rowsCount := getRowsCount(tss)
+		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	}
 }
 
 func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
-	defer func() {
-		if rctx == nil {
-			return
-		}
-		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
-	}()
-
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels in order to prevent
 		// from affecting time series for other remoteWrite.url configs.
@@ -995,11 +986,19 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
-	return pss[idx].TryPush(tss)
+	ok := pss[idx].TryPush(tss)
+
+	if rctx != nil {
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
+	}
+
+	return ok
 }
 
 var tssPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}
 		return &a
 	},

app/vmagent/remotewrite/remotewrite_test.go

@@ -3,12 +3,14 @@ package remotewrite
 import (
 	"fmt"
 	"math"
+	"os"
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -77,28 +79,24 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil)
 		}
 
-		if streamAggrConfig != "" {
-			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
+		if len(streamAggrConfig) > 0 {
+			f := createFile(t, []byte(streamAggrConfig))
+			sas, err := streamaggr.LoadFromFile(f.Name(), nil, nil)
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
-			defer sas.MustStop()
 			rwctx.sas.Store(sas)
 		}
 
-		offsetMsecs := time.Now().UnixMilli()
-		inputTss := prompbmarshal.MustParsePromMetrics(input, offsetMsecs)
+		inputTss := mustParsePromMetrics(input)
 		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
 
 		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		if !rwctx.TryPush(inputTss, false) {
-			t.Fatalf("cannot push samples to rwctx")
-		}
+		rwctx.TryPush(inputTss, false)
 
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
@@ -167,3 +165,51 @@ metric{env="dev"} 15
 metric{env="bar"} 25
 `)
 }
+
+func mustParsePromMetrics(s string) []prompbmarshal.TimeSeries {
+	var rows prometheus.Rows
+	errLogger := func(s string) {
+		panic(fmt.Errorf("unexpected error when parsing Prometheus metrics: %s", s))
+	}
+	rows.UnmarshalWithErrLogger(s, errLogger)
+	var tss []prompbmarshal.TimeSeries
+	samples := make([]prompbmarshal.Sample, 0, len(rows.Rows))
+	for _, row := range rows.Rows {
+		labels := make([]prompbmarshal.Label, 0, len(row.Tags)+1)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: row.Metric,
+		})
+		for _, tag := range row.Tags {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     row.Value,
+			Timestamp: row.Timestamp,
+		})
+		ts := prompbmarshal.TimeSeries{
+			Labels:  labels,
+			Samples: samples[len(samples)-1:],
+		}
+		tss = append(tss, ts)
+	}
+	return tss
+}
+
+func createFile(t *testing.T, data []byte) *os.File {
+	t.Helper()
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(f.Name(), data, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Sync(); err != nil {
+		t.Fatal(err)
+	}
+	return f
+}

app/vmagent/remotewrite/streamaggr.go

@@ -36,207 +36,125 @@ var (
 		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 
 	// Per URL config
-	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
+	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
 		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
+		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
+		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
+		"with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
-		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
+		"aggregation interval for the corresponding -remoteWrite.streamAggr.config . "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
-		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
-		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
+	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if "+
+		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
-		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
 func CheckStreamAggrConfigs() error {
-	// Check global config
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		return err
-	}
-	sas.MustStop()
-
-	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
-		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
-	}
-
 	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+
+	if _, err := newStreamAggrConfig(-1, pushNoop); err != nil {
+		return fmt.Errorf("could not load -streamAggr.config stream aggregation config: %w", err)
+	}
+	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
+		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
+			len(*streamAggrConfig), len(*remoteWriteURLs))
+	}
 	for idx := range *streamAggrConfig {
-		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
-		if err != nil {
+		if _, err := newStreamAggrConfig(idx, pushNoop); err != nil {
 			return err
 		}
-		sas.MustStop()
 	}
 	return nil
 }
 
+// HasAnyStreamAggrConfigured checks if any streaming aggregation config provided
+func HasAnyStreamAggrConfigured() bool {
+	return len(*streamAggrConfig) > 0 || *streamAggrGlobalConfig != ""
+}
+
 func reloadStreamAggrConfigs() {
-	reloadStreamAggrConfigGlobal()
-	for _, rwctx := range rwctxsGlobal {
-		rwctx.reloadStreamAggrConfig()
+	reloadStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
+	for idx, rwctx := range rwctxs {
+		reloadStreamAggrConfig(idx, rwctx.pushInternalTrackDropped)
 	}
 }
 
-func reloadStreamAggrConfigGlobal() {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
-		return
-	}
-
-	sas := sasGlobal.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := sasGlobal.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
-}
-
-func initStreamAggrConfigGlobal() {
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		sasGlobal.Store(sas)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrGlobalDedupInterval.Duration()
-		if dedupInterval > 0 {
-			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
-		}
-	}
-}
-
-func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
-	idx := rwctx.idx
-
-	sas, err := rwctx.newStreamAggrConfig()
-	if err != nil {
-		logger.Fatalf("cannot initialize stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-		if dedupInterval > 0 {
-			alias := fmt.Sprintf("dedup-%d", idx+1)
-			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
-		}
-	}
-}
-
-func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
-	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if path == "" {
-		return
-	}
-
+func reloadStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) {
+	path, opts := getStreamAggrOpts(idx)
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
 
-	sasNew, err := rwctx.newStreamAggrConfig()
+	sasNew, err := newStreamAggrConfigWithOpts(pushFunc, path, opts)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		logger.Errorf("cannot reload stream aggregation config at %q; continue using the previously loaded config; error: %s", path, err)
 		return
 	}
 
-	sas := rwctx.sas.Load()
+	var sas *streamaggr.Aggregators
+	if idx < 0 {
+		sas = sasGlobal.Load()
+	} else {
+		sas = rwctxs[idx].sas.Load()
+	}
+
 	if !sasNew.Equal(sas) {
-		sasOld := rwctx.sas.Swap(sasNew)
+		var sasOld *streamaggr.Aggregators
+		if idx < 0 {
+			sasOld = sasGlobal.Swap(sasNew)
+		} else {
+			sasOld = rwctxs[idx].sas.Swap(sasNew)
+		}
 		sasOld.MustStop()
-		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
+		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
 	} else {
 		sasNew.MustStop()
-		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
+		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
 }
 
-func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return nil, nil
+func getStreamAggrOpts(idx int) (string, *streamaggr.Options) {
+	if idx < 0 {
+		return *streamAggrGlobalConfig, &streamaggr.Options{
+			DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
+			DropInputLabels:      *streamAggrGlobalDropInputLabels,
+			IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
+			IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+		}
 	}
-
-	opts := &streamaggr.Options{
-		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
-		DropInputLabels:      *streamAggrGlobalDropInputLabels,
-		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
-		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
-		KeepInput:            *streamAggrGlobalKeepInput,
-	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
-	}
-	return sas, nil
-}
-
-func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
-	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
-}
-
-func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path := streamAggrConfig.GetOptionalArg(idx)
-	if path == "" {
-		return nil, nil
-	}
-
-	alias := fmt.Sprintf("%d:secret-url", idx+1)
-	if *showRemoteWriteURL {
-		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
-	}
-	opts := &streamaggr.Options{
+	opts := streamaggr.Options{
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
-		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
+		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
 	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
+	if len(*streamAggrConfig) == 0 {
+		return "", &opts
 	}
-	return sas, nil
+	return streamAggrConfig.GetOptionalArg(idx), &opts
+}
+
+func newStreamAggrConfigWithOpts(pushFunc streamaggr.PushFunc, path string, opts *streamaggr.Options) (*streamaggr.Aggregators, error) {
+	if len(path) == 0 {
+		// Skip empty stream aggregation config.
+		return nil, nil
+	}
+	return streamaggr.LoadFromFile(path, pushFunc, opts)
+}
+
+func newStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
+	path, opts := getStreamAggrOpts(idx)
+	return newStreamAggrConfigWithOpts(pushFunc, path, opts)
 }

app/vmagent/statsd/request_handler.go

@@ -0,0 +1,72 @@
+package statsd
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd/stream"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="statsd"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="statsd"}`)
+)
+
+// InsertHandler processes remote write for statsd plaintext protocol.
+//
+// See https://github.com/statsd/statsd/blob/master/docs/metric_types.md
+func InsertHandler(r io.Reader) error {
+	return stream.Parse(r, false, func(rows []parser.Row) error {
+		return insertRows(nil, rows)
+	})
+}
+
+func insertRows(at *auth.Token, rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samplesLen := len(samples)
+		for _, v := range r.Values {
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     v,
+				Timestamp: r.Timestamp,
+			})
+		}
+
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmalert-tool/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image AS certs
+FROM $certs_image as certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmalert/Makefile

@@ -101,7 +101,8 @@ replay-vmalert: vmalert
 		-remoteWrite.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \
 		-external.label=replica=a \
-		-replay.timeFrom=2024-06-01T00:00:00Z
+		-replay.timeFrom=2021-05-11T07:21:43Z \
+		-replay.timeTo=2021-05-29T18:40:43Z
 
 vmalert-linux-amd64:
 	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch

app/vmalert/config/config.go

@@ -45,11 +45,11 @@ type Group struct {
 	// EvalAlignment will make the timestamp of group query requests be aligned with interval
 	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
+func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type group Group
 	if err := unmarshal((*group)(g)); err != nil {
 		return err
@@ -142,11 +142,11 @@ type Rule struct {
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (r *Rule) UnmarshalYAML(unmarshal func(any) error) error {
+func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rule Rule
 	if err := unmarshal((*rule)(r)); err != nil {
 		return err
@@ -234,7 +234,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 	cLogger.Suppress(true)
 	defer cLogger.Suppress(false)
 
-	files, err := ReadFromFS(pathPatterns)
+	files, err := readFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
@@ -243,7 +243,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := ReadFromFS(pathPatterns)
+	files, err := readFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
@@ -301,7 +301,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.
-		XXX map[string]any `yaml:",inline"`
+		XXX map[string]interface{} `yaml:",inline"`
 	}{}
 	err = yaml.Unmarshal(data, &g)
 	if err != nil {
@@ -310,7 +310,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	return g.Groups, checkOverflow(g.XXX, "config")
 }
 
-func checkOverflow(m map[string]any, ctx string) error {
+func checkOverflow(m map[string]interface{}, ctx string) error {
 	if len(m) > 0 {
 		var keys []string
 		for k := range m {

app/vmalert/config/config_test.go

@@ -23,6 +23,12 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
 func TestParseFromURL(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/bad", func(w http.ResponseWriter, _ *http.Request) {
@@ -49,353 +55,438 @@ groups:
 	defer srv.Close()
 
 	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
-		t.Fatalf("error parsing URLs %s", err)
+		t.Errorf("error parsing URLs %s", err)
 	}
 
 	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
-		t.Fatalf("expected parsing error: %s", err)
+		t.Errorf("expected parsing error: %s", err)
 	}
 }
 
-func TestParse_Success(t *testing.T) {
-	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
-	if err != nil {
-		t.Fatalf("error parsing files %s", err)
+func TestParseBad(t *testing.T) {
+	testCases := []struct {
+		path   []string
+		expErr string
+	}{
+		{
+			[]string{"testdata/rules/rules_interval_bad.rules"},
+			"eval_offset should be smaller than interval",
+		},
+		{
+			[]string{"testdata/rules/rules0-bad.rules"},
+			"unexpected token",
+		},
+		{
+			[]string{"testdata/dir/rules0-bad.rules"},
+			"error parsing annotation",
+		},
+		{
+			[]string{"testdata/dir/rules1-bad.rules"},
+			"duplicate in file",
+		},
+		{
+			[]string{"testdata/dir/rules2-bad.rules"},
+			"function \"unknown\" not defined",
+		},
+		{
+			[]string{"testdata/dir/rules3-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/dir/rules4-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/rules/rules1-bad.rules"},
+			"bad graphite expr",
+		},
+		{
+			[]string{"testdata/dir/rules6-bad.rules"},
+			"missing ':' in header",
+		},
+		{
+			[]string{"http://unreachable-url"},
+			"failed to",
+		},
 	}
-}
-
-func TestParse_Failure(t *testing.T) {
-	f := func(paths []string, errStrExpected string) {
-		t.Helper()
-
-		_, err := Parse(paths, notifier.ValidateTemplates, true)
+	for _, tc := range testCases {
+		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
 		if err == nil {
-			t.Fatalf("expected to get error")
+			t.Errorf("expected to get error")
+			return
 		}
-		if !strings.Contains(err.Error(), errStrExpected) {
-			t.Fatalf("expected err to contain %q; got %q instead", errStrExpected, err)
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
 		}
 	}
-
-	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
-	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
-	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
-	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
-	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
-	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
-	f([]string{"http://unreachable-url"}, "failed to")
 }
 
-func TestRuleValidate(t *testing.T) {
+func TestRule_Validate(t *testing.T) {
 	if err := (&Rule{}).Validate(); err == nil {
-		t.Fatalf("expected empty name error")
+		t.Errorf("expected empty name error")
 	}
 	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
-		t.Fatalf("expected empty expr error")
+		t.Errorf("expected empty expr error")
 	}
 	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
-		t.Fatalf("expected valid rule; got %s", err)
+		t.Errorf("expected valid rule; got %s", err)
 	}
 }
 
-func TestGroupValidate_Failure(t *testing.T) {
-	f := func(group *Group, validateExpressions bool, errStrExpected string) {
-		t.Helper()
-
-		err := group.Validate(nil, validateExpressions)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		errStr := err.Error()
-		if !strings.Contains(errStr, errStrExpected) {
-			t.Fatalf("missing %q in the returned error %q", errStrExpected, errStr)
-		}
-	}
-
-	f(&Group{}, false, "group name must be set")
-
-	f(&Group{
-		Name:     "negative interval",
-		Interval: promutils.NewDuration(-1),
-	}, false, "interval shouldn't be lower than 0")
-
-	f(&Group{
-		Name:       "wrong eval_offset",
-		Interval:   promutils.NewDuration(time.Minute),
-		EvalOffset: promutils.NewDuration(2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
-
-	f(&Group{
-		Name:  "wrong limit",
-		Limit: -1,
-	}, false, "invalid limit")
-
-	f(&Group{
-		Name:        "wrong concurrency",
-		Concurrency: -1,
-	}, false, "invalid concurrency")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
+func TestGroup_Validate(t *testing.T) {
+	testCases := []struct {
+		group               *Group
+		rules               []Rule
+		validateAnnotations bool
+		validateExpressions bool
+		expErr              string
+	}{
+		{
+			group:  &Group{},
+			expErr: "group name must be set",
+		},
+		{
+			group: &Group{
+				Name:     "negative interval",
+				Interval: promutils.NewDuration(-1),
 			},
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
+			expErr: "interval shouldn't be lower than 0",
+		},
+		{
+			group: &Group{
+				Name:       "wrong eval_offset",
+				Interval:   promutils.NewDuration(time.Minute),
+				EvalOffset: promutils.NewDuration(2 * time.Minute),
 			},
+			expErr: "eval_offset should be smaller than interval",
 		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test graphite prometheus bad expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
+		{
+			group: &Group{
+				Name:  "wrong limit",
+				Limit: -1,
 			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
+			expErr: "invalid limit",
+		},
+		{
+			group: &Group{
+				Name:        "wrong concurrency",
+				Concurrency: -1,
 			},
+			expErr: "invalid concurrency",
 		},
-	}, false, "invalid rule")
-
-	f(&Group{
-		Name: "test graphite inherit",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sum(up == 0 ) by (host)",
-			},
-		},
-	}, false, "either `record` or `alert` must be set")
-
-	// validate expressions
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "invalid expression")
-
-	f(&Group{
-		Name: "test thanos",
-		Type: NewRawType("thanos"),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, true, "unknown datasource type")
-
-	f(&Group{
-		Name: "test graphite",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "some-description",
-			}},
-		},
-	}, true, "bad graphite expr")
-}
-
-func TestGroupValidate_Success(t *testing.T) {
-	f := func(group *Group, validateAnnotations, validateExpressions bool) {
-		t.Helper()
-
-		var validateTplFn ValidateTplFn
-		if validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
-		}
-		err := group.Validate(validateTplFn, validateExpressions)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, false, false)
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": "{{ value|query }}",
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
 				},
 			},
+			expErr: "",
 		},
-	}, false, false)
-
-	// validate annotiations
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": `
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr:              "invalid expression",
+			validateExpressions: true,
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": `
 {{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
   {{ . | first | value | humanize1024 }}B
 {{ end }}`,
+						},
+					},
+				},
+			},
+			validateAnnotations: true,
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test thanos",
+				Type: NewRawType("thanos"),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "unknown datasource type",
+		},
+		{
+			group: &Group{
+				Name: "test graphite",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "some-description",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{
+				Name: "test prometheus",
+				Type: NewPrometheusType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{
+				Name: "test graphite inherit",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sum(up == 0 ) by (host)",
+					},
 				},
 			},
 		},
-	}, true, false)
-
-	// validate expressions
-	f(&Group{
-		Name: "test prometheus",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
+		{
+			group: &Group{
+				Name: "test graphite prometheus bad expr",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sum(up == 0 ) by (host)",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+					},
+				},
+			},
+			expErr: "invalid rule",
 		},
-	}, false, true)
-}
-
-func TestHashRule_NotEqual(t *testing.T) {
-	f := func(a, b Rule) {
-		t.Helper()
-
-		aID, bID := HashRule(a), HashRule(b)
-		if aID == bID {
-			t.Fatalf("rule hashes mustn't be equal; got %d", aID)
-		}
 	}
 
-	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
-
-	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 2"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-		"foo": "baz",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1"})
-}
-
-func TestHashRule_Equal(t *testing.T) {
-	f := func(a, b Rule) {
-		t.Helper()
-
-		aID, bID := HashRule(a), HashRule(b)
-		if aID != bID {
-			t.Fatalf("rule hashes must be equal; got %d and %d", aID, bID)
+	for _, tc := range testCases {
+		var validateTplFn ValidateTplFn
+		if tc.validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
+		if err == nil {
+			if tc.expErr != "" {
+				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
+			}
+			continue
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
 		}
 	}
+}
 
-	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1"}, Rule{Alert: "alert", Expr: "up == 1"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-		"foo": "bar",
-	}})
-
-	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Alert: "record", Expr: "up == 1"})
-
-	f(Rule{
-		Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute),
-	}, Rule{Alert: "alert", Expr: "up == 1"})
+func TestHashRule(t *testing.T) {
+	testCases := []struct {
+		a, b  Rule
+		equal bool
+	}{
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1"},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "bar",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Alert: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			false,
+		},
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 2"},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "baz",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			false,
+		},
+	}
+	for i, tc := range testCases {
+		aID, bID := HashRule(tc.a), HashRule(tc.b)
+		if tc.equal != (aID == bID) {
+			t.Fatalf("missmatch for rule %d", i)
+		}
+	}
 }
 
 func TestGroupChecksum(t *testing.T) {

app/vmalert/config/fs.go

@@ -32,14 +32,14 @@ var (
 	fsRegistry   = make(map[string]FS)
 )
 
-// ReadFromFS parses the given path list and inits FS for each item.
-// Once initialed, ReadFromFS will try to read and return files from each FS.
-// ReadFromFS returns an error if at least one FS failed to init.
+// readFromFS parses the given path list and inits FS for each item.
+// Once initialed, readFromFS will try to read and return files from each FS.
+// readFromFS returns an error if at least one FS failed to init.
 // The function can be called multiple times but each unique path
 // will be initialed only once.
 //
 // It is allowed to mix different FS types in path list.
-func ReadFromFS(paths []string) (map[string][]byte, error) {
+func readFromFS(paths []string) (map[string][]byte, error) {
 	var err error
 	result := make(map[string][]byte)
 	for _, path := range paths {

app/vmalert/config/log/logger.go

@@ -29,7 +29,7 @@ func (l *Logger) isDisabled() bool {
 }
 
 // Errorf logs error message.
-func (l *Logger) Errorf(format string, args ...any) {
+func (l *Logger) Errorf(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -37,7 +37,7 @@ func (l *Logger) Errorf(format string, args ...any) {
 }
 
 // Warnf logs warning message.
-func (l *Logger) Warnf(format string, args ...any) {
+func (l *Logger) Warnf(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -45,7 +45,7 @@ func (l *Logger) Warnf(format string, args ...any) {
 }
 
 // Infof logs info message.
-func (l *Logger) Infof(format string, args ...any) {
+func (l *Logger) Infof(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -54,6 +54,6 @@ func (l *Logger) Infof(format string, args ...any) {
 
 // Panicf logs panic message and panics.
 // Panicf can't be suppressed
-func (l *Logger) Panicf(format string, args ...any) {
+func (l *Logger) Panicf(format string, args ...interface{}) {
 	logger.Panicf(format, args...)
 }

app/vmalert/config/log/logger_test.go

@@ -18,14 +18,14 @@ func TestOutput(t *testing.T) {
 
 	mustMatch := func(exp string) {
 		t.Helper()
-
 		if exp == "" {
 			if testOutput.String() != "" {
-				t.Fatalf("expected output to be empty; got %q", testOutput.String())
+				t.Errorf("expected output to be empty; got %q", testOutput.String())
+				return
 			}
 		}
 		if !strings.Contains(testOutput.String(), exp) {
-			t.Fatalf("output %q should contain %q", testOutput.String(), exp)
+			t.Errorf("output %q should contain %q", testOutput.String(), exp)
 		}
 		fmt.Println(testOutput.String())
 		testOutput.Reset()

app/vmalert/config/types.go

@@ -69,7 +69,7 @@ func (t *Type) ValidateExpr(expr string) error {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
+func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err
@@ -87,7 +87,7 @@ func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
 }
 
 // MarshalYAML implements the yaml.Unmarshaler interface.
-func (t Type) MarshalYAML() (any, error) {
+func (t Type) MarshalYAML() (interface{}, error) {
 	return t.Name, nil
 }
 
@@ -98,7 +98,7 @@ type Header struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (h *Header) UnmarshalYAML(unmarshal func(any) error) error {
+func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err

app/vmalert/datasource/init.go

@@ -12,12 +12,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
-		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
 		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
@@ -99,9 +98,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
-	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
+	tr.DialContext = httputils.GetStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {

app/vmalert/datasource/vm_prom_api.go

@@ -119,7 +119,7 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 type promRange struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TVs    [][2]any          `json:"values"`
+		TVs    [][2]interface{}  `json:"values"`
 	} `json:"result"`
 }
 
@@ -147,7 +147,7 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
-type promScalar [2]any
+type promScalar [2]interface{}
 
 func (r promScalar) metrics() ([]Metric, error) {
 	var m Metric

app/vmalert/datasource/vm_prom_api_timing_test.go

@@ -13,7 +13,7 @@ func BenchmarkMetrics(b *testing.B) {
 
 	var pi promInstant
 	if err := pi.Unmarshal(payload); err != nil {
-		b.Fatal(err.Error())
+		b.Fatalf(err.Error())
 	}
 	b.Run("Instant", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {

app/vmalert/datasource/vm_test.go

@@ -31,26 +31,26 @@ var (
 func TestVMInstantQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Fatalf("expected POST method got %s", r.Method)
+			t.Errorf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		timeParam := r.URL.Query().Get("time")
 		if timeParam == "" {
-			t.Fatalf("expected 'time' in query param, got nil instead")
+			t.Errorf("expected 'time' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, timeParam); err != nil {
-			t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
+			t.Errorf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
 		case 0:
@@ -197,13 +197,13 @@ func TestVMInstantQuery(t *testing.T) {
 func TestVMInstantQueryWithRetry(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.URL.Query().Get("query") != query {
-			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		switch c {
 		case 0:
@@ -289,37 +289,37 @@ func metricsEqual(t *testing.T, gotM, expectedM []Metric) {
 func TestVMRangeQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query_range", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Fatalf("expected POST method got %s", r.Method)
+			t.Errorf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		startTS := r.URL.Query().Get("start")
 		if startTS == "" {
-			t.Fatalf("expected 'start' in query param, got nil instead")
+			t.Errorf("expected 'start' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, startTS); err != nil {
-			t.Fatalf("failed to parse 'start' query param: %s", err)
+			t.Errorf("failed to parse 'start' query param: %s", err)
 		}
 		endTS := r.URL.Query().Get("end")
 		if endTS == "" {
-			t.Fatalf("expected 'end' in query param, got nil instead")
+			t.Errorf("expected 'end' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, endTS); err != nil {
-			t.Fatalf("failed to parse 'end' query param: %s", err)
+			t.Errorf("failed to parse 'end' query param: %s", err)
 		}
 		step := r.URL.Query().Get("step")
 		if step != "15s" {
-			t.Fatalf("expected 'step' query param to be 15s; got %q instead", step)
+			t.Errorf("expected 'step' query param to be 15s; got %q instead", step)
 		}
 		switch c {
 		case 0:
@@ -370,299 +370,368 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
-	query := "up"
-	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
-
-	f := func(isQueryRange bool, vm *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
-		t.Helper()
-
-		req, err := vm.newRequest(ctx)
-		if err != nil {
-			t.Fatalf("error in newRequest: %s", err)
-		}
-
-		switch vm.dataSourceType {
-		case "", datasourcePrometheus:
-			if isQueryRange {
-				vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
-			} else {
-				vm.setPrometheusInstantReqParams(req, query, timestamp)
-			}
-		case datasourceGraphite:
-			vm.setGraphiteReqParams(req, query)
-		}
-
-		checkFn(t, req)
-	}
-
 	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+		t.Fatalf("unexpected: %s", err)
 	}
+	query := "up"
+	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
 	storage := VMStorage{
 		extraParams: url.Values{"round_digits": {"10"}},
 	}
-
-	// prometheus path
-	f(false, &VMStorage{
-		dataSourceType: datasourcePrometheus,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/api/v1/query", r.URL.Path)
-	})
-
-	// prometheus prefix
-	f(false, &VMStorage{
-		dataSourceType:   datasourcePrometheus,
-		appendTypePrefix: true,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
-	})
-
-	// prometheus range path
-	f(true, &VMStorage{
-		dataSourceType: datasourcePrometheus,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/api/v1/query_range", r.URL.Path)
-	})
-
-	// prometheus range prefix
-	f(true, &VMStorage{
-		dataSourceType:   datasourcePrometheus,
-		appendTypePrefix: true,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
-	})
-
-	// graphite path
-	f(false, &VMStorage{
-		dataSourceType: datasourceGraphite,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, graphitePath, r.URL.Path)
-	})
-
-	// graphite prefix
-	f(false, &VMStorage{
-		dataSourceType:   datasourceGraphite,
-		appendTypePrefix: true,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
-	})
-
-	// default params
-	f(false, &VMStorage{}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// default range params
-	f(true, &VMStorage{}, func(t *testing.T, r *http.Request) {
-		ts := timestamp.Format(time.RFC3339)
-		exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// basic auth
-	f(false, &VMStorage{
-		authCfg: authCfg,
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "foo", u)
-		checkEqualString(t, "bar", p)
-	})
-
-	// basic auth range
-	f(true, &VMStorage{
-		authCfg: authCfg,
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "foo", u)
-		checkEqualString(t, "bar", p)
-	})
-
-	// evaluation interval
-	f(false, &VMStorage{
-		evaluationInterval: 15 * time.Second,
-	}, func(t *testing.T, r *http.Request) {
-		evalInterval := 15 * time.Second
-		exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// step override
-	f(false, &VMStorage{
-		queryStep: time.Minute,
-	}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{
-			"query": {query},
-			"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
-			"time":  {timestamp.Format(time.RFC3339)},
-		}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	//  step to seconds
-	f(false, &VMStorage{
-		evaluationInterval: 3 * time.Hour,
-	}, func(t *testing.T, r *http.Request) {
-		evalInterval := 3 * time.Hour
-		exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// prometheus extra params
-	f(false, &VMStorage{
-		extraParams: url.Values{"round_digits": {"10"}},
-	}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// prometheus extra params range
-	f(true, &VMStorage{
-		extraParams: url.Values{
-			"nocache":      {"1"},
-			"max_lookback": {"1h"},
+	testCases := []struct {
+		name       string
+		queryRange bool
+		vm         *VMStorage
+		checkFn    func(t *testing.T, r *http.Request)
+	}{
+		{
+			"prometheus path",
+			false,
+			&VMStorage{
+				dataSourceType: datasourcePrometheus,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/api/v1/query", r.URL.Path)
+			},
 		},
-	}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{
-			"query":        {query},
-			"end":          {timestamp.Format(time.RFC3339)},
-			"start":        {timestamp.Format(time.RFC3339)},
-			"nocache":      {"1"},
-			"max_lookback": {"1h"},
-		}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// custom params overrides the original params
-	f(false, storage.Clone().ApplyParams(QuerierParams{
-		QueryParams: url.Values{"round_digits": {"2"}},
-	}), func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// allow duplicates in query params
-	f(false, storage.Clone().ApplyParams(QuerierParams{
-		QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
-	}), func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// graphite extra params
-	f(false, &VMStorage{
-		dataSourceType: datasourceGraphite,
-		extraParams: url.Values{
-			"nocache":      {"1"},
-			"max_lookback": {"1h"},
+		{
+			"prometheus prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   datasourcePrometheus,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
+			},
 		},
-	}, func(t *testing.T, r *http.Request) {
-		exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
-		checkEqualString(t, exp, r.URL.RawQuery)
-	})
-
-	// graphite extra params allows to override from
-	f(false, &VMStorage{
-		dataSourceType: datasourceGraphite,
-		extraParams: url.Values{
-			"from": {"-10m"},
+		{
+			"prometheus range path",
+			true,
+			&VMStorage{
+				dataSourceType: datasourcePrometheus,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
+			},
 		},
-	}, func(t *testing.T, r *http.Request) {
-		exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
-		checkEqualString(t, exp, r.URL.RawQuery)
-	})
+		{
+			"prometheus range prefix",
+			true,
+			&VMStorage{
+				dataSourceType:   datasourcePrometheus,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
+			},
+		},
+		{
+			"graphite path",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"graphite prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   datasourceGraphite,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"default params",
+			false,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"default range params",
+			true,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				ts := timestamp.Format(time.RFC3339)
+				exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"basic auth",
+			false,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"basic auth range",
+			true,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"evaluation interval",
+			false,
+			&VMStorage{
+				evaluationInterval: 15 * time.Second,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 15 * time.Second
+				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"step override",
+			false,
+			&VMStorage{
+				queryStep: time.Minute,
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{
+					"query": {query},
+					"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
+					"time":  {timestamp.Format(time.RFC3339)},
+				}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"step to seconds",
+			false,
+			&VMStorage{
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params range",
+			true,
+			&VMStorage{
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{
+					"query":        {query},
+					"end":          {timestamp.Format(time.RFC3339)},
+					"start":        {timestamp.Format(time.RFC3339)},
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"custom params overrides the original params",
+			false,
+			storage.Clone().ApplyParams(QuerierParams{
+				QueryParams: url.Values{"round_digits": {"2"}},
+			}),
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"allow duplicates in query params",
+			false,
+			storage.Clone().ApplyParams(QuerierParams{
+				QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
+			}),
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"graphite extra params",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"graphite extra params allows to override from",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+				extraParams: url.Values{
+					"from": {"-10m"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := tc.vm.newRequest(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			switch tc.vm.dataSourceType {
+			case "", datasourcePrometheus:
+				if tc.queryRange {
+					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
+				} else {
+					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
+				}
+			case datasourceGraphite:
+				tc.vm.setGraphiteReqParams(req, query)
+			}
+			tc.checkFn(t, req)
+		})
+	}
 }
 
 func TestHeaders(t *testing.T) {
-	f := func(vmFn func() *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
-		t.Helper()
-
-		vm := vmFn()
-		req, err := vm.newQueryRequest(ctx, "foo", time.Now())
-		if err != nil {
-			t.Fatalf("error in newQueryRequest: %s", err)
-		}
-		checkFn(t, req)
-	}
-
-	// basic auth
-	f(func() *VMStorage {
-		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-		if err != nil {
-			t.Fatalf("Error get auth config: %s", err)
-		}
-		return &VMStorage{authCfg: cfg}
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "foo", u)
-		checkEqualString(t, "bar", p)
-	})
-
-	// bearer auth
-	f(func() *VMStorage {
-		cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
-		if err != nil {
-			t.Fatalf("Error get auth config: %s", err)
-		}
-		return &VMStorage{authCfg: cfg}
-	}, func(t *testing.T, r *http.Request) {
-		reqToken := r.Header.Get("Authorization")
-		splitToken := strings.Split(reqToken, "Bearer ")
-		if len(splitToken) != 2 {
-			t.Fatalf("expected two items got %d", len(splitToken))
-		}
-		token := splitToken[1]
-		checkEqualString(t, "foo", token)
-	})
-
-	// custom extraHeaders
-	f(func() *VMStorage {
-		return &VMStorage{extraHeaders: []keyValue{
-			{key: "Foo", value: "bar"},
-			{key: "Baz", value: "qux"},
-		}}
-	}, func(t *testing.T, r *http.Request) {
-		h1 := r.Header.Get("Foo")
-		checkEqualString(t, "bar", h1)
-		h2 := r.Header.Get("Baz")
-		checkEqualString(t, "qux", h2)
-	})
-
-	// custom header overrides basic auth
-	f(func() *VMStorage {
-		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-		if err != nil {
-			t.Fatalf("Error get auth config: %s", err)
-		}
-		return &VMStorage{
-			authCfg: cfg,
-			extraHeaders: []keyValue{
-				{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+	testCases := []struct {
+		name    string
+		vmFn    func() *VMStorage
+		checkFn func(t *testing.T, r *http.Request)
+	}{
+		{
+			name: "basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
 			},
-		}
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "Aladdin", u)
-		checkEqualString(t, "open sesame", p)
-	})
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			name: "bearer auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				reqToken := r.Header.Get("Authorization")
+				splitToken := strings.Split(reqToken, "Bearer ")
+				if len(splitToken) != 2 {
+					t.Errorf("expected two items got %d", len(splitToken))
+				}
+				token := splitToken[1]
+				checkEqualString(t, "foo", token)
+			},
+		},
+		{
+			name: "custom extraHeaders",
+			vmFn: func() *VMStorage {
+				return &VMStorage{extraHeaders: []keyValue{
+					{key: "Foo", value: "bar"},
+					{key: "Baz", value: "qux"},
+				}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				h1 := r.Header.Get("Foo")
+				checkEqualString(t, "bar", h1)
+				h2 := r.Header.Get("Baz")
+				checkEqualString(t, "qux", h2)
+			},
+		},
+		{
+			name: "custom header overrides basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{
+					authCfg: cfg,
+					extraHeaders: []keyValue{
+						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+					},
+				}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "Aladdin", u)
+				checkEqualString(t, "open sesame", p)
+			},
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			vm := tt.vmFn()
+			req, err := vm.newQueryRequest(ctx, "foo", time.Now())
+			if err != nil {
+				t.Fatal(err)
+			}
+			tt.checkFn(t, req)
+		})
+	}
 }
 
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
-
 	if got != exp {
-		t.Fatalf("expected to get: \n%q; \ngot: \n%q", exp, got)
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }
 
 func expectError(t *testing.T, err error, exp string) {
 	t.Helper()
-
 	if err == nil {
-		t.Fatalf("expected non-nil error")
+		t.Errorf("expected non-nil error")
 	}
 	if !strings.Contains(err.Error(), exp) {
-		t.Fatalf("expected error %q to contain %q", err, exp)
+		t.Errorf("expected error %q to contain %q", err, exp)
 	}
 }

app/vmalert/main.go

@@ -72,7 +72,7 @@ absolute path to all .tpl files in root.
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
 		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
 		`Supports templating - see https://docs.victoriametrics.com/vmalert/#templating . `+
-		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
+		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
 		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
@@ -132,7 +132,7 @@ func main() {
 		validateTplFn = notifier.ValidateTemplates
 	}
 
-	if *replayFrom != "" {
+	if *replayFrom != "" || *replayTo != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
 			logger.Fatalf("failed to init remoteWrite: %s", err)
@@ -153,7 +153,6 @@ func main() {
 		if err := replay(groupsCfg, q, rw); err != nil {
 			logger.Fatalf("replay failed: %s", err)
 		}
-		logger.Infof("replay succeed!")
 		return
 	}
 

app/vmalert/main_test.go

@@ -25,26 +25,26 @@ func TestGetExternalURL(t *testing.T) {
 	invalidURL := "victoriametrics.com/path"
 	_, err := getExternalURL(invalidURL)
 	if err == nil {
-		t.Fatalf("expected error, got nil")
+		t.Errorf("expected error, got nil")
 	}
 
 	expURL := "https://victoriametrics.com/path"
 	u, err := getExternalURL(expURL)
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Fatalf("unexpected url: want %q, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %q, got %s", expURL, u.String())
 	}
 
 	h, _ := os.Hostname()
 	expURL = fmt.Sprintf("http://%s:8880", h)
 	u, err = getExternalURL("")
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Fatalf("unexpected url: want %s, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %s, got %s", expURL, u.String())
 	}
 }
 
@@ -53,22 +53,22 @@ func TestGetAlertURLGenerator(t *testing.T) {
 	u, _ := url.Parse("https://victoriametrics.com/path")
 	fn, err := getAlertURLGenerator(u, "", false)
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
 	if exp != fn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
 	if err == nil {
-		t.Fatalf("expected template validation error got nil")
+		t.Errorf("expected template validation error got nil")
 	}
 	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
 

app/vmalert/manager_test.go

@@ -82,8 +82,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	wg.Wait()
 }
 
-// TestManagerUpdate tests sequential configuration updates.
-func TestManagerUpdate_Success(t *testing.T) {
+// TestManagerUpdate tests sequential configuration
+// updates.
+func TestManagerUpdate(t *testing.T) {
 	const defaultEvalInterval = time.Second * 30
 	currentEvalInterval := *evaluationInterval
 	*evaluationInterval = defaultEvalInterval
@@ -119,127 +120,145 @@ func TestManagerUpdate_Success(t *testing.T) {
 		}
 	)
 
-	f := func(initPath, updatePath string, groupsExpected []*rule.Group) {
-		t.Helper()
-
-		ctx, cancel := context.WithCancel(context.TODO())
-		m := &manager{
-			groups:         make(map[uint64]*rule.Group),
-			querierBuilder: &datasource.FakeQuerier{},
-			notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
-		}
-
-		cfgInit := loadCfg(t, []string{initPath}, true, true)
-		if err := m.update(ctx, cfgInit, false); err != nil {
-			t.Fatalf("failed to complete initial rules update: %s", err)
-		}
-
-		cfgUpdate, err := config.Parse([]string{updatePath}, notifier.ValidateTemplates, true)
-		if err == nil { // update can fail and that's expected
-			_ = m.update(ctx, cfgUpdate, false)
-		}
-		if len(groupsExpected) != len(m.groups) {
-			t.Fatalf("unexpected number of groups; got %d; want %d", len(m.groups), len(groupsExpected))
-		}
-
-		for _, wantG := range groupsExpected {
-			gotG, ok := m.groups[wantG.ID()]
-			if !ok {
-				t.Fatalf("expected to have group %q", wantG.Name)
-			}
-			compareGroups(t, wantG, gotG)
-		}
-
-		cancel()
-		m.close()
-	}
-
-	// update good rules
-	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules1-good.rules", []*rule.Group{
+	testCases := []struct {
+		name       string
+		initPath   string
+		updatePath string
+		want       []*rule.Group
+	}{
 		{
-			File:     "config/testdata/dir/rules1-good.rules",
-			Name:     "duplicatedGroupDiffFiles",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules: []rule.Rule{
-				&rule.AlertingRule{
-					Name:   "VMRows",
-					Expr:   "vm_rows > 0",
-					For:    5 * time.Minute,
-					Labels: map[string]string{"dc": "gcp", "label": "bar"},
-					Annotations: map[string]string{
-						"summary":     "{{ $value }}",
-						"description": "{{$labels}}",
+			name:       "update good rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules1-good.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/dir/rules1-good.rules",
+					Name:     "duplicatedGroupDiffFiles",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules: []rule.Rule{
+						&rule.AlertingRule{
+							Name:   "VMRows",
+							Expr:   "vm_rows > 0",
+							For:    5 * time.Minute,
+							Labels: map[string]string{"dc": "gcp", "label": "bar"},
+							Annotations: map[string]string{
+								"summary":     "{{ $value }}",
+								"description": "{{$labels}}",
+							},
+						},
 					},
 				},
 			},
 		},
-	})
-
-	// update good rules from 1 to 2 groups
-	f("config/testdata/dir/rules/rules1-good.rules", "config/testdata/rules/rules0-good.rules", []*rule.Group{
 		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Name:     "groupGorSingleAlert",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules:    []rule.Rule{VMRows},
-		},
-		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Interval: defaultEvalInterval,
-			Type:     config.NewPrometheusType(),
-			Name:     "TestGroup",
-			Rules: []rule.Rule{
-				Conns,
-				ExampleAlertAlwaysFiring,
+			name:       "update good rules from 1 to 2 groups",
+			initPath:   "config/testdata/dir/rules/rules1-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules:    []rule.Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Type:     config.NewPrometheusType(),
+					Name:     "TestGroup",
+					Rules: []rule.Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
 			},
 		},
-	})
-
-	// update with one bad rule file
-	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules2-bad.rules", []*rule.Group{
 		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Name:     "groupGorSingleAlert",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules:    []rule.Rule{VMRows},
-		},
-		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Interval: defaultEvalInterval,
-			Name:     "TestGroup",
-			Type:     config.NewPrometheusType(),
-			Rules: []rule.Rule{
-				Conns,
-				ExampleAlertAlwaysFiring,
+			name:       "update with one bad rule file",
+			initPath:   "config/testdata/rules/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules2-bad.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules:    []rule.Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup",
+					Type:     config.NewPrometheusType(),
+					Rules: []rule.Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
 			},
 		},
-	})
-
-	// update empty dir rules from 0 to 2 groups
-	f("config/testdata/empty/*", "config/testdata/rules/rules0-good.rules", []*rule.Group{
 		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Name:     "groupGorSingleAlert",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules:    []rule.Rule{VMRows},
-		},
-		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Interval: defaultEvalInterval,
-			Type:     config.NewPrometheusType(),
-			Name:     "TestGroup",
-			Rules: []rule.Rule{
-				Conns,
-				ExampleAlertAlwaysFiring,
+			name:       "update empty dir rules from 0 to 2 groups",
+			initPath:   "config/testdata/empty/*",
+			updatePath: "config/testdata/rules/rules0-good.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules:    []rule.Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Type:     config.NewPrometheusType(),
+					Name:     "TestGroup",
+					Rules: []rule.Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
 			},
 		},
-	})
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+			m := &manager{
+				groups:         make(map[uint64]*rule.Group),
+				querierBuilder: &datasource.FakeQuerier{},
+				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
+			}
+
+			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
+			if err := m.update(ctx, cfgInit, false); err != nil {
+				t.Fatalf("failed to complete initial rules update: %s", err)
+			}
+
+			cfgUpdate, err := config.Parse([]string{tc.updatePath}, notifier.ValidateTemplates, true)
+			if err == nil { // update can fail and that's expected
+				_ = m.update(ctx, cfgUpdate, false)
+			}
+			if len(tc.want) != len(m.groups) {
+				t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
+			}
+
+			for _, wantG := range tc.want {
+				gotG, ok := m.groups[wantG.ID()]
+				if !ok {
+					t.Fatalf("expected to have group %q", wantG.Name)
+				}
+				compareGroups(t, wantG, gotG)
+			}
+
+			cancel()
+			m.close()
+		})
+	}
 }
-
 func compareGroups(t *testing.T, a, b *rule.Group) {
 	t.Helper()
 	if a.Name != b.Name {
@@ -266,59 +285,82 @@ func compareGroups(t *testing.T, a, b *rule.Group) {
 	}
 }
 
-func TestManagerUpdate_Failure(t *testing.T) {
-	f := func(notifiers []notifier.Notifier, rw remotewrite.RWClient, cfg config.Group, errStrExpected string) {
-		t.Helper()
-
-		m := &manager{
-			groups:         make(map[uint64]*rule.Group),
-			querierBuilder: &datasource.FakeQuerier{},
-			rw:             rw,
-		}
-		if notifiers != nil {
-			m.notifiers = func() []notifier.Notifier { return notifiers }
-		}
-		err := m.update(context.Background(), []config.Group{cfg}, false)
-		if err == nil {
-			t.Fatalf("expected to get error; got nil")
-		}
-		errStr := err.Error()
-		if !strings.Contains(errStr, errStrExpected) {
-			t.Fatalf("missing %q in the error %q", errStrExpected, errStr)
-		}
+func TestManagerUpdateNegative(t *testing.T) {
+	testCases := []struct {
+		notifiers []notifier.Notifier
+		rw        remotewrite.RWClient
+		cfg       config.Group
+		expErr    string
+	}{
+		{
+			nil,
+			nil,
+			config.Group{
+				Name: "Recording rule only",
+				Rules: []config.Rule{
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			nil,
+			config.Group{
+				Name: "Alerting rule only",
+				Rules: []config.Rule{
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+		{
+			[]notifier.Notifier{&notifier.FakeNotifier{}},
+			nil,
+			config.Group{
+				Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Alert: "alert1", Expr: "up > 0"},
+					{Alert: "alert2", Expr: "up > 0"},
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			&remotewrite.Client{},
+			config.Group{
+				Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Record: "record1", Expr: "max(up)"},
+					{Record: "record2", Expr: "max(up)"},
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
 	}
 
-	f(nil, nil, config.Group{
-		Name: "Recording rule only",
-		Rules: []config.Rule{
-			{Record: "record", Expr: "max(up)"},
-		},
-	}, "contains recording rules")
-
-	f(nil, nil, config.Group{
-		Name: "Alerting rule only",
-		Rules: []config.Rule{
-			{Alert: "alert", Expr: "up > 0"},
-		},
-	}, "contains alerting rules")
-
-	f([]notifier.Notifier{&notifier.FakeNotifier{}}, nil, config.Group{
-		Name: "Recording and alerting rules",
-		Rules: []config.Rule{
-			{Alert: "alert1", Expr: "up > 0"},
-			{Alert: "alert2", Expr: "up > 0"},
-			{Record: "record", Expr: "max(up)"},
-		},
-	}, "contains recording rules")
-
-	f(nil, &remotewrite.Client{}, config.Group{
-		Name: "Recording and alerting rules",
-		Rules: []config.Rule{
-			{Record: "record1", Expr: "max(up)"},
-			{Record: "record2", Expr: "max(up)"},
-			{Alert: "alert", Expr: "up > 0"},
-		},
-	}, "contains alerting rules")
+	for _, tc := range testCases {
+		t.Run(tc.cfg.Name, func(t *testing.T) {
+			m := &manager{
+				groups:         make(map[uint64]*rule.Group),
+				querierBuilder: &datasource.FakeQuerier{},
+				rw:             tc.rw,
+			}
+			if tc.notifiers != nil {
+				m.notifiers = func() []notifier.Notifier { return tc.notifiers }
+			}
+			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
+			if err == nil {
+				t.Fatalf("expected to get error; got nil")
+			}
+			if !strings.Contains(err.Error(), tc.expErr) {
+				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
+			}
+		})
+	}
 }
 
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {

app/vmalert/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image AS certs
+FROM $certs_image as certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmalert/notifier/alert_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-func TestAlertExecTemplate(t *testing.T) {
+func TestAlert_ExecTemplate(t *testing.T) {
 	extLabels := make(map[string]string)
 	const (
 		extCluster = "prod"
@@ -23,164 +23,201 @@ func TestAlertExecTemplate(t *testing.T) {
 	_, err := Init(nil, extLabels, extURL)
 	checkErr(t, err)
 
-	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
-		t.Helper()
-
-		if err := ValidateTemplates(annotations); err != nil {
-			t.Fatalf("cannot validate annotations: %s", err)
-		}
-
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return []datasource.Metric{
-				{
-					Labels: []datasource.Label{
-						{Name: "foo", Value: "bar"},
-						{Name: "baz", Value: "qux"},
-					},
-					Values:     []float64{1},
-					Timestamps: []int64{1},
+	testCases := []struct {
+		name        string
+		alert       *Alert
+		annotations map[string]string
+		expTpl      map[string]string
+	}{
+		{
+			name:        "empty-alert",
+			alert:       &Alert{},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			name: "no-template",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"instance": "localhost",
 				},
-				{
-					Labels: []datasource.Label{
-						{Name: "foo", Value: "garply"},
-						{Name: "baz", Value: "fred"},
-					},
-					Values:     []float64{2},
-					Timestamps: []int64{1},
+			},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			name: "label-template",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
 				},
-			}, nil
-		}
-
-		tpl, err := alert.ExecTemplate(qFn, alert.Labels, annotations)
-		if err != nil {
-			t.Fatalf("cannot execute template: %s", err)
-		}
-		if len(tpl) != len(tplExpected) {
-			t.Fatalf("unexpected number of elements; got %d; want %d", len(tpl), len(tplExpected))
-		}
-		for k := range tplExpected {
-			got, exp := tpl[k], tplExpected[k]
-			if got != exp {
-				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
-			}
-		}
+				For: 5 * time.Minute,
+			},
+			annotations: map[string]string{
+				"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
+			},
+			expTpl: map[string]string{
+				"summary":     "Too high connection number for localhost for job staging",
+				"description": "It is 10000 connections for localhost for more than 5m0s",
+			},
+		},
+		{
+			name: "expression-template",
+			alert: &Alert{
+				Expr: `vm_rows{"label"="bar"}<0`,
+			},
+			annotations: map[string]string{
+				"exprEscapedQuery":  "{{ $expr|queryEscape }}",
+				"exprEscapedPath":   "{{ $expr|pathEscape }}",
+				"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
+				"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
+				"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
+			},
+			expTpl: map[string]string{
+				"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
+				"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
+				"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
+				"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
+				"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
+			},
+		},
+		{
+			name:  "query",
+			alert: &Alert{Expr: `vm_rows{"label"="bar"}>0`},
+			annotations: map[string]string{
+				"summary": `{{ query "foo" | first | value }}`,
+				"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+			},
+			expTpl: map[string]string{
+				"summary": "1",
+				"desc":    "bar 1;garply 2;",
+			},
+		},
+		{
+			name: "external",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"url":         "{{ $externalURL }}",
+				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+			},
+			expTpl: map[string]string{
+				"url":         extURL,
+				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+			},
+		},
+		{
+			name: "alert and group IDs",
+			alert: &Alert{
+				ID:      42,
+				GroupID: 24,
+			},
+			annotations: map[string]string{
+				"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+			},
+			expTpl: map[string]string{
+				"url": "/api/v1/alert?alertID=42&groupID=24",
+			},
+		},
+		{
+			name: "ActiveAt time",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+			},
+			expTpl: map[string]string{
+				"diagram": "![](http://example.com?render=1660941298",
+			},
+		},
+		{
+			name:  "ActiveAt time is nil",
+			alert: &Alert{},
+			annotations: map[string]string{
+				"default_time": "{{$activeAt}}",
+			},
+			expTpl: map[string]string{
+				"default_time": "0001-01-01 00:00:00 +0000 UTC",
+			},
+		},
+		{
+			name: "ActiveAt custom format",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+			},
+			expTpl: map[string]string{
+				"fire_time": "2022/08/19 20:34:58",
+			},
+		},
+		{
+			name: "ActiveAt query range",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+			},
+			expTpl: map[string]string{
+				"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+			},
+		},
 	}
 
-	// empty-alert
-	f(&Alert{}, map[string]string{}, map[string]string{})
-
-	// no-template
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"instance": "localhost",
-		},
-	}, map[string]string{}, map[string]string{})
-
-	// label-template
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-		For: 5 * time.Minute,
-	}, map[string]string{
-		"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
-	}, map[string]string{
-		"summary":     "Too high connection number for localhost for job staging",
-		"description": "It is 10000 connections for localhost for more than 5m0s",
-	})
-
-	// expression-template
-	f(&Alert{
-		Expr: `vm_rows{"label"="bar"}<0`,
-	}, map[string]string{
-		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
-		"exprEscapedPath":   "{{ $expr|pathEscape }}",
-		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
-		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
-		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
-	}, map[string]string{
-		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
-		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
-		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
-		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
-		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
-	})
-
-	// query
-	f(&Alert{
-		Expr: `vm_rows{"label"="bar"}>0`,
-	}, map[string]string{
-		"summary": `{{ query "foo" | first | value }}`,
-		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
-	}, map[string]string{
-		"summary": "1",
-		"desc":    "bar 1;garply 2;",
-	})
-
-	// external
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"url":         "{{ $externalURL }}",
-		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
-	}, map[string]string{
-		"url":         extURL,
-		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
-		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
-	})
-
-	// alert and group IDs
-	f(&Alert{
-		ID:      42,
-		GroupID: 24,
-	}, map[string]string{
-		"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
-	}, map[string]string{
-		"url": "/api/v1/alert?alertID=42&groupID=24",
-	})
-
-	// ActiveAt time
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
-	}, map[string]string{
-		"diagram": "![](http://example.com?render=1660941298",
-	})
-
-	// ActiveAt time is nil
-	f(&Alert{}, map[string]string{
-		"default_time": "{{$activeAt}}",
-	}, map[string]string{
-		"default_time": "0001-01-01 00:00:00 +0000 UTC",
-	})
-
-	// ActiveAt custom format
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
-	}, map[string]string{
-		"fire_time": "2022/08/19 20:34:58",
-	})
-
-	// ActiveAt query range
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
-	}, map[string]string{
-		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
-	})
+	qFn := func(_ string) ([]datasource.Metric, error) {
+		return []datasource.Metric{
+			{
+				Labels: []datasource.Label{
+					{Name: "foo", Value: "bar"},
+					{Name: "baz", Value: "qux"},
+				},
+				Values:     []float64{1},
+				Timestamps: []int64{1},
+			},
+			{
+				Labels: []datasource.Label{
+					{Name: "foo", Value: "garply"},
+					{Name: "baz", Value: "fred"},
+				},
+				Values:     []float64{2},
+				Timestamps: []int64{1},
+			},
+		}, nil
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if err := ValidateTemplates(tc.annotations); err != nil {
+				t.Fatal(err)
+			}
+			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(tpl) != len(tc.expTpl) {
+				t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
+			}
+			for k := range tc.expTpl {
+				got, exp := tpl[k], tc.expTpl[k]
+				if got != exp {
+					t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
+				}
+			}
+		})
+	}
 }
 
 func TestAlert_toPromLabels(t *testing.T) {

app/vmalert/notifier/alertmanager.go

@@ -76,6 +76,9 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
 
 	if am.timeout > 0 {
 		var cancel context.CancelFunc
@@ -91,11 +94,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 			return err
 		}
 	}
-	// external headers have higher priority
-	for key, value := range headers {
-		req.Header.Set(key, value)
-	}
-
 	resp, err := am.client.Do(req)
 	if err != nil {
 		return err
@@ -132,8 +130,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
-
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
 	ba := new(promauth.BasicAuthConfig)
@@ -148,9 +145,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
-		utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
-	)
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/notifier/alertmanager_test.go

@@ -3,7 +3,6 @@ package notifier
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -17,10 +16,10 @@ func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
 	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+		t.Errorf("unexpected error: %s", err)
 	}
 	if am.Addr() != addr {
-		t.Fatalf("expected to have %q; got %q", addr, am.Addr())
+		t.Errorf("expected to have %q; got %q", addr, am.Addr())
 	}
 }
 
@@ -29,29 +28,27 @@ func TestAlertManager_Send(t *testing.T) {
 	const headerKey, headerValue = "TenantID", "foo"
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
 		user, pass, ok := r.BasicAuth()
 		if !ok {
-			t.Fatalf("unauthorized request")
+			t.Errorf("unauthorized request")
 		}
 		if user != baUser || pass != baPass {
-			t.Fatalf("wrong creds %q:%q; expected %q:%q", user, pass, baUser, baPass)
+			t.Errorf("wrong creds %q:%q; expected %q:%q",
+				user, pass, baUser, baPass)
 		}
 		c++
 		if r.Method != http.MethodPost {
-			t.Fatalf("expected POST method got %s", r.Method)
+			t.Errorf("expected POST method got %s", r.Method)
 		}
 		switch c {
 		case 0:
 			conn, _, _ := w.(http.Hijacker).Hijack()
 			_ = conn.Close()
 		case 1:
-			if r.Header.Get(headerKey) != headerValue {
-				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
-			}
 			w.WriteHeader(500)
 		case 2:
 			var a []struct {
@@ -62,26 +59,25 @@ func TestAlertManager_Send(t *testing.T) {
 				GeneratorURL string            `json:"generatorURL"`
 			}
 			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
-				t.Fatalf("can not unmarshal data into alert %s", err)
+				t.Errorf("can not unmarshal data into alert %s", err)
+				t.FailNow()
 			}
 			if len(a) != 1 {
-				t.Fatalf("expected 1 alert in array got %d", len(a))
+				t.Errorf("expected 1 alert in array got %d", len(a))
 			}
 			if a[0].GeneratorURL != "0/0" {
-				t.Fatalf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
+				t.Errorf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
 			}
 			if a[0].StartsAt.IsZero() {
-				t.Fatalf("expected non-zero start time")
+				t.Errorf("expected non-zero start time")
 			}
 			if a[0].EndAt.IsZero() {
-				t.Fatalf("expected non-zero end time")
-			}
-			if r.Header.Get(headerKey) != "bar" {
-				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+				t.Errorf("expected non-zero end time")
 			}
 		case 3:
 			if r.Header.Get(headerKey) != headerValue {
-				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+				t.Errorf("expected header %q to be set to %q; got %q instead",
+					headerKey, headerValue, r.Header.Get(headerKey))
 			}
 		}
 	})
@@ -93,19 +89,18 @@ func TestAlertManager_Send(t *testing.T) {
 			Username: baUser,
 			Password: promauth.NewSecret(baPass),
 		},
-		Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
 	}, aCfg, nil, 0)
 	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+		t.Errorf("unexpected error: %s", err)
 	}
 	if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
-		t.Fatalf("expected connection error got nil")
+		t.Error("expected connection error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
-		t.Fatalf("expected wrong http code error got nil")
+		t.Error("expected wrong http code error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{{
 		GroupID:     0,
@@ -113,13 +108,13 @@ func TestAlertManager_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, map[string]string{headerKey: "bar"}); err != nil {
-		t.Fatalf("unexpected error %s", err)
+	}}, nil); err != nil {
+		t.Errorf("unexpected error %s", err)
 	}
 	if c != 2 {
-		t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
+		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
 	}
 	if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 }

app/vmalert/notifier/config.go

@@ -51,7 +51,7 @@ type Config struct {
 	Checksum string
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 
 	// This is set to the directory from where the config has been loaded.
 	baseDir string
@@ -73,7 +73,7 @@ type StaticConfig struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (cfg *Config) UnmarshalYAML(unmarshal func(any) error) error {
+func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type config Config
 	if err := unmarshal((*config)(cfg)); err != nil {
 		return err

app/vmalert/notifier/config_test.go

@@ -5,14 +5,10 @@ import (
 	"testing"
 )
 
-func TestParseConfig_Success(t *testing.T) {
+func TestConfigParseGood(t *testing.T) {
 	f := func(path string) {
-		t.Helper()
-
 		_, err := parseConfig(path)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
+		checkErr(t, err)
 	}
 	f("testdata/mixed.good.yaml")
 	f("testdata/consul.good.yaml")
@@ -20,16 +16,14 @@ func TestParseConfig_Success(t *testing.T) {
 	f("testdata/static.good.yaml")
 }
 
-func TestParseConfig_Failure(t *testing.T) {
+func TestConfigParseBad(t *testing.T) {
 	f := func(path, expErr string) {
-		t.Helper()
-
 		_, err := parseConfig(path)
 		if err == nil {
 			t.Fatalf("expected to get non-nil err for config %q", path)
 		}
 		if !strings.Contains(err.Error(), expErr) {
-			t.Fatalf("expected err to contain %q; got %q instead", expErr, err)
+			t.Errorf("expected err to contain %q; got %q instead", expErr, err)
 		}
 	}
 

app/vmalert/notifier/config_watcher_test.go

@@ -319,41 +319,46 @@ func TestMergeHTTPClientConfigs(t *testing.T) {
 	}
 }
 
-func TestParseLabels_Failure(t *testing.T) {
-	f := func(target string, cfg *Config) {
-		t.Helper()
-
-		_, _, err := parseLabels(target, nil, cfg)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
+func TestParseLabels(t *testing.T) {
+	testCases := []struct {
+		name            string
+		target          string
+		cfg             *Config
+		expectedAddress string
+		expectedErr     bool
+	}{
+		{
+			"invalid address",
+			"invalid:*//url",
+			&Config{},
+			"",
+			true,
+		},
+		{
+			"use some default params",
+			"alertmanager:9093",
+			&Config{PathPrefix: "test"},
+			"http://alertmanager:9093/test/api/v2/alerts",
+			false,
+		},
+		{
+			"use target address",
+			"https://alertmanager:9093/api/v1/alerts",
+			&Config{Scheme: "http", PathPrefix: "test"},
+			"https://alertmanager:9093/api/v1/alerts",
+			false,
+		},
 	}
 
-	// invalid address
-	f("invalid:*//url", &Config{})
-}
-
-func TestParseLabels_Success(t *testing.T) {
-	f := func(target string, cfg *Config, expectedAddress string) {
-		t.Helper()
-
-		address, _, err := parseLabels(target, nil, cfg)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if address != expectedAddress {
-			t.Fatalf("unexpected address; got %q; want %q", address, expectedAddress)
-		}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			address, _, err := parseLabels(tc.target, nil, tc.cfg)
+			if err == nil == tc.expectedErr {
+				t.Fatalf("unexpected error; got %t; want %t", err != nil, tc.expectedErr)
+			}
+			if address != tc.expectedAddress {
+				t.Fatalf("unexpected address; got %q; want %q", address, tc.expectedAddress)
+			}
+		})
 	}
-
-	// use some default params
-	f("alertmanager:9093", &Config{
-		PathPrefix: "test",
-	}, "http://alertmanager:9093/test/api/v2/alerts")
-
-	// use target address
-	f("https://alertmanager:9093/api/v1/alerts", &Config{
-		Scheme:     "http",
-		PathPrefix: "test",
-	}, "https://alertmanager:9093/api/v1/alerts")
 }

app/vmalert/notifier/init.go

@@ -25,9 +25,6 @@ var (
 		"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
 		"-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.")
 
-	headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+
-		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
-		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
@@ -174,7 +171,6 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
 				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
 			},
-			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
 		addr = strings.TrimSuffix(addr, "/")

app/vmalert/notifier/notifier_blackhole_test.go

@@ -17,12 +17,12 @@ func TestBlackHoleNotifier_Send(t *testing.T) {
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
 	}}, nil); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 
 	alertCount := bh.metrics.alertsSent.Get()
 	if alertCount != 1 {
-		t.Fatalf("expect value 1; instead got %d", alertCount)
+		t.Errorf("expect value 1; instead got %d", alertCount)
 	}
 }
 
@@ -35,7 +35,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
 	}}, nil); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 
 	bh.Close()
@@ -44,7 +44,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 	alertMetricName := "vmalert_alerts_sent_total{addr=\"blackhole\"}"
 	for _, name := range defaultMetrics.ListMetricNames() {
 		if name == alertMetricName {
-			t.Fatalf("Metric name should have unregistered.But still present")
+			t.Errorf("Metric name should have unregistered.But still present")
 		}
 	}
 }

app/vmalert/remoteread/init.go

@@ -10,14 +10,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
 		"Remote read is used to restore alerts state."+
 		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
 		"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
 
 	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
@@ -68,10 +67,10 @@ func Init() (datasource.QuerierBuilder, error) {
 	}
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
-	tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")
+	tr.DialContext = httputils.GetStatDialFunc("vmalert_remoteread")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/remotewrite/client.go

@@ -212,9 +212,6 @@ var (
 	})
 )
 
-// GetDroppedRows returns value of droppedRows metric
-func GetDroppedRows() int64 { return int64(droppedRows.Get()) }
-
 // flush is a blocking function that marshals WriteRequest and sends
 // it to remote-write endpoint. Flush performs limited amount of retries
 // if request fails.

app/vmalert/remotewrite/client_test.go

@@ -44,7 +44,8 @@ func TestClient_Push(t *testing.T) {
 	}
 
 	r := rand.New(rand.NewSource(1))
-	const rowsN = int(1e4)
+	const rowsN = 1e4
+	var sent int
 	for i := 0; i < rowsN; i++ {
 		s := prompbmarshal.TimeSeries{
 			Samples: []prompbmarshal.Sample{{
@@ -56,11 +57,17 @@ func TestClient_Push(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected err: %s", err)
 		}
+		if err == nil {
+			sent++
+		}
 		err = faultyClient.Push(s)
 		if err != nil {
 			t.Fatalf("unexpected err: %s", err)
 		}
 	}
+	if sent == 0 {
+		t.Fatalf("0 series sent")
+	}
 	if err := client.Close(); err != nil {
 		t.Fatalf("failed to close client: %s", err)
 	}
@@ -68,66 +75,77 @@ func TestClient_Push(t *testing.T) {
 		t.Fatalf("failed to close faulty client: %s", err)
 	}
 	got := testSrv.accepted()
-	if got != rowsN {
-		t.Fatalf("expected to have %d series; got %d", rowsN, got)
+	if got != sent {
+		t.Fatalf("expected to have %d series; got %d", sent, got)
 	}
 	got = faultySrv.accepted()
-	if got != rowsN {
-		t.Fatalf("expected to have %d series for faulty client; got %d", rowsN, got)
+	if got != sent {
+		t.Fatalf("expected to have %d series for faulty client; got %d", sent, got)
 	}
 }
 
 func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
-	const batchSize = 20
+	batchSize := 20
 
-	f := func(pushCnt, batchCntExpected int) {
-		t.Helper()
-
-		// run new server
-		bcServer := newBatchCntRWServer()
-
-		// run new client
-		rwClient, err := NewClient(context.Background(), Config{
-			MaxBatchSize: batchSize,
-
-			// Set everything to 1 to simplify the calculation.
-			Concurrency:   1,
-			MaxQueueSize:  1000,
-			FlushInterval: time.Minute,
-
-			// batch count server
-			Addr: bcServer.URL,
-		})
-		if err != nil {
-			t.Fatalf("cannot create remote write client: %s", err)
-		}
-
-		// push time series to the client.
-		for i := 0; i < pushCnt; i++ {
-			if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
-				t.Fatalf("cannot time series to the client: %s", err)
-			}
-		}
-
-		// close the client so the rest ts will be flushed in `shutdown`
-		if err = rwClient.Close(); err != nil {
-			t.Fatalf("cannot shutdown client: %s", err)
-		}
-
-		// finally check how many batches is sent.
-		if bcServer.acceptedBatches() != batchCntExpected {
-			t.Fatalf("client sent batch count incorrect; got %d; want %d", bcServer.acceptedBatches(), batchCntExpected)
-		}
-		if pushCnt != bcServer.accepted() {
-			t.Fatalf("client sent time series count incorrect; got %d; want %d", bcServer.accepted(), pushCnt)
-		}
+	testTable := []struct {
+		name     string // name of the test case
+		pushCnt  int    // how many time series is pushed to the client
+		batchCnt int    // the expected batch count sent by the client
+	}{
+		{
+			name:     "pushCnt % batchSize == 0",
+			pushCnt:  batchSize * 40,
+			batchCnt: 40,
+		},
+		{
+			name:     "pushCnt % batchSize != 0",
+			pushCnt:  batchSize*40 + 1,
+			batchCnt: 40 + 1,
+		},
 	}
 
-	// pushCnt % batchSize == 0
-	f(batchSize*40, 40)
+	for _, tt := range testTable {
+		t.Run(tt.name, func(t *testing.T) {
+			// run new server
+			bcServer := newBatchCntRWServer()
 
-	//pushCnt % batchSize != 0
-	f(batchSize*40+1, 40+1)
+			// run new client
+			rwClient, err := NewClient(context.Background(), Config{
+				MaxBatchSize: batchSize,
+
+				// Set everything to 1 to simplify the calculation.
+				Concurrency:   1,
+				MaxQueueSize:  1000,
+				FlushInterval: time.Minute,
+
+				// batch count server
+				Addr: bcServer.URL,
+			})
+			if err != nil {
+				t.Fatalf("new remote write client failed, err: %v", err)
+			}
+
+			// push time series to the client.
+			for i := 0; i < tt.pushCnt; i++ {
+				if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
+					t.Fatalf("push time series to the client failed, err: %v", err)
+				}
+			}
+
+			// close the client so the rest ts will be flushed in `shutdown`
+			if err = rwClient.Close(); err != nil {
+				t.Fatalf("shutdown client failed, err: %v", err)
+			}
+
+			// finally check how many batches is sent.
+			if tt.batchCnt != bcServer.acceptedBatches() {
+				t.Errorf("client sent batch count incorrect, want: %d, get: %d", tt.batchCnt, bcServer.acceptedBatches())
+			}
+			if tt.pushCnt != bcServer.accepted() {
+				t.Errorf("client sent time series count incorrect, want: %d, get: %d", tt.pushCnt, bcServer.accepted())
+			}
+		})
+	}
 }
 
 func newRWServer() *rwServer {

app/vmalert/remotewrite/debug_client.go

@@ -32,7 +32,7 @@ func NewDebugClient() (*DebugClient, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	c := &DebugClient{
 		c: &http.Client{

app/vmalert/remotewrite/init.go

@@ -9,13 +9,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. "+
-		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -72,10 +71,10 @@ func Init(ctx context.Context) (*Client, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	t.IdleConnTimeout = *idleConnectionTimeout
-	t.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")
+	t.DialContext = httputils.GetStatDialFunc("vmalert_remotewrite")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/replay.go

@@ -15,10 +15,9 @@ import (
 
 var (
 	replayFrom = flag.String("replay.timeFrom", "",
-		"The time filter in RFC3339 format to start the replay from. E.g. '2020-01-01T20:07:00Z'")
+		"The time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'")
 	replayTo = flag.String("replay.timeTo", "",
-		"The time filter in RFC3339 format to finish the replay by. E.g. '2020-01-01T20:07:00Z'. "+
-			"By default, is set to the current time.")
+		"The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'")
 	replayRulesDelay = flag.Duration("replay.rulesDelay", time.Second,
 		"Delay between rules evaluation within the group. Could be important if there are chained rules inside the group "+
 			"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
@@ -37,20 +36,14 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
 	}
 	tFrom, err := time.Parse(time.RFC3339, *replayFrom)
 	if err != nil {
-		return fmt.Errorf("failed to parse replay.timeFrom=%q: %w", *replayFrom, err)
+		return fmt.Errorf("failed to parse %q: %w", *replayFrom, err)
 	}
-
-	// use tFrom location for default value, otherwise filters could have different locations
-	tTo := time.Now().In(tFrom.Location())
-	if *replayTo != "" {
-		tTo, err = time.Parse(time.RFC3339, *replayTo)
-		if err != nil {
-			return fmt.Errorf("failed to parse replay.timeTo=%q: %w", *replayTo, err)
-		}
+	tTo, err := time.Parse(time.RFC3339, *replayTo)
+	if err != nil {
+		return fmt.Errorf("failed to parse %q: %w", *replayTo, err)
 	}
-
 	if !tTo.After(tFrom) {
-		return fmt.Errorf("replay.timeTo=%v must be bigger than replay.timeFrom=%v", tTo, tFrom)
+		return fmt.Errorf("replay.timeTo must be bigger than replay.timeFrom")
 	}
 	labels := make(map[string]string)
 	for _, s := range *externalLabels {
@@ -75,13 +68,9 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
 		ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
 		total += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar)
 	}
-	logger.Infof("replay evaluation finished, generated %d samples", total)
-	if err := rw.Close(); err != nil {
-		return err
-	}
-	droppedRows := remotewrite.GetDroppedRows()
-	if droppedRows > 0 {
-		return fmt.Errorf("failed to push all generated samples to remote write url, dropped %d samples out of %d", droppedRows, total)
+	logger.Infof("replay finished! Imported %d samples", total)
+	if rw != nil {
+		return rw.Close()
 	}
 	return nil
 }

app/vmalert/replay_test.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -39,102 +38,135 @@ func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to ti
 }
 
 func TestReplay(t *testing.T) {
-	f := func(from, to string, maxDP int, cfg []config.Group, qb *fakeReplayQuerier) {
-		t.Helper()
-
-		fromOrig, toOrig, maxDatapointsOrig := *replayFrom, *replayTo, *replayMaxDatapoints
-		retriesOrig, delayOrig := *replayRuleRetryAttempts, *replayRulesDelay
-		defer func() {
-			*replayFrom, *replayTo = fromOrig, toOrig
-			*replayMaxDatapoints, *replayRuleRetryAttempts = maxDatapointsOrig, retriesOrig
-			*replayRulesDelay = delayOrig
-		}()
-
-		*replayRuleRetryAttempts = 1
-		*replayRulesDelay = time.Millisecond
-		rwb := &remotewrite.DebugClient{}
-		*replayFrom = from
-		*replayTo = to
-		*replayMaxDatapoints = maxDP
-		if err := replay(cfg, qb, rwb); err != nil {
-			t.Fatalf("replay failed: %s", err)
-		}
-		if len(qb.registry) > 0 {
-			t.Fatalf("not all requests were sent: %#v", qb.registry)
-		}
+	testCases := []struct {
+		name     string
+		from, to string
+		maxDP    int
+		cfg      []config.Group
+		qb       *fakeReplayQuerier
+	}{
+		{
+			name:  "one rule + one response",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:00.000Z",
+			maxDP: 10,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {"12:00:00+12:02:00": {}},
+				},
+			},
+		},
+		{
+			name:  "one rule + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "datapoints per step",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T15:02:30.000Z",
+			maxDP: 60,
+			cfg: []config.Group{
+				{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+13:00:00": {},
+						"13:00:00+14:00:00": {},
+						"14:00:00+15:00:00": {},
+						"15:00:00+15:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "multiple recording rules + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+				{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+					"max(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "multiple alerting rules + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
+				{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up) > 1": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+					"max(up) < 1": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
 	}
 
-	// one rule + one response
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, []config.Group{
-		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {"12:00:00+12:02:00": {}},
-		},
-	})
+	from, to, maxDP := *replayFrom, *replayTo, *replayMaxDatapoints
+	retries, delay := *replayRuleRetryAttempts, *replayRulesDelay
+	defer func() {
+		*replayFrom, *replayTo = from, to
+		*replayMaxDatapoints, *replayRuleRetryAttempts = maxDP, retries
+		*replayRulesDelay = delay
+	}()
 
-	// one rule + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
-		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-		},
-	})
-
-	// datapoints per step
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, []config.Group{
-		{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {
-				"12:00:00+13:00:00": {},
-				"13:00:00+14:00:00": {},
-				"14:00:00+15:00:00": {},
-				"15:00:00+15:02:30": {},
-			},
-		},
-	})
-
-	// multiple recording rules + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
-		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-		{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-			"max(up)": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-		},
-	})
-
-	// multiple alerting rules + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
-		{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
-		{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up) > 1": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-			"max(up) < 1": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-		},
-	})
+	*replayRuleRetryAttempts = 1
+	*replayRulesDelay = time.Millisecond
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			*replayFrom = tc.from
+			*replayTo = tc.to
+			*replayMaxDatapoints = tc.maxDP
+			if err := replay(tc.cfg, tc.qb, nil); err != nil {
+				t.Fatalf("replay failed: %s", err)
+			}
+			if len(tc.qb.registry) > 0 {
+				t.Fatalf("not all requests were sent: %#v", tc.qb.registry)
+			}
+		})
+	}
 }

app/vmalert/rule/alerting.go

@@ -183,7 +183,7 @@ func (ar *AlertingRule) GetAlert(id uint64) *notifier.Alert {
 	return ar.alerts[id]
 }
 
-func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...any) {
+func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...interface{}) {
 	if !ar.Debug {
 		return
 	}
@@ -441,6 +441,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			}
 			a.Value = m.Values[0]
 			a.Annotations = annotations
+			if err != nil {
+				return nil, err
+			}
 			a.KeepFiringSince = time.Time{}
 			continue
 		}

app/vmalert/rule/group.go

@@ -475,7 +475,7 @@ func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *
 	return randSleep
 }
 
-func (g *Group) infof(format string, args ...any) {
+func (g *Group) infof(format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
 	logger.Infof("group %q %s; interval=%v; eval_offset=%v; concurrency=%d",
 		g.Name, msg, g.Interval, g.EvalOffset, g.Concurrency)

app/vmalert/rule/group_test.go

@@ -37,142 +37,158 @@ func TestMain(m *testing.M) {
 }
 
 func TestUpdateWith(t *testing.T) {
-	f := func(currentRules, newRules []config.Rule) {
-		t.Helper()
-
-		g := &Group{
-			Name: "test",
-		}
-		qb := &datasource.FakeQuerier{}
-		for _, r := range currentRules {
-			r.ID = config.HashRule(r)
-			g.Rules = append(g.Rules, g.newRule(qb, r))
-		}
-
-		ng := &Group{
-			Name: "test",
-		}
-		for _, r := range newRules {
-			r.ID = config.HashRule(r)
-			ng.Rules = append(ng.Rules, ng.newRule(qb, r))
-		}
-
-		err := g.updateWith(ng)
-		if err != nil {
-			t.Fatalf("cannot update rule: %s", err)
-		}
-
-		if len(g.Rules) != len(newRules) {
-			t.Fatalf("expected to have %d rules; got: %d", len(g.Rules), len(newRules))
-		}
-		sort.Slice(g.Rules, func(i, j int) bool {
-			return g.Rules[i].ID() < g.Rules[j].ID()
-		})
-		sort.Slice(ng.Rules, func(i, j int) bool {
-			return ng.Rules[i].ID() < ng.Rules[j].ID()
-		})
-		for i, r := range g.Rules {
-			got, want := r, ng.Rules[i]
-			if got.ID() != want.ID() {
-				t.Fatalf("expected to have rule %q; got %q", want, got)
-			}
-			if err := CompareRules(t, got, want); err != nil {
-				t.Fatalf("comparison error: %s", err)
-			}
-		}
+	testCases := []struct {
+		name         string
+		currentRules []config.Rule
+		newRules     []config.Rule
+	}{
+		{
+			"new rule",
+			nil,
+			[]config.Rule{{Alert: "bar"}},
+		},
+		{
+			"update alerting rule",
+			[]config.Rule{
+				{
+					Alert: "foo",
+					Expr:  "up > 0",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+					Annotations: map[string]string{
+						"summary":     "{{ $value|humanize }}",
+						"description": "{{$labels}}",
+					},
+				},
+				{
+					Alert: "bar",
+					Expr:  "up > 0",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+				},
+			},
+			[]config.Rule{
+				{
+					Alert: "foo",
+					Expr:  "up > 10",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"baz": "bar",
+					},
+					Annotations: map[string]string{
+						"summary": "none",
+					},
+				},
+				{
+					Alert:         "bar",
+					Expr:          "up > 0",
+					For:           promutils.NewDuration(2 * time.Second),
+					KeepFiringFor: promutils.NewDuration(time.Minute),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+				},
+			},
+		},
+		{
+			"update recording rule",
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "max(up)",
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+			}},
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "min(up)",
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+			}},
+		},
+		{
+			"empty rule",
+			[]config.Rule{{Alert: "foo"}, {Record: "bar"}},
+			nil,
+		},
+		{
+			"multiple rules",
+			[]config.Rule{
+				{Alert: "bar"},
+				{Alert: "baz"},
+				{Alert: "foo"},
+			},
+			[]config.Rule{
+				{Alert: "baz"},
+				{Record: "foo"},
+			},
+		},
+		{
+			"replace rule",
+			[]config.Rule{{Alert: "foo1"}},
+			[]config.Rule{{Alert: "foo2"}},
+		},
+		{
+			"replace multiple rules",
+			[]config.Rule{
+				{Alert: "foo1"},
+				{Record: "foo2"},
+				{Alert: "foo3"},
+			},
+			[]config.Rule{
+				{Alert: "foo3"},
+				{Alert: "foo4"},
+				{Record: "foo5"},
+			},
+		},
 	}
 
-	// new rule
-	f(nil, []config.Rule{
-		{Alert: "bar"},
-	})
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := &Group{Name: "test"}
+			qb := &datasource.FakeQuerier{}
+			for _, r := range tc.currentRules {
+				r.ID = config.HashRule(r)
+				g.Rules = append(g.Rules, g.newRule(qb, r))
+			}
 
-	// update alerting rule
-	f([]config.Rule{
-		{
-			Alert: "foo",
-			Expr:  "up > 0",
-			For:   promutils.NewDuration(time.Second),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-			Annotations: map[string]string{
-				"summary":     "{{ $value|humanize }}",
-				"description": "{{$labels}}",
-			},
-		},
-		{
-			Alert: "bar",
-			Expr:  "up > 0",
-			For:   promutils.NewDuration(time.Second),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-		},
-	}, []config.Rule{
-		{
-			Alert: "foo",
-			Expr:  "up > 10",
-			For:   promutils.NewDuration(time.Second),
-			Labels: map[string]string{
-				"baz": "bar",
-			},
-			Annotations: map[string]string{
-				"summary": "none",
-			},
-		},
-		{
-			Alert:         "bar",
-			Expr:          "up > 0",
-			For:           promutils.NewDuration(2 * time.Second),
-			KeepFiringFor: promutils.NewDuration(time.Minute),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-		},
-	})
+			ng := &Group{Name: "test"}
+			for _, r := range tc.newRules {
+				r.ID = config.HashRule(r)
+				ng.Rules = append(ng.Rules, ng.newRule(qb, r))
+			}
 
-	// update recording rule
-	f([]config.Rule{{
-		Record: "foo",
-		Expr:   "max(up)",
-		Labels: map[string]string{
-			"bar": "baz",
-		},
-	}}, []config.Rule{{
-		Record: "foo",
-		Expr:   "min(up)",
-		Labels: map[string]string{
-			"baz": "bar",
-		},
-	}})
+			err := g.updateWith(ng)
+			if err != nil {
+				t.Fatal(err)
+			}
 
-	// empty rule
-	f([]config.Rule{{Alert: "foo"}, {Record: "bar"}}, nil)
-
-	// multiple rules
-	f([]config.Rule{
-		{Alert: "bar"},
-		{Alert: "baz"},
-		{Alert: "foo"},
-	}, []config.Rule{
-		{Alert: "baz"},
-		{Record: "foo"},
-	})
-
-	// replace rule
-	f([]config.Rule{{Alert: "foo1"}}, []config.Rule{{Alert: "foo2"}})
-
-	// replace multiple rules
-	f([]config.Rule{
-		{Alert: "foo1"},
-		{Record: "foo2"},
-		{Alert: "foo3"},
-	}, []config.Rule{
-		{Alert: "foo3"},
-		{Alert: "foo4"},
-		{Record: "foo5"},
-	})
+			if len(g.Rules) != len(tc.newRules) {
+				t.Fatalf("expected to have %d rules; got: %d",
+					len(g.Rules), len(tc.newRules))
+			}
+			sort.Slice(g.Rules, func(i, j int) bool {
+				return g.Rules[i].ID() < g.Rules[j].ID()
+			})
+			sort.Slice(ng.Rules, func(i, j int) bool {
+				return ng.Rules[i].ID() < ng.Rules[j].ID()
+			})
+			for i, r := range g.Rules {
+				got, want := r, ng.Rules[i]
+				if got.ID() != want.ID() {
+					t.Fatalf("expected to have rule %q; got %q", want, got)
+				}
+				if err := CompareRules(t, got, want); err != nil {
+					t.Fatalf("comparison error: %s", err)
+				}
+			}
+		})
+	}
 }
 
 func TestGroupStart(t *testing.T) {
@@ -296,23 +312,30 @@ func TestGroupStart(t *testing.T) {
 	<-finished
 }
 
-func TestGetResolveDuration(t *testing.T) {
-	f := func(groupInterval, maxDuration, resendDelay, resultExpected time.Duration) {
-		t.Helper()
-
-		result := getResolveDuration(groupInterval, resendDelay, maxDuration)
-		if result != resultExpected {
-			t.Fatalf("unexpected result; got %s; want %s", result, resultExpected)
-		}
+func TestResolveDuration(t *testing.T) {
+	testCases := []struct {
+		groupInterval time.Duration
+		maxDuration   time.Duration
+		resendDelay   time.Duration
+		expected      time.Duration
+	}{
+		{time.Minute, 0, 0, 4 * time.Minute},
+		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
+		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
+		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
+		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
+		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
+		{0, 0, 0, 0},
 	}
 
-	f(0, 0, 0, 0)
-	f(time.Minute, 0, 0, 4*time.Minute)
-	f(time.Minute, 0, 2*time.Minute, 8*time.Minute)
-	f(time.Minute, 4*time.Minute, 4*time.Minute, 4*time.Minute)
-	f(2*time.Minute, time.Minute, 2*time.Minute, time.Minute)
-	f(time.Minute, 2*time.Minute, 1*time.Minute, 2*time.Minute)
-	f(2*time.Minute, 0, 1*time.Minute, 8*time.Minute)
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
+			got := getResolveDuration(tc.groupInterval, tc.resendDelay, tc.maxDuration)
+			if got != tc.expected {
+				t.Errorf("expected to have %v; got %v", tc.expected, got)
+			}
+		})
+	}
 }
 
 func TestGetStaleSeries(t *testing.T) {
@@ -322,7 +345,6 @@ func TestGetStaleSeries(t *testing.T) {
 	}
 	f := func(r Rule, labels, expLabels [][]prompbmarshal.Label) {
 		t.Helper()
-
 		var tss []prompbmarshal.TimeSeries
 		for _, l := range labels {
 			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
@@ -584,7 +606,7 @@ func TestGroupStartDelay(t *testing.T) {
 		delay := delayBeforeStart(at, key, g.Interval, g.EvalOffset)
 		gotStart := at.Add(delay)
 		if expTS != gotStart {
-			t.Fatalf("expected to get %v; got %v instead", expTS, gotStart)
+			t.Errorf("expected to get %v; got %v instead", expTS, gotStart)
 		}
 	}
 
@@ -625,118 +647,157 @@ func TestGroupStartDelay(t *testing.T) {
 }
 
 func TestGetPrometheusReqTimestamp(t *testing.T) {
-	f := func(g *Group, tsOrigin, tsExpected string) {
-		t.Helper()
-
-		originT, _ := time.Parse(time.RFC3339, tsOrigin)
-		expT, _ := time.Parse(time.RFC3339, tsExpected)
-		gotTS := g.adjustReqTimestamp(originT)
-		if !gotTS.Equal(expT) {
-			t.Fatalf("get wrong prometheus request timestamp: %s; want %s", gotTS, expT)
-		}
-	}
-
 	offset := 30 * time.Minute
 	evalDelay := 1 * time.Minute
 	disableAlign := false
-
-	// with query align + default evalDelay
-	f(&Group{
-		Interval: time.Hour,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:00:00+00:00")
-
-	// without query align + default evalDelay
-	f(&Group{
-		Interval:      time.Hour,
-		evalAlignment: &disableAlign,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:10:30+00:00")
-
-	// with eval_offset, find previous offset point + default evalDelay
-	f(&Group{
-		EvalOffset: &offset,
-		Interval:   time.Hour,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T10:30:00+00:00")
-
-	// with eval_offset + default evalDelay
-	f(&Group{
-		EvalOffset: &offset,
-		Interval:   time.Hour,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:30:00+00:00")
-
-	// 1h interval with eval_delay
-	f(&Group{
-		EvalDelay: &evalDelay,
-		Interval:  time.Hour,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:00:00+00:00")
-
-	// 1m interval with eval_delay
-	f(&Group{
-		EvalDelay: &evalDelay,
-		Interval:  time.Minute,
-	}, "2023-08-28T11:41:13+00:00", "2023-08-28T11:40:00+00:00")
-
-	// disable alignment with eval_delay
-	f(&Group{
-		EvalDelay:     &evalDelay,
-		Interval:      time.Hour,
-		evalAlignment: &disableAlign,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:40:00+00:00")
+	testCases := []struct {
+		name            string
+		g               *Group
+		originTS, expTS string
+	}{
+		{
+			"with query align + default evalDelay",
+			&Group{
+				Interval: time.Hour,
+			},
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T11:00:00+00:00",
+		},
+		{
+			"without query align + default evalDelay",
+			&Group{
+				Interval:      time.Hour,
+				evalAlignment: &disableAlign,
+			},
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T11:10:30+00:00",
+		},
+		{
+			"with eval_offset, find previous offset point + default evalDelay",
+			&Group{
+				EvalOffset: &offset,
+				Interval:   time.Hour,
+			},
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T10:30:00+00:00",
+		},
+		{
+			"with eval_offset + default evalDelay",
+			&Group{
+				EvalOffset: &offset,
+				Interval:   time.Hour,
+			},
+			"2023-08-28T11:41:00+00:00",
+			"2023-08-28T11:30:00+00:00",
+		},
+		{
+			"1h interval with eval_delay",
+			&Group{
+				EvalDelay: &evalDelay,
+				Interval:  time.Hour,
+			},
+			"2023-08-28T11:41:00+00:00",
+			"2023-08-28T11:00:00+00:00",
+		},
+		{
+			"1m interval with eval_delay",
+			&Group{
+				EvalDelay: &evalDelay,
+				Interval:  time.Minute,
+			},
+			"2023-08-28T11:41:13+00:00",
+			"2023-08-28T11:40:00+00:00",
+		},
+		{
+			"disable alignment with eval_delay",
+			&Group{
+				EvalDelay:     &evalDelay,
+				Interval:      time.Hour,
+				evalAlignment: &disableAlign,
+			},
+			"2023-08-28T11:41:00+00:00",
+			"2023-08-28T11:40:00+00:00",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			originT, _ := time.Parse(time.RFC3339, tc.originTS)
+			expT, _ := time.Parse(time.RFC3339, tc.expTS)
+			gotTS := tc.g.adjustReqTimestamp(originT)
+			if !gotTS.Equal(expT) {
+				t.Fatalf("get wrong prometheus request timestamp, expect %s, got %s", expT, gotTS)
+			}
+		})
+	}
 }
 
 func TestRangeIterator(t *testing.T) {
-	f := func(ri rangeIterator, resultExpected [][2]time.Time) {
-		t.Helper()
-
-		var j int
-		for ri.next() {
-			if len(resultExpected) < j+1 {
-				t.Fatalf("unexpected result for iterator on step %d: %v - %v", j, ri.s, ri.e)
-			}
-			s, e := ri.s, ri.e
-			expS, expE := resultExpected[j][0], resultExpected[j][1]
-			if s != expS {
-				t.Fatalf("expected to get start=%v; got %v", expS, s)
-			}
-			if e != expE {
-				t.Fatalf("expected to get end=%v; got %v", expE, e)
-			}
-			j++
-		}
+	testCases := []struct {
+		ri     rangeIterator
+		result [][2]time.Time
+	}{
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+				step:  5 * time.Minute,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:05:00.000Z")},
+				{parseTime(t, "2021-01-01T12:05:00.000Z"), parseTime(t, "2021-01-01T12:10:00.000Z")},
+				{parseTime(t, "2021-01-01T12:10:00.000Z"), parseTime(t, "2021-01-01T12:15:00.000Z")},
+				{parseTime(t, "2021-01-01T12:15:00.000Z"), parseTime(t, "2021-01-01T12:20:00.000Z")},
+				{parseTime(t, "2021-01-01T12:20:00.000Z"), parseTime(t, "2021-01-01T12:25:00.000Z")},
+				{parseTime(t, "2021-01-01T12:25:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+			},
+		},
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+				step:  45 * time.Minute,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+				{parseTime(t, "2021-01-01T12:30:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+			},
+		},
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:12.000Z"),
+				end:   parseTime(t, "2021-01-01T12:00:17.000Z"),
+				step:  time.Second,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:12.000Z"), parseTime(t, "2021-01-01T12:00:13.000Z")},
+				{parseTime(t, "2021-01-01T12:00:13.000Z"), parseTime(t, "2021-01-01T12:00:14.000Z")},
+				{parseTime(t, "2021-01-01T12:00:14.000Z"), parseTime(t, "2021-01-01T12:00:15.000Z")},
+				{parseTime(t, "2021-01-01T12:00:15.000Z"), parseTime(t, "2021-01-01T12:00:16.000Z")},
+				{parseTime(t, "2021-01-01T12:00:16.000Z"), parseTime(t, "2021-01-01T12:00:17.000Z")},
+			},
+		},
 	}
 
-	f(rangeIterator{
-		start: parseTime(t, "2021-01-01T12:00:00.000Z"),
-		end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
-		step:  5 * time.Minute,
-	}, [][2]time.Time{
-		{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:05:00.000Z")},
-		{parseTime(t, "2021-01-01T12:05:00.000Z"), parseTime(t, "2021-01-01T12:10:00.000Z")},
-		{parseTime(t, "2021-01-01T12:10:00.000Z"), parseTime(t, "2021-01-01T12:15:00.000Z")},
-		{parseTime(t, "2021-01-01T12:15:00.000Z"), parseTime(t, "2021-01-01T12:20:00.000Z")},
-		{parseTime(t, "2021-01-01T12:20:00.000Z"), parseTime(t, "2021-01-01T12:25:00.000Z")},
-		{parseTime(t, "2021-01-01T12:25:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-	})
-
-	f(rangeIterator{
-		start: parseTime(t, "2021-01-01T12:00:00.000Z"),
-		end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
-		step:  45 * time.Minute,
-	}, [][2]time.Time{
-		{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-		{parseTime(t, "2021-01-01T12:30:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-	})
-
-	f(rangeIterator{
-		start: parseTime(t, "2021-01-01T12:00:12.000Z"),
-		end:   parseTime(t, "2021-01-01T12:00:17.000Z"),
-		step:  time.Second,
-	}, [][2]time.Time{
-		{parseTime(t, "2021-01-01T12:00:12.000Z"), parseTime(t, "2021-01-01T12:00:13.000Z")},
-		{parseTime(t, "2021-01-01T12:00:13.000Z"), parseTime(t, "2021-01-01T12:00:14.000Z")},
-		{parseTime(t, "2021-01-01T12:00:14.000Z"), parseTime(t, "2021-01-01T12:00:15.000Z")},
-		{parseTime(t, "2021-01-01T12:00:15.000Z"), parseTime(t, "2021-01-01T12:00:16.000Z")},
-		{parseTime(t, "2021-01-01T12:00:16.000Z"), parseTime(t, "2021-01-01T12:00:17.000Z")},
-	})
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			var j int
+			for tc.ri.next() {
+				if len(tc.result) < j+1 {
+					t.Fatalf("unexpected result for iterator on step %d: %v - %v",
+						j, tc.ri.s, tc.ri.e)
+				}
+				s, e := tc.ri.s, tc.ri.e
+				expS, expE := tc.result[j][0], tc.result[j][1]
+				if s != expS {
+					t.Fatalf("expected to get start=%v; got %v", expS, s)
+				}
+				if e != expE {
+					t.Fatalf("expected to get end=%v; got %v", expE, e)
+				}
+				j++
+			}
+		})
+	}
 }
 
 func parseTime(t *testing.T, s string) time.Time {

app/vmalert/rule/recording_test.go

@@ -13,225 +13,218 @@ import (
 )
 
 func TestRecordingRule_Exec(t *testing.T) {
-	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
-		t.Helper()
-
-		fq := &datasource.FakeQuerier{}
-		fq.Add(metrics...)
-		rule.q = fq
-		rule.state = &ruleState{
-			entries: make([]StateEntry, 10),
-		}
-		tss, err := rule.exec(context.TODO(), time.Now(), 0)
-		if err != nil {
-			t.Fatalf("unexpected RecordingRule.exec error: %s", err)
-		}
-		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries missmatch: %s", err)
-		}
-	}
-
 	timestamp := time.Now()
-
-	f(&RecordingRule{
-		Name: "foo",
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 10, "__name__", "bar"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foo",
-		}),
-	})
-
-	f(&RecordingRule{
-		Name: "foobarbaz",
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
-		metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
-		metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "foo",
-		}),
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "bar",
-		}),
-		newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "baz",
-		}),
-	})
-
-	f(&RecordingRule{
-		Name: "job:foo",
-		Labels: map[string]string{
-			"source": "test",
+	testCases := []struct {
+		rule    *RecordingRule
+		metrics []datasource.Metric
+		expTS   []prompbmarshal.TimeSeries
+	}{
+		{
+			&RecordingRule{Name: "foo"},
+			[]datasource.Metric{metricWithValueAndLabels(t, 10,
+				"__name__", "bar",
+			)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foo",
+				}),
+			},
 		},
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "foo",
-			"source":   "test",
-		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":        "job:foo",
-			"job":             "bar",
-			"source":          "test",
-			"exported_source": "origin",
-		}),
-	})
+		{
+			&RecordingRule{Name: "foobarbaz"},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
+				metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "foo",
+				}),
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "bar",
+				}),
+				newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "baz",
+				}),
+			},
+		},
+		{
+			&RecordingRule{
+				Name: "job:foo",
+				Labels: map[string]string{
+					"source": "test",
+				},
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "foo",
+					"source":   "test",
+				}),
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":        "job:foo",
+					"job":             "bar",
+					"source":          "test",
+					"exported_source": "origin",
+				}),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &datasource.FakeQuerier{}
+			fq.Add(tc.metrics...)
+			tc.rule.q = fq
+			tc.rule.state = &ruleState{entries: make([]StateEntry, 10)}
+			tss, err := tc.rule.exec(context.TODO(), time.Now(), 0)
+			if err != nil {
+				t.Fatalf("unexpected Exec err: %s", err)
+			}
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
 }
 
 func TestRecordingRule_ExecRange(t *testing.T) {
-	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
-		t.Helper()
-
-		fq := &datasource.FakeQuerier{}
-		fq.Add(metrics...)
-		rule.q = fq
-		tss, err := rule.execRange(context.TODO(), time.Now(), time.Now())
-		if err != nil {
-			t.Fatalf("unexpected RecordingRule.execRange error: %s", err)
-		}
-		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries missmatch: %s", err)
-		}
-	}
-
 	timestamp := time.Now()
+	testCases := []struct {
+		rule    *RecordingRule
+		metrics []datasource.Metric
+		expTS   []prompbmarshal.TimeSeries
+	}{
+		{
+			&RecordingRule{Name: "foo"},
+			[]datasource.Metric{metricWithValuesAndLabels(t, []float64{10, 20, 30},
+				"__name__", "bar",
+			)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{10, 20, 30},
+					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
+					map[string]string{
+						"__name__": "foo",
+					}),
+			},
+		},
+		{
+			&RecordingRule{Name: "foobarbaz"},
+			[]datasource.Metric{
+				metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+				metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+				metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "foo",
+				}),
+				newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "bar",
+				}),
+				newTimeSeries([]float64{4, 5, 6},
+					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
+					map[string]string{
+						"__name__": "foobarbaz",
+						"job":      "baz",
+					}),
+			},
+		},
+		{
+			&RecordingRule{Name: "job:foo", Labels: map[string]string{
+				"source": "test",
+			}},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "foo",
+					"source":   "test",
+				}),
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "bar",
+					"source":   "test",
+				}),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &datasource.FakeQuerier{}
+			fq.Add(tc.metrics...)
+			tc.rule.q = fq
+			tss, err := tc.rule.execRange(context.TODO(), time.Now(), time.Now())
+			if err != nil {
+				t.Fatalf("unexpected Exec err: %s", err)
+			}
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
 
-	f(&RecordingRule{
-		Name: "foo",
-	}, []datasource.Metric{
-		metricWithValuesAndLabels(t, []float64{10, 20, 30}, "__name__", "bar"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-			"__name__": "foo",
-		}),
-	})
-
-	f(&RecordingRule{
-		Name: "foobarbaz",
-	}, []datasource.Metric{
+func TestRecordingRuleLimit(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		limit int
+		err   string
+	}{
+		{
+			limit: 0,
+		},
+		{
+			limit: -1,
+		},
+		{
+			limit: 1,
+			err:   "exec exceeded limit of 1 with 3 series",
+		},
+		{
+			limit: 2,
+			err:   "exec exceeded limit of 2 with 3 series",
+		},
+	}
+	testMetrics := []datasource.Metric{
 		metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
 		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
 		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "foo",
-		}),
-		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "bar",
-		}),
-		newTimeSeries([]float64{4, 5, 6},
-			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-				"__name__": "foobarbaz",
-				"job":      "baz",
-			}),
-	})
-
-	f(&RecordingRule{
-		Name: "job:foo",
+	}
+	rule := &RecordingRule{Name: "job:foo",
+		state: &ruleState{entries: make([]StateEntry, 10)},
 		Labels: map[string]string{
-			"source": "test",
+			"source": "test_limit",
 		},
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "foo",
-			"source":   "test",
-		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "bar",
-			"source":   "test",
-		}),
-	})
-}
-
-func TestRecordingRuleLimit_Failure(t *testing.T) {
-	f := func(limit int, errStrExpected string) {
-		t.Helper()
-
-		testMetrics := []datasource.Metric{
-			metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
-			metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
-			metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-		}
-
+		metrics: &recordingRuleMetrics{
+			errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
+		},
+	}
+	var err error
+	for _, testCase := range testCases {
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-
-		rule := &RecordingRule{Name: "job:foo",
-			state: &ruleState{entries: make([]StateEntry, 10)},
-			Labels: map[string]string{
-				"source": "test_limit",
-			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
-		}
 		rule.q = fq
-
-		_, err := rule.exec(context.TODO(), time.Now(), limit)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		errStr := err.Error()
-		if !strings.Contains(errStr, errStrExpected) {
-			t.Fatalf("missing %q in the error %q", errStrExpected, errStr)
+		_, err = rule.exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
 		}
 	}
-
-	f(1, "exec exceeded limit of 1 with 3 series")
-	f(2, "exec exceeded limit of 2 with 3 series")
 }
 
-func TestRecordingRuleLimit_Success(t *testing.T) {
-	f := func(limit int) {
-		t.Helper()
-
-		testMetrics := []datasource.Metric{
-			metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
-			metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
-			metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-		}
-
-		fq := &datasource.FakeQuerier{}
-		fq.Add(testMetrics...)
-
-		rule := &RecordingRule{Name: "job:foo",
-			state: &ruleState{entries: make([]StateEntry, 10)},
-			Labels: map[string]string{
-				"source": "test_limit",
-			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
-		}
-		rule.q = fq
-
-		_, err := rule.exec(context.TODO(), time.Now(), limit)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	f(0)
-	f(-1)
-}
-
-func TestRecordingRuleExec_Negative(t *testing.T) {
+func TestRecordingRule_ExecNegative(t *testing.T) {
 	rr := &RecordingRule{
 		Name: "job:foo",
 		Labels: map[string]string{
@@ -263,6 +256,6 @@ func TestRecordingRuleExec_Negative(t *testing.T) {
 
 	_, err = rr.exec(context.TODO(), time.Now(), 0)
 	if err != nil {
-		t.Fatalf("cannot execute recroding rule: %s", err)
+		t.Fatal(err)
 	}
 }

app/vmalert/templates/template.go

@@ -316,7 +316,7 @@ func templateFuncs() textTpl.FuncMap {
 
 		// humanize converts given number to a human readable format
 		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(i any) (string, error) {
+		"humanize": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -347,7 +347,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(i any) (string, error) {
+		"humanize1024": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -359,7 +359,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizeDuration converts given seconds to a human-readable duration
-		"humanizeDuration": func(i any) (string, error) {
+		"humanizeDuration": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -405,7 +405,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(i any) (string, error) {
+		"humanizePercentage": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -414,7 +414,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(i any) (string, error) {
+		"humanizeTimestamp": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -427,7 +427,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// toTime converts given timestamp to a time.Time.
-		"toTime": func(i any) (time.Time, error) {
+		"toTime": func(i interface{}) (time.Time, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return time.Time{}, err
@@ -524,8 +524,8 @@ func templateFuncs() textTpl.FuncMap {
 
 		// Converts a list of objects to a map with keys arg0, arg1 etc.
 		// This is intended to allow multiple arguments to be passed to templates.
-		"args": func(args ...any) map[string]any {
-			result := make(map[string]any)
+		"args": func(args ...interface{}) map[string]interface{} {
+			result := make(map[string]interface{})
 			for i, a := range args {
 				result[fmt.Sprintf("arg%d", i)] = a
 			}
@@ -565,7 +565,7 @@ func (t Time) Time() time.Time {
 	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
 }
 
-func toFloat64(v any) (float64, error) {
+func toFloat64(v interface{}) (float64, error) {
 	switch i := v.(type) {
 	case float64:
 		return i, nil

app/vmalert/templates/template_test.go

@@ -7,11 +7,10 @@ import (
 	textTpl "text/template"
 )
 
-func TestTemplateFuncs_StringConversion(t *testing.T) {
+func TestTemplateFuncs(t *testing.T) {
+	funcs := templateFuncs()
 	f := func(funcName, s, resultExpected string) {
 		t.Helper()
-
-		funcs := templateFuncs()
 		v := funcs[funcName]
 		fLocal := v.(func(s string) string)
 		result := fLocal(s)
@@ -19,7 +18,6 @@ func TestTemplateFuncs_StringConversion(t *testing.T) {
 			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
 		}
 	}
-
 	f("title", "foo bar", "Foo Bar")
 	f("toUpper", "foo", "FOO")
 	f("toLower", "FOO", "foo")
@@ -33,10 +31,7 @@ func TestTemplateFuncs_StringConversion(t *testing.T) {
 	f("stripPort", "foo:1234", "foo")
 	f("stripDomain", "foo.bar.baz", "foo")
 	f("stripDomain", "foo.bar:123", "foo:123")
-}
 
-func TestTemplateFuncs_Match(t *testing.T) {
-	funcs := templateFuncs()
 	// check "match" func
 	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
 	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
@@ -56,15 +51,11 @@ func TestTemplateFuncs_Match(t *testing.T) {
 	if !ok {
 		t.Fatalf("unexpected mismatch")
 	}
-}
 
-func TestTemplateFuncs_Formatting(t *testing.T) {
-	f := func(funcName string, p any, resultExpected string) {
+	formatting := func(funcName string, p interface{}, resultExpected string) {
 		t.Helper()
-
-		funcs := templateFuncs()
 		v := funcs[funcName]
-		fLocal := v.(func(s any) (string, error))
+		fLocal := v.(func(s interface{}) (string, error))
 		result, err := fLocal(p)
 		if err != nil {
 			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
@@ -73,36 +64,35 @@ func TestTemplateFuncs_Formatting(t *testing.T) {
 			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
 		}
 	}
+	formatting("humanize1024", float64(0), "0")
+	formatting("humanize1024", math.Inf(0), "+Inf")
+	formatting("humanize1024", math.NaN(), "NaN")
+	formatting("humanize1024", float64(127087), "124.1ki")
+	formatting("humanize1024", float64(130137088), "124.1Mi")
+	formatting("humanize1024", float64(133260378112), "124.1Gi")
+	formatting("humanize1024", float64(136458627186688), "124.1Ti")
+	formatting("humanize1024", float64(139733634239168512), "124.1Pi")
+	formatting("humanize1024", float64(143087241460908556288), "124.1Ei")
+	formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
+	formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
+	formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
 
-	f("humanize1024", float64(0), "0")
-	f("humanize1024", math.Inf(0), "+Inf")
-	f("humanize1024", math.NaN(), "NaN")
-	f("humanize1024", float64(127087), "124.1ki")
-	f("humanize1024", float64(130137088), "124.1Mi")
-	f("humanize1024", float64(133260378112), "124.1Gi")
-	f("humanize1024", float64(136458627186688), "124.1Ti")
-	f("humanize1024", float64(139733634239168512), "124.1Pi")
-	f("humanize1024", float64(143087241460908556288), "124.1Ei")
-	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
-	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
-	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
+	formatting("humanize", float64(127087), "127.1k")
+	formatting("humanize", float64(136458627186688), "136.5T")
 
-	f("humanize", float64(127087), "127.1k")
-	f("humanize", float64(136458627186688), "136.5T")
+	formatting("humanizeDuration", 1, "1s")
+	formatting("humanizeDuration", 0.2, "200ms")
+	formatting("humanizeDuration", 42000, "11h 40m 0s")
+	formatting("humanizeDuration", 16790555, "194d 8h 2m 35s")
 
-	f("humanizeDuration", 1, "1s")
-	f("humanizeDuration", 0.2, "200ms")
-	f("humanizeDuration", 42000, "11h 40m 0s")
-	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
+	formatting("humanizePercentage", 1, "100%")
+	formatting("humanizePercentage", 0.8, "80%")
+	formatting("humanizePercentage", 0.015, "1.5%")
 
-	f("humanizePercentage", 1, "100%")
-	f("humanizePercentage", 0.8, "80%")
-	f("humanizePercentage", 0.015, "1.5%")
-
-	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
+	formatting("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
 }
 
-func mkTemplate(current, replacement any) textTemplate {
+func mkTemplate(current, replacement interface{}) textTemplate {
 	tmpl := textTemplate{}
 	if current != nil {
 		switch val := current.(type) {
@@ -148,201 +138,224 @@ func equalTemplates(tmpls ...*textTpl.Template) bool {
 	return true
 }
 
-func TestTemplatesLoad_Failure(t *testing.T) {
-	f := func(pathPatterns []string, expectedErrStr string) {
-		t.Helper()
-
-		err := Load(pathPatterns, false)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-
-		errStr := err.Error()
-		if !strings.Contains(errStr, expectedErrStr) {
-			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
-		}
+func TestTemplates_Load(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		pathPatterns     []string
+		overwrite        bool
+		expectedTemplate textTemplate
+		expErr           string
+	}{
+		{
+			"non existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"non existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`, nil),
+			"",
+		},
+		{
+			"existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				`, `
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`),
+			"",
+		},
+		{
+			"load template with syntax error",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/bad0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			"failed to parse template glob",
+		},
 	}
 
-	// load template with syntax error
-	f([]string{
-		"templates/other/nested/bad0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}, "failed to parse template glob")
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			err := Load(tc.pathPatterns, tc.overwrite)
+			if tc.expErr == "" && err != nil {
+				t.Error("happened error that wasn't expected: %w", err)
+			}
+			if tc.expErr != "" && err == nil {
+				t.Error("%+w", err)
+				t.Error("expected error that didn't happened")
+			}
+			if err != nil && !strings.Contains(err.Error(), tc.expErr) {
+				t.Error("%+w", err)
+				t.Error("expected string doesn't exist in error message")
+			}
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
 }
 
-func TestTemplatesLoad_Success(t *testing.T) {
-	f := func(initialTmpl textTemplate, pathPatterns []string, overwrite bool, expectedTmpl textTemplate) {
-		t.Helper()
-
-		masterTmplOrig := masterTmpl
-		masterTmpl = initialTmpl
-		defer func() {
-			masterTmpl = masterTmplOrig
-		}()
-
-		if err := Load(pathPatterns, overwrite); err != nil {
-			t.Fatalf("cannot load templates: %s", err)
-		}
-
-		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
-			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
-		}
-		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
-			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
-		}
+func TestTemplates_Reload(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		expectedTemplate textTemplate
+	}{
+		{
+			"empty current and replacement templates",
+			mkTemplate(nil, nil),
+			mkTemplate(nil, nil),
+		},
+		{
+			"empty current template only",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"empty replacement template only",
+			mkTemplate(nil, `
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"defined both templates",
+			mkTemplate(`
+				{{- define "test.0" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{- printf "before" -}}
+				{{- end -}}
+			`, `
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`, nil),
+		},
 	}
 
-	// non existing path undefined template override
-	initialTmpl := mkTemplate(nil, nil)
-	pathPatterns := []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			Reload()
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
 	}
-	overwrite := true
-	expectedTmpl := mkTemplate(``, nil)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-
-	// non existing path defined template override
-	initialTmpl = mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil)
-	pathPatterns = []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
-	}
-	overwrite = true
-	expectedTmpl = mkTemplate(``, nil)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-
-	// existing path undefined template override
-	initialTmpl = mkTemplate(nil, nil)
-	pathPatterns = []string{
-		"templates/other/nested/good0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}
-	overwrite = false
-	expectedTmpl = mkTemplate(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`, nil)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-
-	// existing path defined template override
-	initialTmpl = mkTemplate(`
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" "world" }}
-		{{- end -}}
-	`, nil)
-	pathPatterns = []string{
-		"templates/other/nested/good0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}
-	overwrite = false
-	expectedTmpl = mkTemplate(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" "world" }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`, `
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-}
-
-func TestTemplatesReload(t *testing.T) {
-	f := func(initialTmpl, expectedTmpl textTemplate) {
-		t.Helper()
-
-		masterTmplOrig := masterTmpl
-		masterTmpl = initialTmpl
-		defer func() {
-			masterTmpl = masterTmplOrig
-		}()
-
-		Reload()
-
-		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
-			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
-		}
-		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
-			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
-		}
-	}
-
-	// empty current and replacement templates
-	f(mkTemplate(nil, nil), mkTemplate(nil, nil))
-
-	// empty current template only
-	f(mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil), mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil))
-
-	// empty replacement template only
-	f(mkTemplate(nil, `
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`), mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil))
-
-	// defined both templates
-	f(mkTemplate(`
-		{{- define "test.0" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{- printf "before" -}}
-		{{- end -}}
-	`, `
-		{{- define "test.1" -}}
-			{{- printf "after" -}}
-		{{- end -}}
-	`), mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "after" -}}
-		{{- end -}}
-	`, nil))
 }

app/vmalert/utils/err_group_test.go

@@ -7,31 +7,35 @@ import (
 )
 
 func TestErrGroup(t *testing.T) {
-	f := func(errs []error, resultExpected string) {
-		t.Helper()
-
-		eg := &ErrGroup{}
-		for _, err := range errs {
+	testCases := []struct {
+		errs []error
+		exp  string
+	}{
+		{nil, ""},
+		{[]error{errors.New("timeout")}, "errors(1): timeout"},
+		{
+			[]error{errors.New("timeout"), errors.New("deadline")},
+			"errors(2): timeout\ndeadline",
+		},
+	}
+	for _, tc := range testCases {
+		eg := new(ErrGroup)
+		for _, err := range tc.errs {
 			eg.Add(err)
 		}
-		if len(errs) == 0 {
+		if len(tc.errs) == 0 {
 			if eg.Err() != nil {
 				t.Fatalf("expected to get nil error")
 			}
-			return
+			continue
 		}
 		if eg.Err() == nil {
 			t.Fatalf("expected to get non-nil error")
 		}
-		result := eg.Error()
-		if result != resultExpected {
-			t.Fatalf("unexpected result\ngot\n%v\nwant\n%v", result, resultExpected)
+		if eg.Error() != tc.exp {
+			t.Fatalf("expected to have: \n%q\ngot:\n%q", tc.exp, eg.Error())
 		}
 	}
-
-	f(nil, "")
-	f([]error{errors.New("timeout")}, "errors(1): timeout")
-	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): timeout\ndeadline")
 }
 
 // TestErrGroupConcurrent supposed to test concurrent