Implement SBOM generation

Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>

.github/workflows/build.yml

@@ -0,0 +1,54 @@
+name: build
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Makefile'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Makefile'
+
+permissions:
+  contents: read
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
+
+      - name: Run crossbuild
+        run: make crossbuild

.github/workflows/codeql-analysis-go.yml

@@ -0,0 +1,62 @@
+name: 'CodeQL Go'
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: stable
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: go
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: 'language:go'

.github/workflows/codeql-analysis-js-ts.yml

@@ -1,22 +1,27 @@
-name: "CodeQL - JS"
+name: 'CodeQL JS/TS'
 
 on:
   push:
-    branches: [master, cluster]
+    branches:
+      - cluster
+      - master
     paths:
-      - "**.js"
+      - '**.js'
+      - '**.ts'
+      - '**.tsx'
   pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master, cluster]
+    branches:
+      - cluster
+      - master
     paths:
-      - "**.js"
-  schedule:
-    - cron: "30 18 * * 2"
+      - '**.js'
+      - '**.ts'
+      - '**.tsx'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
-  
+
 jobs:
   analyze:
     name: Analyze
@@ -26,11 +31,6 @@ jobs:
       contents: read
       security-events: write
 
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["javascript"]
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -38,9 +38,9 @@ jobs:
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with:
-          languages: ${{ matrix.language }}
+          languages: javascript-typescript
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v3
         with:
-          category: "javascript"
+          category: 'language:js/ts'

.github/workflows/codeql-analysis.yml

@@ -1,103 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
-  schedule:
-    - cron: "30 18 * * 2"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["go"]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-        if: ${{ matrix.language == 'go' }}
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-        if: ${{ matrix.language == 'go' }}
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
-
-      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-      #    and modify them (or add more) to build your code if your project
-      #    uses a compiled language
-
-      #- run: |
-      #   make bootstrap
-      #   make release
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3

.github/workflows/main.yml

@@ -1,29 +1,25 @@
 name: main
+
 on:
   push:
     branches:
-      - master
       - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - master
+    paths:
+      - '**.go'
   pull_request:
     branches:
-      - master
       - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - master
+    paths:
+      - '**.go'
+
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
 
 jobs:
   lint:
@@ -37,16 +33,16 @@ jobs:
         id: go
         uses: actions/setup-go@v5
         with:
-          go-version: stable
           cache: false
+          go-version: stable
 
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
-            ~/go/pkg/mod
             ~/go/bin
+            ~/go/pkg/mod
           key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-check-all-
 
@@ -55,41 +51,18 @@ jobs:
           make check-all
           git diff --exit-code
 
-  build:
-    needs: lint
-    name: build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
-      - name: Build
-        run: make crossbuild
-
   test:
+    name: test
     needs: lint
+    runs-on: ubuntu-latest
+
     strategy:
       matrix:
-        scenario: ["test-full", "test-pure", "test-full-386"]
-    name: test
-    runs-on: ubuntu-latest
+        scenario:
+          - 'test-full'
+          - 'test-full-386'
+          - 'test-pure'
+
     steps:
       - name: Code checkout
         uses: actions/checkout@v4
@@ -98,20 +71,20 @@ jobs:
         id: go
         uses: actions/setup-go@v5
         with:
-          go-version: stable
           cache: false
+          go-version: stable
 
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
-            ~/go/pkg/mod
             ~/go/bin
+            ~/go/pkg/mod
           key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
 
-      - name: run tests
+      - name: Run tests
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage

.github/workflows/sync-docs.yml

@@ -25,7 +25,7 @@ jobs:
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: docs
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v5
+        uses: crazy-max/ghaction-import-gpg@v6
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}

.github/workflows/test_package.yml

@@ -0,0 +1,59 @@
+name: Package tests
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '.github/workflows/test_package.yml'
+      - '**/Dockerfile*'
+      - '**/Makefile'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '.github/workflows/test_package.yml'
+      - '**/Dockerfile*'
+      - '**/Makefile'
+
+permissions:
+  contents: read
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  test_release:
+    name: Test Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-test-release-${{ steps.go.outputs.go-version }}-${{ hashFiles('Makefile', '**/Dockerfile*', '**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-test-release-
+
+      - name: Run make release
+        run: |
+          make clean
+          make release
+
+      - name: Run release tests
+        run: make test-release-victoria-metrics

.gitignore

@@ -22,4 +22,6 @@ Gemfile.lock
 /_site
 _site
 *.tmp
-/docs/.jekyll-metadata
+/docs/.jekyll-metadata
+coverage.txt
+cspell.json

.golangci.yml

@@ -16,4 +16,7 @@ issues:
 
 linters-settings:
   errcheck:
-    exclude: ./errcheck_excludes.txt
+    exclude-functions:
+      - "fmt.Fprintf"
+      - "fmt.Fprint"
+      - "(net/http.ResponseWriter).Write"

CODE_OF_CONDUCT_RU.md

@@ -1,120 +0,0 @@
-
-# Кодекс Поведения участника
-
-## Наши обязательства
-
-Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
-свободным от притеснений для всех, независимо от возраста, телосложения,
-видимых или невидимых ограничений способности, этнической принадлежности,
-половых признаков, гендерной идентичности и выражения, уровня опыта,
-образования, социо-экономического статуса, национальности, внешности,
-расы, религии, или сексуальной идентичности и ориентации.
-
-Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
-дружелюбное, многообразное, инклюзивное и здоровое сообщество.
-
-## Наши стандарты
-
-Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
-
-* Проявление доброты и эмпатии к другим участникам проекта
-* Уважение к чужой точке зрения и опыту
-* Конструктивная критика и принятие конструктивной критики
-* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
-  и извлечение уроков из опыта
-* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
-
-Примеры неприемлемого поведения участников включают в себя:
-
-* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
-* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
-* Публичное или приватное домогательство
-* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
-* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
-
-## Обязанности
-
-Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
-поведения и будут предпринимать соответствующие и честные меры по исправлению положения
-в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
-
-Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
-комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
-с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
-
-## Область применения
-
-Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
-а также когда человек официально представляет сообщество в публичных местах.
-Примеры представления проекта или сообщества включают использование официальной электронной почты,
-публикации в официальном аккаунте в социальных сетях,
-или упоминания как представителя в онлайн или оффлайн мероприятии.
-
-## Приведение в исполнение
-
-О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
-поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
-Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
-
-Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
-неприкосновенность автора сообщения.
-
-## Руководство по исполнению
-
-Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
-чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
-
-### 1. Исправление
-
-**Общественное влияние**: Использование недопустимой лексики или другое поведение,
-считающиеся непрофессиональным или нежелательным в сообществе.
-
-**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
-объясняющее суть нарушения и почему такое поведение
-было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
-
-### 2. Предупреждение
-
-**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
-
-**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
-На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
-включая незапрошенное взаимодействие
-с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
-в публичных пространствах, а так же во внешних каналах,
-таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
-
-### 3. Временный бан
-
-**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
-включая продолжительное неуместное поведение.
-
-**Последствия**: Временный запрет (бан) на любое взаимодействие
-или публичное общение с сообществом на определенный период времени.
-На этот период не допускается публичное или личное взаимодействие с людьми,
-вовлеченными в инцидент, включая незапрошенное взаимодействие
-с теми, кто обеспечивает соблюдение Кодекса.
-Нарушение этих правил влечет за собой вечный бан.
-
-### 4. Вечный бан
-
-**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
-включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
-или проявление агрессии либо пренебрежительного отношения к категориям лиц.
-
-**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
-
-## Атрибуция
-
-Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
-версии 2.0, доступной по адресу
-<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
-
-Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
-
-[homepage]: https://www.contributor-covenant.org
-
-Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
-<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
-<https://www.contributor-covenant.org/translations>.

Makefile

@@ -14,15 +14,86 @@ endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 
+VICTORIA_LOGS_COMPONENTS = victoria-logs
+VICTORIA_METRICS_COMPONENTS = victoria-metrics
+VICTORIA_METRICS_UTILS_COMPONENTS = vmagent vmalert vmalert-tool vmauth vmbackup vmrestore vmctl
+
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
+include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
-include snap/local/Makefile
 include package/release/Makefile
 
+define RELEASE_GOOS_GOARCH
+	$(eval PKG_NAME := $(1))
+	$(eval PKG_COMPONENTS := $(2))
+
+	# Build
+	$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+
+	# Generate SBOM
+	mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cyclonedx-gomod app -assert-licenses -json -licenses -packages \
+			-main app/$(pkg_component) \
+			-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
+	
+	# Pack and compress
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar \
+			$(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+	cd bin && gzip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar
+	cd bin && tar -czf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+
+	# Generate checksums
+	cd bin && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ >> \
+			$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
+
+	# Clean up
+	cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+	$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+endef
+
+define RELEASE_WINDOWS_GOARCH
+	$(eval PKG_NAME := $(1))
+	$(eval PKG_COMPONENTS := $(2))
+
+	# Build
+	$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+
+	# Generate SBOM
+	mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cyclonedx-gomod app -assert-licenses -json -licenses -packages \
+			-main app/$(pkg_component) \
+			-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
+	
+	# Pack and compress
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && zip -u $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip \
+			$(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
+	cd bin && zip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip -r $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+
+	# Generate checksums
+	cd bin && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
+
+	# Clean up
+	cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+	$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
+endef
+
+
 all: \
 	victoria-metrics-prod \
 	victoria-logs-prod \
@@ -241,26 +312,13 @@ release-victoria-metrics-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
 release-victoria-metrics-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
+	GOOS=windows GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
 
-release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
+release-victoria-metrics-goos-goarch:
+	$(call RELEASE_GOOS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
 
-release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-			> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		victoria-metrics-windows-$(GOARCH)-prod.exe
+release-victoria-metrics-windows-goarch:
+	$(call RELEASE_WINDOWS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
 
 release-victoria-logs:
 	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
@@ -355,77 +413,13 @@ release-vmutils-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
+	GOOS=windows GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
 
-release-vmutils-goos-goarch: \
-	vmagent-$(GOOS)-$(GOARCH)-prod \
-	vmalert-$(GOOS)-$(GOARCH)-prod \
-	vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-	vmauth-$(GOOS)-$(GOARCH)-prod \
-	vmbackup-$(GOOS)-$(GOARCH)-prod \
-	vmrestore-$(GOOS)-$(GOARCH)-prod \
-	vmctl-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOOS)-$(GOARCH)-prod \
-			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-			vmauth-$(GOOS)-$(GOARCH)-prod \
-			vmbackup-$(GOOS)-$(GOARCH)-prod \
-			vmrestore-$(GOOS)-$(GOARCH)-prod \
-			vmctl-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOOS)-$(GOARCH)-prod \
-			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-			vmauth-$(GOOS)-$(GOARCH)-prod \
-			vmbackup-$(GOOS)-$(GOARCH)-prod \
-			vmrestore-$(GOOS)-$(GOARCH)-prod \
-			vmctl-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vmagent-$(GOOS)-$(GOARCH)-prod \
-		vmalert-$(GOOS)-$(GOARCH)-prod \
-		vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-		vmauth-$(GOOS)-$(GOARCH)-prod \
-		vmbackup-$(GOOS)-$(GOARCH)-prod \
-		vmrestore-$(GOOS)-$(GOARCH)-prod \
-		vmctl-$(GOOS)-$(GOARCH)-prod
+release-vmutils-goos-goarch:
+	$(call RELEASE_GOOS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
 
-release-vmutils-windows-goarch: \
-	vmagent-windows-$(GOARCH)-prod \
-	vmalert-windows-$(GOARCH)-prod \
-	vmalert-tool-windows-$(GOARCH)-prod \
-	vmauth-windows-$(GOARCH)-prod \
-	vmbackup-windows-$(GOARCH)-prod \
-	vmrestore-windows-$(GOARCH)-prod \
-	vmctl-windows-$(GOARCH)-prod
-	cd bin && \
-		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vmagent-windows-$(GOARCH)-prod.exe \
-			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
-			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
-			vmctl-windows-$(GOARCH)-prod.exe \
-		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vmagent-windows-$(GOARCH)-prod.exe \
-			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
-			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
-			vmctl-windows-$(GOARCH)-prod.exe \
-			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vmagent-windows-$(GOARCH)-prod.exe \
-		vmalert-windows-$(GOARCH)-prod.exe \
-		vmalert-tool-windows-$(GOARCH)-prod.exe \
-		vmauth-windows-$(GOARCH)-prod.exe \
-		vmbackup-windows-$(GOARCH)-prod.exe \
-		vmrestore-windows-$(GOARCH)-prod.exe \
-		vmctl-windows-$(GOARCH)-prod.exe
+release-vmutils-windows-goarch:
+	$(call RELEASE_WINDOWS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
 
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
@@ -440,6 +434,8 @@ vet:
 
 check-all: fmt vet golangci-lint govulncheck
 
+clean-checkers: remove-golangci-lint remove-govulncheck
+
 test:
 	go test ./lib/... ./app/...
 
@@ -492,7 +488,10 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.57.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
+
+remove-golangci-lint:
+	rm -rf `which golangci-lint`
 
 govulncheck: install-govulncheck
 	govulncheck ./...
@@ -500,12 +499,18 @@ govulncheck: install-govulncheck
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 
+remove-govulncheck:
+	rm -rf `which govulncheck`
+
 install-wwhrd:
 	which wwhrd || go install github.com/frapposelli/wwhrd@latest
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
 
+cyclonedx-gomod-install:
+	which cyclonedx-gomod || go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest
+
 copy-docs:
 # The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
 # For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
@@ -523,6 +528,7 @@ copy-docs:
 	echo "---" >> ${DST}
 	cat ${SRC} >> ${DST}
 	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"/' ${DST}
 	rm -rf docs/*.tmp
 
 # Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.

README.md

@@ -2,21 +2,23 @@
 
 [![Latest Release](https://img.shields.io/github/release/VictoriaMetrics/VictoriaMetrics.svg?style=flat-square)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics.svg?maxAge=604800)](https://hub.docker.com/r/victoriametrics/victoria-metrics)
-[![victoriametrics](https://snapcraft.io/victoriametrics/badge.svg)](https://snapcraft.io/victoriametrics)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)
 [![GitHub license](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/workflows/main/badge.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions)
 [![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 
-<img src="docs/logo.webp" width="300" alt="VictoriaMetrics logo">
+<picture>
+  <source srcset="docs/logo_white.webp" media="(prefers-color-scheme: dark)">
+  <source srcset="docs/logo.webp" media="(prefers-color-scheme: light)">
+  <img src="docs/logo.webp" width="300" alt="VictoriaMetrics logo">
+</picture>
 
 VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.
 See [case studies for VictoriaMetrics](https://docs.victoriametrics.com/casestudies/).
 
 VictoriaMetrics is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
-[Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap packages](https://snapcraft.io/victoriametrics)
-and [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). 
+[Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). 
 
 Documentation for the cluster version of VictoriaMetrics is available [here](https://docs.victoriametrics.com/cluster-victoriametrics/).
 
@@ -86,7 +88,6 @@ VictoriaMetrics has the following prominent features:
   * [Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format).
   * [InfluxDB line protocol](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) over HTTP, TCP and UDP.
   * [Graphite plaintext protocol](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon).
-  * [Statsd plaintext protocol](#how-to-send-data-from-statsd-compatible-clients)
   * [OpenTSDB put message](#sending-data-via-telnet-put-protocol).
   * [HTTP OpenTSDB /api/put requests](#sending-opentsdb-data-via-http-apiput-requests).
   * [JSON line format](#how-to-import-data-in-json-line-format).
@@ -117,12 +118,12 @@ VictoriaMetrics ecosystem contains the following components additionally to [sin
 - [vmalert](https://docs.victoriametrics.com/vmalert/) - a service for processing Prometheus-compatible alerting and recording rules.
 - [vmalert-tool](https://docs.victoriametrics.com/vmalert-tool/) -  a tool for validating alerting and recording rules.
 - [vmauth](https://docs.victoriametrics.com/vmauth/) - authorization proxy and load balancer optimized for VictoriaMetrics products.
-- [vmgateway](https://docs.victoriametrics.com/vmgateway/) - authorization proxy with per-[tenant](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy) rate limiting cababilities.
+- [vmgateway](https://docs.victoriametrics.com/vmgateway/) - authorization proxy with per-[tenant](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy) rate limiting capabilities.
 - [vmctl](https://docs.victoriametrics.com/vmctl/) - a tool for migrating and copying data between different storage systems for metrics.
 - [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/) and [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/) -
   tools for creating backups and restoring from backups for VictoriaMetrics data.
 - `vminsert`, `vmselect` and `vmstorage` - components of [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/).
-- [VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/) - user-friendly cost-efficient database for logs.
+- [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) - user-friendly cost-efficient database for logs.
 
 ## Operation
 
@@ -139,7 +140,6 @@ VictoriaMetrics can also be installed via these installation methods:
 * [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
 * [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
 * [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
-* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
 
 ### How to start VictoriaMetrics
 
@@ -180,27 +180,6 @@ Additionally, all the VictoriaMetrics components allow setting flag values via e
 * For repeating flags an alternative syntax can be used by joining the different values into one using `,` char as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`).
 * Environment var prefix can be set via `-envflag.prefix` flag. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`.
 
-### Configuration with snap package
-
-Snap package for VictoriaMetrics is available [here](https://snapcraft.io/victoriametrics).
-
-Command-line flags for Snap package can be set with following command:
-
-```sh
-echo 'FLAGS="-selfScrapeInterval=10s -search.logSlowQueryDuration=20s"' > $SNAP_DATA/var/snap/victoriametrics/current/extra_flags
-snap restart victoriametrics
-```
-
-Do not change value for `-storageDataPath` flag, because snap package has limited access to host filesystem.
-
-Changing scrape configuration is possible with text editor:
-
-```sh
-vi $SNAP_DATA/var/snap/victoriametrics/current/etc/victoriametrics-scrape-config.yaml
-```
-
-After changes were made, trigger config re-read with the command `curl 127.0.0.1:8428/-/reload`.
-
 ### Running as Windows service
 
 In order to run VictoriaMetrics as a Windows service it is required to create a service configuration for [WinSW](https://github.com/winsw/winsw)
@@ -338,8 +317,8 @@ This allows Grafana to use a more efficient API to get label values.
 Then build graphs and dashboards for the created datasource using [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/)
 or [MetricsQL](https://docs.victoriametrics.com/metricsql/).
 
-Alternatively, use VictoriaMetrics [datasource plugin](https://github.com/VictoriaMetrics/grafana-datasource) with support of extra features. 
-See more in [description](https://github.com/VictoriaMetrics/grafana-datasource#victoriametrics-data-source-for-grafana).
+Alternatively, use VictoriaMetrics [datasource plugin](https://github.com/VictoriaMetrics/victoriametrics-datasource) with support of extra features. 
+See more in [description](https://docs.victoriametrics.com/victoriametrics-datasource/).
 
 Creating a datasource may require [specific permissions](https://grafana.com/docs/grafana/latest/administration/data-source-management/).
 If you don't see an option to create a data source - try contacting system administrator.
@@ -702,45 +681,6 @@ The `/api/v1/export` endpoint should return the following response:
 {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[1.23],"timestamps":[1695902762311]}
 ```
 
-## How to send data from Statsd-compatible clients
-
-VictoriaMetrics supports extended statsd protocol with tags. Also it does not support sampling and metric types(it will be ignored).  
-Enable Statsd receiver in VictoriaMetrics by setting `-statsdListenAddr` command line flag. For instance,
-the following command will enable Statsd receiver in VictoriaMetrics on TCP and UDP port `8125`:
-
-```console
-/path/to/victoria-metrics-prod -statsdListenAddr=:8125
-```
-
-Example for writing data with Statsd plaintext protocol to local VictoriaMetrics using `nc`:
-
-```console
-echo "foo.bar:123|g|#foo:bar" | nc -N localhost 8125
-```
-
-Explicit setting of timestamps is not supported for statsd protocol. Timestamp is set to the current time when VictoriaMetrics or vmagent receives it.
-
-An arbitrary number of lines delimited by `\n` (aka newline char) can be sent in one go.
-After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
-
-<div class="with-copy" markdown="1">
-
-```console
-curl -G 'http://localhost:8428/api/v1/export' -d 'match=foo.bar.baz'
-```
-
-</div>
-
-The `/api/v1/export` endpoint should return the following response:
-
-```json
-{"metric":{"__name__":"foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1560277406000]}
-```
-
-Some examples of compatible statsd clients:
-- [statsd-instrument](https://github.com/Shopify/statsd-instrument)
-- [dogstatsd-ruby](https://github.com/DataDog/dogstatsd-ruby)
-- [go-statsd-client](https://github.com/cactus/go-statsd-client)
 ## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
 
 Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,
@@ -759,6 +699,13 @@ Example for writing data with Graphite plaintext protocol to local VictoriaMetri
 echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N localhost 2003
 ```
 
+To sanitize ingested metric names and labels according to Prometheus naming convention enable
+`-graphite.sanitizeMetricName` cmd-line flag. When enabled, VictoriaMetrics will apply the following modifications:
+- replace `/`,`@`,`*` with `_`;
+- drop `\`;
+- remove redundant dots, e.g: `metric..name` => `metric.name`;
+- replace characters not matching the expression `^a-zA-Z0-9:._` with `_`.
+
 VictoriaMetrics sets the current time if the timestamp is omitted.
 An arbitrary number of lines delimited by `\n` (aka newline char) can be sent in one go.
 After that the data may be read via [/api/v1/export](#how-to-export-data-in-json-line-format) endpoint:
@@ -1154,13 +1101,13 @@ ROOT_IMAGE=scratch make package-victoria-metrics
 
 VictoriaMetrics can be built with Podman in either rootful or rootless mode.
 
-When building via rootlful Podman, simply add `DOCKER=podman` to the relevant `make` commandline.  To build
+When building via rootful Podman, simply add `DOCKER=podman` to the relevant `make` commandline.  To build
 via rootless Podman, add `DOCKER=podman DOCKER_RUN="podman run --userns=keep-id"` to the `make`
 commandline.
 
 For example: `make victoria-metrics-pure DOCKER=podman DOCKER_RUN="podman run --userns=keep-id"`
 
-Note that `production` builds are not supported via Podman becuase Podman does not support `buildx`.
+Note that `production` builds are not supported via Podman because Podman does not support `buildx`.
 
 ## Start with docker-compose
 
@@ -1171,7 +1118,7 @@ More details may be found [here](https://github.com/VictoriaMetrics/VictoriaMetr
 ## Setting up service
 
 Read [instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics
-as a service for your OS. A [snap package](https://snapcraft.io/victoriametrics) is available for Ubuntu.
+as a service for your OS.
 
 ## How to work with snapshots
 
@@ -1396,7 +1343,6 @@ Additionally, VictoriaMetrics can accept metrics via the following popular data
 * DataDog `submit metrics` API. See [these docs](#how-to-send-data-from-datadog-agent) for details.
 * InfluxDB line protocol. See [these docs](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details.
 * Graphite plaintext protocol. See [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
-* Statsd plaintext protocol. See [these docs](#how-to-send-data-from-statsd-compatible-clients) for details.
 * OpenTelemetry http API. See [these docs](#sending-data-via-opentelemetry) for details.
 * OpenTSDB telnet put protocol. See [these docs](#sending-data-via-telnet-put-protocol) for details.
 * OpenTSDB http `/api/put` protocol. See [these docs](#sending-opentsdb-data-via-http-apiput-requests) for details.
@@ -1590,6 +1536,25 @@ Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed da
 VictoriaMetrics stores the ingested OpenTelemetry [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) as is without any transformations.
 Pass `-opentelemetry.usePrometheusNaming` command-line flag to VictoriaMetrics for automatic conversion of metric names and labels into Prometheus-compatible format.
 
+Using the following exporter configuration in the opentelemetry collector will allow you to send metrics into VictoriaMetrics:
+
+```yaml
+exporters:
+  otlphttp/victoriametrics:
+    compression: gzip
+    encoding: proto
+    endpoint: http://<collector/vmagent>.<namespace>.svc.cluster.local:<port>/opentelemetry
+```
+Remember to add the exporter to the desired service pipeline in order to activate the exporter.
+```yaml
+service:
+  pipelines:
+    metrics:
+      exporters:
+        - otlphttp/victoriametrics
+      receivers:
+        - otlp
+```
 See [How to use OpenTelemetry metrics with VictoriaMetrics](https://docs.victoriametrics.com/guides/getting-started-with-opentelemetry/).
 
 ## JSON line format
@@ -2370,7 +2335,7 @@ and [cardinality explorer docs](#cardinality-explorer).
 
 * If you run VictoriaMetrics on a host with 16 or more CPU cores, then it may be needed to tune the `-search.maxWorkersPerQuery` command-line flag
   in order to improve query performance. If VictoriaMetrics serves big number of concurrent `select` queries, then try reducing the value for this flag.
-  If VcitoriaMetrics serves heavy queries, which select `>10K` of [time series](https://docs.victoriametrics.com/keyconcepts/#time-series) and/or process `>100M`
+  If VictoriaMetrics serves heavy queries, which select `>10K` of [time series](https://docs.victoriametrics.com/keyconcepts/#time-series) and/or process `>100M`
   of [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) per query, then try setting the value for this flag to the number of available CPU cores.
 
 * VictoriaMetrics buffers incoming data in memory for up to a few seconds before flushing it to persistent storage.
@@ -2396,7 +2361,8 @@ and [cardinality explorer docs](#cardinality-explorer).
 * If VictoriaMetrics doesn't work because of certain parts are corrupted due to disk errors,
   then just remove directories with broken parts. It is safe removing subdirectories under `<-storageDataPath>/data/{big,small}/YYYY_MM` directories
   when VictoriaMetrics isn't running. This recovers VictoriaMetrics at the cost of data loss stored in the deleted broken parts.
-  In the future, `vmrecover` tool will be created for automatic recovering from such errors.
+  The names of broken parts should be present in the error message. If you see that error message is truncated and doesn't contain all the information
+  try increasing `-loggerMaxArgLen` cmd-line flag to higher values to avoid error messages truncation.
 
 * If you see gaps on the graphs, try resetting the cache by sending request to `/internal/resetRollupResultCache`.
   If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset`
@@ -2416,7 +2382,7 @@ and [cardinality explorer docs](#cardinality-explorer).
 * New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics.
 
 * VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag
-  and drops superflouos labels. This prevents from ingesting metrics with too many labels.
+  and drops superfluous labels. This prevents from ingesting metrics with too many labels.
   It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
   metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
 
@@ -2478,7 +2444,7 @@ It is also possible removing [rollup result cache](#rollup-result-cache) on star
 
 ## Rollup result cache
 
-VictoriaMetrics caches query responses by default. This allows increasing performance for repated queries
+VictoriaMetrics caches query responses by default. This allows increasing performance for repeated queries
 to [`/api/v1/query`](https://docs.victoriametrics.com/keyconcepts/#instant-query) and [`/api/v1/query_range`](https://docs.victoriametrics.com/keyconcepts/#range-query)
 with the increasing `time`, `start` and `end` query args.
 
@@ -2672,10 +2638,39 @@ If you like VictoriaMetrics and want to contribute, then please [read these docs
 
 Report bugs and propose new features [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
 
-## Images in documentation
+## Documentation
+
+VictoriaMetrics documentation is available at [https://docs.victoriametrics.com/](https://docs.victoriametrics.com/).
+It is built from `*.md` files located in [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) folder
+and gets automatically updated once changes are merged to [master](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master) branch.
+To update the documentation follow the steps below:
+- [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) 
+VictoriaMetrics repo and apply changes to the docs:
+  - To update [the main page](https://docs.victoriametrics.com/) modify [this file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md).
+  - To update other pages, apply changes to the corresponding file in [docs folder](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs).
+- If your changes contain an image then see [images in documentation](https://docs.victoriametrics.com/#images-in-documentation).
+- Once changes are made, execute the command below to finalize and sync the changes:
+```sh
+make docs-sync
+```
+- Create [a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request)
+with proposed changes and wait for it to be merged.
+
+Requirements for changes to docs:
+- Keep backward compatibility of existing links. Avoid changing anchors or deleting pages as they could have been
+used or posted in other docs, GitHub issues, stackoverlow answers, etc. 
+- Keep docs simple. Try using as simple wording as possible.
+- Keep docs consistent. When modifying existing docs, verify that other places referencing to this doc are still relevant.
+- Prefer improving the existing docs instead of adding new ones.
+- Use absolute links.
+
+### Images in documentation
 
 Please, keep image size and number of images per single page low. Keep the docs page as lightweight as possible.
 
+Image files must be placed in the same folder as the doc itself and they must have the same prefix as the doc filename.
+For example, all the images for `docs/foo/bar.md` should have filenames starting from `docs/foo/bar`.
+
 If the page needs to have many images, consider using WEB-optimized image format [webp](https://developers.google.com/speed/webp).
 When adding a new doc with many images use `webp` format right away. Or use a Makefile command below to
 convert already existing images at `docs` folder automatically to `web` format:
@@ -2728,7 +2723,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -cacheExpireDuration duration
      Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s)
   -configAuthKey value
-     Authorization key for accessing /config page. It must be passed via authKey query arg
+     Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -configAuthKey=file:///abs/path/to/file or -configAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -configAuthKey=http://host/path or -configAuthKey=https://host/path
   -csvTrimTimestamp duration
      Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
@@ -2765,7 +2760,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -finalMergeDelay duration
      Deprecated: this flag does nothing
   -flagsAuthKey value
-     Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
   -forceFlushAuthKey value
      authKey, which must be passed in query string to /internal/force_flush pages
@@ -2775,6 +2770,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Flag value can be read from the given file when using -forceMergeAuthKey=file:///abs/path/to/file or -forceMergeAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -forceMergeAuthKey=http://host/path or -forceMergeAuthKey=https://host/path
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
+  -graphite.sanitizeMetricName
+     Sanitize metric names for the ingested Graphite data. See https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd
   -graphiteListenAddr string
      TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. See also -graphiteListenAddr.useProxyProtocol
   -graphiteListenAddr.useProxyProtocol
@@ -2847,9 +2844,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -internStringMaxLen int
      The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -license string
-     Lisense key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed via file specified by -licenseFile command-line flag
+     License key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed via file specified by -licenseFile command-line flag
   -license.forceOffline
-     Whether to enable offline verification for VictoriaMetrics Enterprise license key, which has been passed either via -license or via -licenseFile command-line flag. The issued license key must support offline verification feature. Contact info@victoriametrics.com if you need offline license verification. This flag is avilable only in Enterprise binaries
+     Whether to enable offline verification for VictoriaMetrics Enterprise license key, which has been passed either via -license or via -licenseFile command-line flag. The issued license key must support offline verification feature. Contact info@victoriametrics.com if you need offline license verification. This flag is available only in Enterprise binaries
   -licenseFile string
      Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/ . Trial Enterprise license can be obtained from https://victoriametrics.com/products/enterprise/trial/ . This flag is available only in Enterprise binaries. The license key can be also passed inline via -license command-line flag
   -logNewSeries
@@ -2878,7 +2875,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
   -maxLabelValueLen int
-     The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 16384)
+     The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 4096)
   -maxLabelsPerTimeseries int
      The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
   -memory.allowedBytes size
@@ -2889,7 +2886,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -metrics.exposeMetadata
      Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type
   -metricsAuthKey value
-     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -metricsAuthKey=file:///abs/path/to/file or -metricsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -metricsAuthKey=http://host/path or -metricsAuthKey=https://host/path
   -mtls array
      Whether to require valid client certificate for https requests to the corresponding -httpListenAddr . This flag works only if -tls flag is set. See also -mtlsCAFile . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/enterprise/
@@ -2920,7 +2917,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -opentsdbhttpTrimTimestamp duration
      Trim timestamps for OpenTSDB HTTP data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -pprofAuthKey value
-     Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
+     Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -pprofAuthKey=file:///abs/path/to/file or -pprofAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path
   -precisionBits int
      The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss (default 64)
@@ -2993,7 +2990,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.kumaSDCheckInterval duration
      Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#kuma_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int
-     The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000)
+     The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 10000)
   -promscrape.maxResponseHeadersSize size
      The maximum size of http response headers from Prometheus scrape targets
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 4096)
@@ -3042,7 +3039,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -relabelConfig string
      Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal
   -reloadAuthKey value
-     Auth key for /-/reload http endpoint. It must be passed as authKey=...
+     Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path
   -retentionFilter array
      Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise/
@@ -3059,14 +3056,18 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Whether to disable automatic response cache reset if a sample with timestamp outside -search.cacheTimestampOffset is inserted into VictoriaMetrics
   -search.disableCache
      Whether to disable response caching. This may be useful when ingesting historical data. See https://docs.victoriametrics.com/#backfilling . See also -search.resetRollupResultCacheOnStartup
+  -search.disableImplicitConversion
+     Whether to return an error for queries that rely on implicit subquery conversions, see https://docs.victoriametrics.com/metricsql/#subqueries for details. See also -search.logImplicitConversion
   -search.graphiteMaxPointsPerSeries int
      The maximum number of points per series Graphite render API can return (default 1000000)
   -search.graphiteStorageStep duration
      The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API (default 10s)
   -search.ignoreExtraFiltersAtLabelsAPI
-     Whether to ignore match[], extra_filters[] and extra_label query args at /api/v1/labels and /api/v1/label/.../values . This may be useful for decreasing load on VictoriaMetrics when extra filters match too many time series. The downside is that suprflouos labels or series could be returned, which do not match the extra filters. See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration
+     Whether to ignore match[], extra_filters[] and extra_label query args at /api/v1/labels and /api/v1/label/.../values . This may be useful for decreasing load on VictoriaMetrics when extra filters match too many time series. The downside is that superfluous labels or series could be returned, which do not match the extra filters. See also -search.maxLabelsAPISeries and -search.maxLabelsAPIDuration
   -search.latencyOffset duration
      The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
+  -search.logImplicitConversion
+     Whether to log queries with implicit subquery conversions, see https://docs.victoriametrics.com/metricsql/#subqueries for details. Such conversion can be disabled using -search.disableImplicitConversion
   -search.logQueryMemoryUsage size
      Log query and increment vm_memory_intensive_queries_total metric each time the query requires more memory than specified by this flag. This may help detecting and optimizing heavy queries. Query logging is disabled by default. See also -search.logSlowQueryDuration and -search.maxMemoryPerQuery
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -3089,7 +3090,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxLabelsAPIDuration duration
      The maximum duration for /api/v1/labels, /api/v1/label/.../values and /api/v1/series requests. See also -search.maxLabelsAPISeries and -search.ignoreExtraFiltersAtLabelsAPI (default 5s)
   -search.maxLabelsAPISeries int
-     The maximum number of time series, which could be scanned when searching for the the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
+     The maximum number of time series, which could be scanned when searching for the matching time series at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, -search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI (default 1000000)
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
   -search.maxMemoryPerQuery size
@@ -3100,7 +3101,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -search.maxPointsSubqueryPerTimeseries int
      The maximum number of points per series, which can be generated by subquery. See https://valyala.medium.com/prometheus-subqueries-in-victoriametrics-9b1492b720b3 (default 100000)
   -search.maxQueryDuration duration
-     The maximum duration for query execution (default 30s)
+     The maximum duration for query execution. It can be overridden on a per-query basis via 'timeout' query arg (default 30s)
   -search.maxQueryLen size
      The maximum search query length in bytes
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16384)

app/victoria-logs/Makefile

@@ -81,6 +81,9 @@ victoria-logs-linux-ppc64le:
 victoria-logs-linux-s390x:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
+victoria-logs-linux-loong64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+
 victoria-logs-linux-386:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
@@ -101,3 +104,10 @@ victoria-logs-windows-amd64:
 
 victoria-logs-pure:
 	APP_NAME=victoria-logs $(MAKE) app-local-pure
+
+run-victoria-logs:
+	mkdir -p victoria-logs-data
+	DOCKER_OPTS='-v $(shell pwd)/victoria-logs-data:/victoria-logs-data' \
+	APP_NAME=victoria-logs \
+	ARGS='' \
+	$(MAKE) run-via-docker

app/victoria-logs/main.go

@@ -47,7 +47,7 @@ func main() {
 	vlinsert.Init()
 
 	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
-	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
+	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
@@ -77,7 +77,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/VictoriaLogs/'>https://docs.victoriametrics.com/VictoriaLogs/</a></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/'>https://docs.victoriametrics.com/victorialogs/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"select/vmui", "Web UI for VictoriaLogs"},
@@ -99,7 +99,7 @@ func usage() {
 	const s = `
 victoria-logs is a log management and analytics service.
 
-See the docs at https://docs.victoriametrics.com/VictoriaLogs/
+See the docs at https://docs.victoriametrics.com/victorialogs/
 `
 	flagutil.Usage(s)
 }

app/victoria-logs/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/victoria-metrics/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -117,7 +118,7 @@ func (q *Query) metrics() ([]Metric, error) {
 type QueryInstant struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TV     [2]interface{}    `json:"value"`
+		TV     [2]any            `json:"value"`
 	} `json:"result"`
 }
 
@@ -140,7 +141,7 @@ func (q QueryInstant) metrics() ([]Metric, error) {
 type QueryRange struct {
 	Result []struct {
 		Metric map[string]string `json:"metric"`
-		Values [][]interface{}   `json:"values"`
+		Values [][]any           `json:"values"`
 	} `json:"result"`
 }
 
@@ -249,24 +250,24 @@ func TestWriteRead(t *testing.T) {
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
-		for _, test := range readIn("prometheus", t, insertionTime) {
+		for _, test := range readIn("prometheus", insertionTime) {
 			if test.Data == nil {
 				continue
 			}
-			s := newSuite(t)
 			r := testutil.WriteRequest{}
-			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
-			data, err := testutil.Compress(r)
-			s.greaterThan(len(r.Timeseries), 0)
-			if err != nil {
-				t.Errorf("error compressing %v %s", r, err)
-				t.Fail()
+			testData := strings.Join(test.Data, "\n")
+			if err := json.Unmarshal([]byte(testData), &r.Timeseries); err != nil {
+				panic(fmt.Errorf("BUG: cannot unmarshal TimeSeries: %s\ntest data\n%s", err, testData))
 			}
+			if n := len(r.Timeseries); n <= 0 {
+				panic(fmt.Errorf("BUG: expecting non-empty Timeseries in test data:\n%s", testData))
+			}
+			data := testutil.Compress(r)
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
 	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", t, insertionTime) {
+		for _, test := range readIn("csv", insertionTime) {
 			if test.Data == nil {
 				continue
 			}
@@ -275,7 +276,7 @@ func testWrite(t *testing.T) {
 	})
 
 	t.Run("influxdb", func(t *testing.T) {
-		for _, x := range readIn("influxdb", t, insertionTime) {
+		for _, x := range readIn("influxdb", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -284,7 +285,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("graphite", func(t *testing.T) {
-		for _, x := range readIn("graphite", t, insertionTime) {
+		for _, x := range readIn("graphite", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -293,7 +294,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdb", func(t *testing.T) {
-		for _, x := range readIn("opentsdb", t, insertionTime) {
+		for _, x := range readIn("opentsdb", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -302,7 +303,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdbhttp", func(t *testing.T) {
-		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
+		for _, x := range readIn("opentsdbhttp", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -316,7 +317,7 @@ func testWrite(t *testing.T) {
 func testRead(t *testing.T) {
 	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
-			for _, x := range readIn(engine, t, insertionTime) {
+			for _, x := range readIn(engine, insertionTime) {
 				test := x
 				t.Run(test.Name, func(t *testing.T) {
 					t.Parallel()
@@ -365,11 +366,10 @@ func testRead(t *testing.T) {
 	}
 }
 
-func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
-	t.Helper()
-	s := newSuite(t)
+func readIn(readFor string, insertTime time.Time) []test {
+	testDir := filepath.Join(testFixturesDir, readFor)
 	var tt []test
-	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
+	err := filepath.Walk(testDir, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -377,84 +377,129 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 			return nil
 		}
 		b, err := os.ReadFile(path)
-		s.noError(err)
+		if err != nil {
+			panic(fmt.Errorf("BUG: cannot read %s: %s", path, err))
+		}
 		item := test{}
-		s.noError(json.Unmarshal(b, &item))
+		if err := json.Unmarshal(b, &item); err != nil {
+			panic(fmt.Errorf("cannot parse %T from %s: %s; data:\n%s", &item, path, err, b))
+		}
 		for i := range item.Data {
 			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
 		}
 		tt = append(tt, item)
 		return nil
-	}))
+	})
+	if err != nil {
+		panic(fmt.Errorf("BUG: cannot read test data at %s: %w", testDir, err))
+	}
 	if len(tt) == 0 {
-		t.Fatalf("no test found in %s", filepath.Join(testFixturesDir, readFor))
+		panic(fmt.Errorf("BUG: no tests found in %s", testDir))
 	}
 	return tt
 }
 
 func httpWrite(t *testing.T, address, query string, r io.Reader) {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Post(address+query, "", r)
-	s.noError(err)
-	s.noError(resp.Body.Close())
-	s.equalInt(resp.StatusCode, 204)
+
+	requestURL := address + query
+	resp, err := http.Post(requestURL, "", r)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != 204 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 204", requestURL, resp.StatusCode)
+	}
 }
 
-func tcpWrite(t *testing.T, address string, data string) {
+func tcpWrite(t *testing.T, address, data string) {
 	t.Helper()
-	s := newSuite(t)
+
 	conn, err := net.Dial("tcp", address)
-	s.noError(err)
+	if err != nil {
+		t.Fatalf("cannot dial %s: %s", address, err)
+	}
 	defer func() {
 		_ = conn.Close()
 	}()
 	n, err := conn.Write([]byte(data))
-	s.noError(err)
-	s.equalInt(n, len(data))
+	if err != nil {
+		t.Fatalf("cannot write %d bytes to %s: %s", len(data), address, err)
+	}
+	if n != len(data) {
+		panic(fmt.Errorf("BUG: conn.Write() returned unexpected number of written bytes to %s; got %d; want %d", address, n, len(data)))
+	}
 }
 
 func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+
 	var rows []Metric
-	for dec := json.NewDecoder(resp.Body); dec.More(); {
+	dec := json.NewDecoder(resp.Body)
+	for {
 		var row Metric
-		s.noError(dec.Decode(&row))
+		err := dec.Decode(&row)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return rows
+			}
+			t.Fatalf("cannot decode %T from response received from %s: %s", &row, requestURL, err)
+		}
 		rows = append(rows, row)
 	}
-	return rows
 }
 
-func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
+func httpReadStruct(t *testing.T, address, query string, dst any) {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
-	s.noError(json.NewDecoder(resp.Body).Decode(dst))
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+	err = json.NewDecoder(resp.Body).Decode(dst)
+	if err != nil {
+		t.Fatalf("cannot decode %T from response received from %s: %s", dst, requestURL, err)
+	}
 }
 
 func httpReadData(t *testing.T, address, query string) []byte {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
 	data, err := io.ReadAll(resp.Body)
-	s.noError(err)
+	if err != nil {
+		t.Fatalf("cannot read response from %s: %s", requestURL, err)
+	}
 	return data
 }
 
@@ -503,34 +548,6 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
-type suite struct{ t *testing.T }
-
-func newSuite(t *testing.T) *suite { return &suite{t: t} }
-
-func (s *suite) noError(err error) {
-	s.t.Helper()
-	if err != nil {
-		s.t.Errorf("unexpected error %v", err)
-		s.t.FailNow()
-	}
-}
-
-func (s *suite) equalInt(a, b int) {
-	s.t.Helper()
-	if a != b {
-		s.t.Errorf("%d not equal %d", a, b)
-		s.t.FailNow()
-	}
-}
-
-func (s *suite) greaterThan(a, b int) {
-	s.t.Helper()
-	if a <= b {
-		s.t.Errorf("%d less or equal then %d", a, b)
-		s.t.FailNow()
-	}
-}
-
 func TestImportJSONLines(t *testing.T) {
 	f := func(labelsCount, labelLen int) {
 		t.Helper()

app/victoria-metrics/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/victoria-metrics/test/prom_writer.go

@@ -1,12 +1,16 @@
 package test
 
-import "github.com/golang/snappy"
+import (
+	"fmt"
+
+	"github.com/golang/snappy"
+)
 
 // Compress marshals and compresses wr.
-func Compress(wr WriteRequest) ([]byte, error) {
+func Compress(wr WriteRequest) []byte {
 	data, err := wr.Marshal()
 	if err != nil {
-		return nil, err
+		panic(fmt.Errorf("BUG: cannot compress WriteRequest: %s", err))
 	}
-	return snappy.Encode(nil, data), nil
+	return snappy.Encode(nil, data)
 }

app/vlinsert/elasticsearch/elasticsearch.go

@@ -20,7 +20,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
@@ -98,12 +97,10 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-		processLogMessage := cp.GetProcessLogMessageFunc(lr)
+		lmp := cp.NewLogMessageProcessor()
 		isGzip := r.Header.Get("Content-Encoding") == "gzip"
-		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
-		vlstorage.MustAddRows(lr)
-		logstorage.PutLogRows(lr)
+		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, lmp)
+		lmp.MustClose()
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
 			return true
@@ -132,9 +129,7 @@ var (
 	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (int, error) {
+func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp insertutils.LogMessageProcessor) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
 	if isGzip {
@@ -159,7 +154,7 @@ func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
 	n := 0
 	nCheckpoint := 0
 	for {
-		ok, err := readBulkLine(sc, timeField, msgField, processLogMessage)
+		ok, err := readBulkLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil || !ok {
 			rowsIngestedTotal.Add(n - nCheckpoint)
@@ -175,9 +170,7 @@ func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
 
 var lineBufferPool bytesutil.ByteBufferPool
 
-func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (bool, error) {
+func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 
 	// Read the command, must be "create" or "index"
@@ -210,7 +203,7 @@ func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
 		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
 	}
 	line = sc.Bytes()
-	p := logjson.GetParser()
+	p := logstorage.GetJSONParser()
 	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
@@ -222,9 +215,9 @@ func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
 	if ts == 0 {
 		ts = time.Now().UnixNano()
 	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
-	logjson.PutParser(p)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
+	logstorage.PutJSONParser(p)
 
 	return true, nil
 }
@@ -273,9 +266,9 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
 		}
 		return t.UnixNano(), nil
 	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
+	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
+	if !ok {
+		return 0, fmt.Errorf("cannot parse timestamp %q", s)
 	}
-	return t.UnixNano(), nil
+	return nsecs, nil
 }

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -4,23 +4,18 @@ import (
 	"bytes"
 	"compress/gzip"
 	"fmt"
-	"reflect"
-	"strings"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestReadBulkRequestFailure(t *testing.T) {
+func TestReadBulkRequest_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
 
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
-		}
-
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, "_time", "_msg", processLogMessage)
+		rows, err := readBulkRequest(r, false, "_time", "_msg", tlp)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -37,58 +32,38 @@ func TestReadBulkRequestFailure(t *testing.T) {
 foobar`)
 }
 
-func TestReadBulkRequestSuccess(t *testing.T) {
+func TestReadBulkRequest_Success(t *testing.T) {
 	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
+		tlp := &insertutils.TestLogMessageProcessor{}
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, timeField, msgField, processLogMessage)
+		rows, err := readBulkRequest(r, false, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if rows != rowsExpected {
 			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 
 		// Read the request with compression
-		timestamps = nil
-		result = ""
+		tlp = &insertutils.TestLogMessageProcessor{}
 		compressedData := compressData(data)
 		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest(r, true, timeField, msgField, processLogMessage)
+		rows, err = readBulkRequest(r, true, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if rows != rowsExpected {
 			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatalf("verification failure after compression: %s", err)
 		}
 	}
 
@@ -111,8 +86,7 @@ func TestReadBulkRequestSuccess(t *testing.T) {
 	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
 	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
+{"_msg":"xyz","@timestamp":"","x":"y"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
 func BenchmarkReadBulkRequest(b *testing.B) {
@@ -33,7 +33,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 
 	timeField := "@timestamp"
 	msgField := "message"
-	processLogMessage := func(_ int64, _ []logstorage.Field) {}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))
@@ -41,7 +41,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
+			_, err := readBulkRequest(r, isGzip, timeField, msgField, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutils/common_params.go

@@ -2,6 +2,8 @@ package insertutils
 
 import (
 	"net/http"
+	"sync"
+	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
@@ -10,11 +12,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 // CommonParams contains common HTTP parameters used by log ingestion APIs.
 //
-// See https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/#http-parameters
+// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
 type CommonParams struct {
 	TenantID     logstorage.TenantID
 	TimeField    string
@@ -71,29 +74,126 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 	return cp, nil
 }
 
-// GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
-func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
-	return func(timestamp int64, fields []logstorage.Field) {
-		if len(fields) > *MaxFieldsPerLine {
-			rf := logstorage.RowFormatter(fields)
-			logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
-			rowsDroppedTotalTooManyFields.Inc()
-			return
-		}
-
-		lr.MustAdd(cp.TenantID, timestamp, fields)
-		if cp.Debug {
-			s := lr.GetRowString(0)
-			lr.ResetKeepSettings()
-			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
-			rowsDroppedTotalDebug.Inc()
-			return
-		}
-		if lr.NeedFlush() {
-			vlstorage.MustAddRows(lr)
-			lr.ResetKeepSettings()
-		}
+// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
+	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
+	cp := &CommonParams{
+		TenantID:  tenantID,
+		TimeField: "timestamp",
+		MsgField:  "message",
+		StreamFields: []string{
+			"hostname",
+			"app_name",
+			"proc_id",
+		},
 	}
+
+	return cp
+}
+
+// LogMessageProcessor is an interface for log message processors.
+type LogMessageProcessor interface {
+	// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
+	//
+	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
+	AddRow(timestamp int64, fields []logstorage.Field)
+
+	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
+	MustClose()
+}
+
+type logMessageProcessor struct {
+	mu            sync.Mutex
+	wg            sync.WaitGroup
+	stopCh        chan struct{}
+	lastFlushTime time.Time
+
+	cp *CommonParams
+	lr *logstorage.LogRows
+}
+
+func (lmp *logMessageProcessor) initPeriodicFlush() {
+	lmp.lastFlushTime = time.Now()
+
+	lmp.wg.Add(1)
+	go func() {
+		defer lmp.wg.Done()
+
+		d := timeutil.AddJitterToDuration(time.Second)
+		ticker := time.NewTicker(d)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-lmp.stopCh:
+				return
+			case <-ticker.C:
+				lmp.mu.Lock()
+				if time.Since(lmp.lastFlushTime) >= d {
+					lmp.flushLocked()
+				}
+				lmp.mu.Unlock()
+			}
+		}
+	}()
+}
+
+// AddRow adds new log message to lmp with the given timestamp and fields.
+func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	lmp.mu.Lock()
+	defer lmp.mu.Unlock()
+
+	if len(fields) > *MaxFieldsPerLine {
+		rf := logstorage.RowFormatter(fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		rowsDroppedTotalTooManyFields.Inc()
+		return
+	}
+
+	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
+	if lmp.cp.Debug {
+		s := lmp.lr.GetRowString(0)
+		lmp.lr.ResetKeepSettings()
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		rowsDroppedTotalDebug.Inc()
+		return
+	}
+	if lmp.lr.NeedFlush() {
+		lmp.flushLocked()
+	}
+}
+
+// flushLocked must be called under locked lmp.mu.
+func (lmp *logMessageProcessor) flushLocked() {
+	lmp.lastFlushTime = time.Now()
+	vlstorage.MustAddRows(lmp.lr)
+	lmp.lr.ResetKeepSettings()
+}
+
+// MustClose flushes the remaining data to the underlying storage and closes lmp.
+func (lmp *logMessageProcessor) MustClose() {
+	close(lmp.stopCh)
+	lmp.wg.Wait()
+
+	lmp.flushLocked()
+	logstorage.PutLogRows(lmp.lr)
+	lmp.lr = nil
+}
+
+// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
+//
+// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
+func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	lmp := &logMessageProcessor{
+		cp: cp,
+		lr: lr,
+
+		stopCh: make(chan struct{}),
+	}
+	lmp.initPeriodicFlush()
+
+	return lmp
 }
 
 var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)

app/vlinsert/insertutils/testutils.go

@@ -0,0 +1,53 @@
+package insertutils
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// TestLogMessageProcessor implements LogMessageProcessor for testing.
+type TestLogMessageProcessor struct {
+	timestamps []int64
+	rows       []string
+}
+
+// AddRow adds row with the given timestamp and fields to tlp
+func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	tlp.timestamps = append(tlp.timestamps, timestamp)
+	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
+}
+
+// MustClose closes tlp.
+func (tlp *TestLogMessageProcessor) MustClose() {
+}
+
+// Verify verifies the number of rows, timestamps and results after AddRow calls.
+func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error {
+	result := strings.Join(tlp.rows, "\n")
+	if len(tlp.rows) != rowsExpected {
+		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), rowsExpected, result, resultExpected)
+	}
+
+	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
+		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
+	}
+	if result != resultExpected {
+		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+	}
+
+	return nil
+}
+
+// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
+type BenchmarkLogMessageProcessor struct{}
+
+// AddRow implements LogMessageProcessor interface.
+func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field) {
+}
+
+// MustClose implements LogMessageProcessor interface.
+func (blp *BenchmarkLogMessageProcessor) MustClose() {
+}

app/vlinsert/insertutils/timestamp.go

@@ -0,0 +1,33 @@
+package insertutils
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+//
+// The value for the timeField is set to empty string after returning from the function,
+// so it could be ignored during data ingestion.
+//
+// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
+func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		if f.Value == "" || f.Value == "0" {
+			return time.Now().UnixNano(), nil
+		}
+		nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
+		if !ok {
+			return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
+		}
+		f.Value = ""
+		return nsecs, nil
+	}
+	return time.Now().UnixNano(), nil
+}

app/vlinsert/insertutils/timestamp_test.go

@@ -0,0 +1,75 @@
+package insertutils
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
+	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
+		t.Helper()
+
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if nsecs != nsecsExpected {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
+		}
+
+		for _, f := range fields {
+			if f.Name == timeField {
+				if f.Value != "" {
+					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
+				}
+			}
+		}
+	}
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20Z"},
+	}, 1718753840000000000)
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
+	}, 1718725040000000000)
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
+	}, 1718773640123000000)
+
+	f("time", []logstorage.Field{
+		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
+		{Name: "foo", Value: "bar"},
+	}, 1718773640123456789)
+}
+
+func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+
+		fields := []logstorage.Field{
+			{Name: "time", Value: s},
+		}
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if nsecs != 0 {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
+		}
+	}
+
+	f("foobar")
+
+	// no Z at the end
+	f("2024-06-18T23:37:20")
+
+	// incomplete time
+	f("2024-06-18")
+	f("2024-06-18T23:37")
+}

app/vlinsert/jsonline/jsonline.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 
@@ -12,7 +13,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
@@ -20,13 +20,13 @@ import (
 )
 
 // RequestHandler processes jsonline insert requests
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+func RequestHandler(w http.ResponseWriter, r *http.Request) {
 	startTime := time.Now()
 	w.Header().Add("Content-Type", "application/json")
 
 	if r.Method != "POST" {
 		w.WriteHeader(http.StatusMethodNotAllowed)
-		return true
+		return
 	}
 
 	requestsTotal.Inc()
@@ -34,27 +34,40 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
 	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
 
 	reader := r.Body
 	if r.Header.Get("Content-Encoding") == "gzip" {
 		zr, err := common.GetGzipReader(reader)
 		if err != nil {
-			logger.Errorf("cannot read gzipped _bulk request: %s", err)
-			return true
+			logger.Errorf("cannot read gzipped jsonline request: %s", err)
+			return
 		}
 		defer common.PutGzipReader(zr)
 		reader = zr
 	}
 
-	wcr := writeconcurrencylimiter.GetReader(reader)
+	lmp := cp.NewLogMessageProcessor()
+	err = processStreamInternal(reader, cp.TimeField, cp.MsgField, lmp)
+	lmp.MustClose()
+
+	if err != nil {
+		logger.Errorf("jsonline: %s", err)
+	} else {
+		// update requestDuration only for successfully parsed requests.
+		// There is no need in updating requestDuration for request errors,
+		// since their timings are usually much smaller than the timing for successful request parsing.
+		requestDuration.UpdateDuration(startTime)
+	}
+}
+
+func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertutils.LogMessageProcessor) error {
+	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
 	lb := lineBufferPool.Get()
@@ -66,31 +79,21 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	n := 0
 	for {
-		ok, err := readLine(sc, cp.TimeField, cp.MsgField, processLogMessage)
+		ok, err := readLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil {
-			logger.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
-			break
+			errorsTotal.Inc()
+			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
 		}
 		if !ok {
-			break
+			return nil
 		}
 		n++
 		rowsIngestedTotal.Inc()
 	}
-
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-
-	// update jsonlineRequestDuration only for successfully parsed requests.
-	// There is no need in updating jsonlineRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	jsonlineRequestDuration.UpdateDuration(startTime)
-
-	return true
 }
 
-func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
+func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 	for len(line) == 0 {
 		if !sc.Scan() {
@@ -105,57 +108,28 @@ func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage f
 		line = sc.Bytes()
 	}
 
-	p := logjson.GetParser()
+	p := logstorage.GetJSONParser()
 	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
+	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
 	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
+		return false, fmt.Errorf("cannot get timestamp: %w", err)
 	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
-	logjson.PutParser(p)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
+	logstorage.PutJSONParser(p)
 
 	return true, nil
 }
 
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		timestamp, err := parseISO8601Timestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
-	}
-	return 0, nil
-}
-
-func parseISO8601Timestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case for returning the current timestamp.
-		// It must be automatically converted to the current timestamp by the caller.
-		return 0, nil
-	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
-	}
-	return t.UnixNano(), nil
-}
-
 var lineBufferPool bytesutil.ByteBufferPool
 
 var (
-	requestsTotal           = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	rowsIngestedTotal       = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
-	jsonlineRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+
+	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
+	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
+
+	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
 )

app/vlinsert/jsonline/jsonline_test.go

@@ -1,59 +1,27 @@
 package jsonline
 
 import (
-	"bufio"
 	"bytes"
-	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"reflect"
-	"strings"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestReadBulkRequestSuccess(t *testing.T) {
+func TestProcessStreamInternal_Success(t *testing.T) {
 	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
-
-		// Read the request without compression
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		sc := bufio.NewScanner(r)
-		rows := 0
-		for {
-			ok, err := readLine(sc, timeField, msgField, processLogMessage)
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-			if !ok {
-				break
-			}
-			rows++
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		if err := processStreamInternal(r, timeField, msgField, tlp); err != nil {
+			t.Fatalf("unexpected error: %s", err)
 		}
 
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 	}
 
-	// Verify non-empty data
 	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
 {"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
@@ -64,7 +32,24 @@ func TestReadBulkRequestSuccess(t *testing.T) {
 	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
 	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
+{"_msg":"xyz","@timestamp":"","x":"y"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
+
+func TestProcessStreamInternal_Failure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal(r, "time", "", tlp); err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// invalid json
+	f("foobar")
+
+	// invalid timestamp field
+	f(`{"time":"foobar"}`)
+}

app/vlinsert/loki/loki.go

@@ -11,7 +11,8 @@ import (
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
 	case "/api/v1/push":
-		return handleInsert(r, w)
+		handleInsert(r, w)
+		return true
 	case "/ready":
 		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
 		w.WriteHeader(http.StatusOK)
@@ -23,14 +24,14 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 }
 
 // See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-func handleInsert(r *http.Request, w http.ResponseWriter) bool {
+func handleInsert(r *http.Request, w http.ResponseWriter) {
 	contentType := r.Header.Get("Content-Type")
 	switch contentType {
 	case "application/json":
-		return handleJSON(r, w)
+		handleJSON(r, w)
 	default:
 		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		return handleProtobuf(r, w)
+		handleProtobuf(r, w)
 	}
 }
 
@@ -45,7 +46,7 @@ func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
 	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
 		org := r.Header.Get("X-Scope-OrgID")
 		if org != "" {
-			tenantID, err := logstorage.GetTenantIDFromString(org)
+			tenantID, err := logstorage.ParseTenantID(org)
 			if err != nil {
 				return nil, err
 			}

app/vlinsert/loki/loki_json.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -20,15 +21,15 @@ import (
 
 var parserPool fastjson.ParserPool
 
-func handleJSON(r *http.Request, w http.ResponseWriter) bool {
+func handleJSON(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
-	lokiRequestsJSONTotal.Inc()
+	requestsJSONTotal.Inc()
 	reader := r.Body
 	if r.Header.Get("Content-Encoding") == "gzip" {
 		zr, err := common.GetGzipReader(reader)
 		if err != nil {
 			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return true
+			return
 		}
 		defer common.PutGzipReader(zr)
 		reader = zr
@@ -39,45 +40,41 @@ func handleJSON(r *http.Request, w http.ResponseWriter) bool {
 	writeconcurrencylimiter.PutReader(wcr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
+		return
 	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
+		return
 	}
 	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseJSONRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseJSONRequest(data, lmp)
+	lmp.MustClose()
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
-		return true
+		return
 	}
 
 	rowsIngestedJSONTotal.Add(n)
 
-	// update lokiRequestJSONDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestJSONDuration for request errors,
+	// update requestJSONDuration only for successfully parsed requests
+	// There is no need in updating requestJSONDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestJSONDuration.UpdateDuration(startTime)
-
-	return true
+	requestJSONDuration.UpdateDuration(startTime)
 }
 
 var (
-	lokiRequestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	rowsIngestedJSONTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
-	lokiRequestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
+	requestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
+	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
+	requestJSONDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
-func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
 	v, err := p.ParseBytes(data)
@@ -170,7 +167,7 @@ func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, field
 				Name:  "_msg",
 				Value: bytesutil.ToUnsafeString(msg),
 			})
-			processLogMessage(ts, fields)
+			lmp.AddRow(ts, fields)
 		}
 		rowsIngested += len(lines)
 	}

app/vlinsert/loki/loki_json_test.go

@@ -1,19 +1,17 @@
 package loki
 
 import (
-	"fmt"
-	"strings"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestParseJSONRequestFailure(t *testing.T) {
+func TestParseJSONRequest_Failure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
-			t.Fatalf("unexpected call to parseJSONRequest callback!")
-		})
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		n, err := parseJSONRequest([]byte(s), tlp)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -56,39 +54,30 @@ func TestParseJSONRequestFailure(t *testing.T) {
 	f(`{"streams":[{"values":[["123",1234]]}]}`)
 }
 
-func TestParseJSONRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
+func TestParseJSONRequest_Success(t *testing.T) {
+	f := func(s string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
-		var lines []string
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+
+		n, err := parseJSONRequest([]byte(s), tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+	f(`{"streams":[]}`, nil, ``)
+	f(`{"streams":[{"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
 
 	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 
 	// Non-empty stream labels
 	f(`{"streams":[{"stream":{
@@ -98,9 +87,9 @@ func TestParseJSONRequestSuccess(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+{"label1":"value1","label2":"value2","_msg":"abc"}
+{"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
 	// Multiple streams
 	f(`{
@@ -124,7 +113,7 @@ _time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
 			]
 		}
 	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
+{"foo":"bar","a":"b","_msg":"abc"}
+{"x":"y","_msg":"yx"}`)
 }

app/vlinsert/loki/loki_json_timing_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func BenchmarkParseJSONRequest(b *testing.B) {
@@ -22,12 +22,13 @@ func BenchmarkParseJSONRequest(b *testing.B) {
 }
 
 func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseJSONRequest(data, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/loki/loki_protobuf.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -23,53 +24,49 @@ var (
 	pushReqsPool sync.Pool
 )
 
-func handleProtobuf(r *http.Request, w http.ResponseWriter) bool {
+func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
-	lokiRequestsProtobufTotal.Inc()
+	requestsProtobufTotal.Inc()
 	wcr := writeconcurrencylimiter.GetReader(r.Body)
 	data, err := io.ReadAll(wcr)
 	writeconcurrencylimiter.PutReader(wcr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
+		return
 	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
+		return
 	}
 	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseProtobufRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseProtobufRequest(data, lmp)
+	lmp.MustClose()
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
-		return true
+		return
 	}
 
 	rowsIngestedProtobufTotal.Add(n)
 
-	// update lokiRequestProtobufDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestProtobufDuration for request errors,
+	// update requestProtobufDuration only for successfully parsed requests
+	// There is no need in updating requestProtobufDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestProtobufDuration.UpdateDuration(startTime)
-
-	return true
+	requestProtobufDuration.UpdateDuration(startTime)
 }
 
 var (
-	lokiRequestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	rowsIngestedProtobufTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
-	lokiRequestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	requestsProtobufTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
+	requestProtobufDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
-func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
 	bb := bytesBufPool.Get()
 	defer bytesBufPool.Put(bb)
 
@@ -82,12 +79,14 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err = req.Unmarshal(bb.B)
+	err = req.UnmarshalProtobuf(bb.B)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
-	var commonFields []logstorage.Field
+	fields := getFields()
+	defer putFields(fields)
+
 	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
@@ -95,30 +94,60 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
 		// Labels are same for all entries in the stream.
-		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
+		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
 		if err != nil {
 			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
-		fields := commonFields
+		commonFieldsLen := len(fields.fields)
 
 		entries := stream.Entries
 		for j := range entries {
-			entry := &entries[j]
-			fields = append(fields[:len(commonFields)], logstorage.Field{
+			e := &entries[j]
+			fields.fields = fields.fields[:commonFieldsLen]
+
+			for _, lp := range e.StructuredMetadata {
+				fields.fields = append(fields.fields, logstorage.Field{
+					Name:  lp.Name,
+					Value: lp.Value,
+				})
+			}
+
+			fields.fields = append(fields.fields, logstorage.Field{
 				Name:  "_msg",
-				Value: entry.Line,
+				Value: e.Line,
 			})
-			ts := entry.Timestamp.UnixNano()
+
+			ts := e.Timestamp.UnixNano()
 			if ts == 0 {
 				ts = currentTimestamp
 			}
-			processLogMessage(ts, fields)
+
+			lmp.AddRow(ts, fields.fields)
 		}
 		rowsIngested += len(stream.Entries)
 	}
 	return rowsIngested, nil
 }
 
+func getFields() *fields {
+	v := fieldsPool.Get()
+	if v == nil {
+		return &fields{}
+	}
+	return v.(*fields)
+}
+
+func putFields(f *fields) {
+	f.fields = f.fields[:0]
+	fieldsPool.Put(f)
+}
+
+var fieldsPool sync.Pool
+
+type fields struct {
+	fields []logstorage.Field
+}
+
 // parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
 //
 // See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
@@ -184,6 +213,6 @@ func getPushRequest() *PushRequest {
 }
 
 func putPushRequest(req *PushRequest) {
-	req.Reset()
+	req.reset()
 	pushReqsPool.Put(req)
 }

app/vlinsert/loki/loki_protobuf_test.go

@@ -6,83 +6,80 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/golang/snappy"
 )
 
-func TestParseProtobufRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
+type testLogMessageProcessor struct {
+	pr PushRequest
+}
+
+func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	msg := ""
+	for _, f := range fields {
+		if f.Name == "_msg" {
+			msg = f.Value
+		}
+	}
+	var a []string
+	for _, f := range fields {
+		if f.Name == "_msg" {
+			continue
+		}
+		item := fmt.Sprintf("%s=%q", f.Name, f.Value)
+		a = append(a, item)
+	}
+	labels := "{" + strings.Join(a, ", ") + "}"
+	tlp.pr.Streams = append(tlp.pr.Streams, Stream{
+		Labels: labels,
+		Entries: []Entry{
+			{
+				Timestamp: time.Unix(0, timestamp),
+				Line:      strings.Clone(msg),
+			},
+		},
+	})
+}
+
+func (tlp *testLogMessageProcessor) MustClose() {
+}
+
+func TestParseProtobufRequest_Success(t *testing.T) {
+	f := func(s string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
-		var pr PushRequest
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			msg := ""
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					msg = f.Value
-				}
-			}
-			var a []string
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					continue
-				}
-				item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-				a = append(a, item)
-			}
-			labels := "{" + strings.Join(a, ", ") + "}"
-			pr.Streams = append(pr.Streams, Stream{
-				Labels: labels,
-				Entries: []Entry{
-					{
-						Timestamp: time.Unix(0, timestamp),
-						Line:      msg,
-					},
-				},
-			})
-		})
+
+		tlp := &testLogMessageProcessor{}
+		n, err := parseJSONRequest([]byte(s), tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if n != len(pr.Streams) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(pr.Streams), n)
+		if n != len(tlp.pr.Streams) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
 		}
 
-		data, err := pr.Marshal()
-		if err != nil {
-			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
-		}
+		data := tlp.pr.MarshalProtobuf(nil)
 		encodedData := snappy.Encode(nil, data)
 
-		var lines []string
-		n, err = parseProtobufRequest(encodedData, func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
+		tlp2 := &insertutils.TestLogMessageProcessor{}
+		n, err = parseProtobufRequest(encodedData, tlp2)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp2.Verify(n, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+	f(`{"streams":[]}`, nil, ``)
+	f(`{"streams":[{"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
 
 	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 
 	// Non-empty stream labels
 	f(`{"streams":[{"stream":{
@@ -92,9 +89,9 @@ func TestParseProtobufRequestSuccess(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+{"label1":"value1","label2":"value2","_msg":"abc"}
+{"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
 	// Multiple streams
 	f(`{
@@ -118,12 +115,12 @@ _time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
 			]
 		}
 	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
+{"foo":"bar","a":"b","_msg":"abc"}
+{"x":"y","_msg":"yx"}`)
 }
 
-func TestParsePromLabelsSuccess(t *testing.T) {
+func TestParsePromLabels_Success(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		fields, err := parsePromLabels(nil, s)
@@ -147,7 +144,7 @@ func TestParsePromLabelsSuccess(t *testing.T) {
 	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
 }
 
-func TestParsePromLabelsFailure(t *testing.T) {
+func TestParsePromLabels_Failure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		fields, err := parsePromLabels(nil, s)

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -8,7 +8,8 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkParseProtobufRequest(b *testing.B) {
@@ -24,12 +25,13 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
 }
 
 func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseProtobufRequest(body, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
@@ -37,29 +39,47 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	})
 }
 
-func getProtobufBody(streams, rows, labels int) []byte {
-	var pr PushRequest
-
-	for i := 0; i < streams; i++ {
-		var st Stream
-
-		st.Labels = `{`
-		for j := 0; j < labels; j++ {
-			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
-			if j < labels-1 {
-				st.Labels += `,`
+func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
+	var b []byte
+	var entries []Entry
+	streams := make([]Stream, streamsCount)
+	for i := range streams {
+		b = b[:0]
+		b = append(b, '{')
+		for j := 0; j < labelsCount; j++ {
+			b = append(b, "label_"...)
+			b = strconv.AppendInt(b, int64(j), 10)
+			b = append(b, `="value_`...)
+			b = strconv.AppendInt(b, int64(j), 10)
+			b = append(b, '"')
+			if j < labelsCount-1 {
+				b = append(b, ',')
 			}
 		}
-		st.Labels += `}`
+		b = append(b, '}')
+		labels := string(b)
 
-		for j := 0; j < rows; j++ {
-			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
+		var rowsBuf []byte
+		entriesLen := len(entries)
+		for j := 0; j < rowsCount; j++ {
+			rowsBufLen := len(rowsBuf)
+			rowsBuf = append(rowsBuf, "value_"...)
+			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
+			entries = append(entries, Entry{
+				Timestamp: time.Now(),
+				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
+			})
 		}
 
-		pr.Streams = append(pr.Streams, st)
+		st := &streams[i]
+		st.Labels = labels
+		st.Entries = entries[entriesLen:]
+	}
+	pr := PushRequest{
+		Streams: streams,
 	}
 
-	body, _ := pr.Marshal()
+	body := pr.MarshalProtobuf(nil)
 	encodedBody := snappy.Encode(nil, body)
 
 	return encodedBody

app/vlinsert/loki/pb.go

@@ -0,0 +1,302 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: push_request.proto
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package loki
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/easyproto"
+)
+
+var mp easyproto.MarshalerPool
+
+// PushRequest represents Loki PushRequest
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
+type PushRequest struct {
+	Streams []Stream
+
+	entriesBuf   []Entry
+	labelPairBuf []LabelPair
+}
+
+func (pr *PushRequest) reset() {
+	pr.Streams = pr.Streams[:0]
+
+	pr.entriesBuf = pr.entriesBuf[:0]
+	pr.labelPairBuf = pr.labelPairBuf[:0]
+}
+
+// UnmarshalProtobuf unmarshals pr from protobuf message at src.
+//
+// pr remains valid until src is modified.
+func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
+	pr.reset()
+	var err error
+	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
+	return err
+}
+
+// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
+func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
+	m := mp.Get()
+	pr.marshalProtobuf(m.MessageMarshaler())
+	dst = m.Marshal(dst)
+	mp.Put(m)
+	return dst
+}
+
+func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	for _, s := range pr.Streams {
+		s.marshalProtobuf(mm.AppendMessage(1))
+	}
+}
+
+func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
+	// message PushRequest {
+	//   repeated Stream streams = 1;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			data, ok := fc.MessageData()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
+			}
+			pr.Streams = append(pr.Streams, Stream{})
+			s := &pr.Streams[len(pr.Streams)-1]
+			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
+			if err != nil {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
+			}
+		}
+	}
+	return entriesBuf, labelPairBuf, nil
+}
+
+// Stream represents Loki stream.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
+type Stream struct {
+	Labels  string
+	Entries []Entry
+}
+
+func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendString(1, s.Labels)
+	for _, e := range s.Entries {
+		e.marshalProtobuf(mm.AppendMessage(2))
+	}
+}
+
+func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
+	// message Stream {
+	//   string labels = 1;
+	//   repeated Entry entries = 2;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	entriesBufLen := len(entriesBuf)
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			labels, ok := fc.String()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
+			}
+			s.Labels = labels
+		case 2:
+			data, ok := fc.MessageData()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
+			}
+			entriesBuf = append(entriesBuf, Entry{})
+			e := &entriesBuf[len(entriesBuf)-1]
+			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
+			if err != nil {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
+			}
+		}
+	}
+	s.Entries = entriesBuf[entriesBufLen:]
+	return entriesBuf, labelPairBuf, nil
+}
+
+// Entry represents Loki entry.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
+type Entry struct {
+	Timestamp          time.Time
+	Line               string
+	StructuredMetadata []LabelPair
+}
+
+func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	marshalTime(mm, 1, e.Timestamp)
+	mm.AppendString(2, e.Line)
+	for _, lp := range e.StructuredMetadata {
+		lp.marshalProtobuf(mm.AppendMessage(3))
+	}
+}
+
+func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
+	// message Entry {
+	//   Timestamp timestamp = 1;
+	//   string line = 2;
+	//   repeated LabelPair structuredMetadata = 3;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	labelPairBufLen := len(labelPairBuf)
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			data, ok := fc.MessageData()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
+			}
+			timestamp, err := unmarshalTime(data)
+			if err != nil {
+				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
+			}
+			e.Timestamp = timestamp
+		case 2:
+			line, ok := fc.String()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read Line")
+			}
+			e.Line = line
+		case 3:
+			data, ok := fc.MessageData()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
+			}
+			labelPairBuf = append(labelPairBuf, LabelPair{})
+			lp := &labelPairBuf[len(labelPairBuf)-1]
+			if err := lp.unmarshalProtobuf(data); err != nil {
+				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
+			}
+		}
+	}
+	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
+	return labelPairBuf, nil
+}
+
+// LabelPair represents Loki label pair.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
+type LabelPair struct {
+	Name  string
+	Value string
+}
+
+func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendString(1, lp.Name)
+	mm.AppendString(2, lp.Value)
+}
+
+func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
+	// message LabelPair {
+	//   string name = 1;
+	//   string value = 2;
+	// }
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			name, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot read name")
+			}
+			lp.Name = name
+		case 2:
+			value, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot unmarshal value")
+			}
+			lp.Value = value
+		}
+	}
+	return nil
+}
+
+func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
+	nsecs := timestamp.UnixNano()
+	ts := Timestamp{
+		Seconds: nsecs / 1e9,
+		Nanos:   int32(nsecs % 1e9),
+	}
+	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
+}
+
+func unmarshalTime(src []byte) (time.Time, error) {
+	var ts Timestamp
+	if err := ts.unmarshalProtobuf(src); err != nil {
+		return time.Time{}, err
+	}
+	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	return timestamp, nil
+}
+
+// Timestamp is protobuf well-known timestamp type.
+type Timestamp struct {
+	Seconds int64
+	Nanos   int32
+}
+
+func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendInt64(1, ts.Seconds)
+	mm.AppendInt32(2, ts.Nanos)
+}
+
+func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
+	// message Timestamp {
+	//   int64 seconds = 1;
+	//   int32 nanos = 2;
+	// }
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			seconds, ok := fc.Int64()
+			if !ok {
+				return fmt.Errorf("cannot read Seconds")
+			}
+			ts.Seconds = seconds
+		case 2:
+			nanos, ok := fc.Int32()
+			if !ok {
+				return fmt.Errorf("cannot read Nanos")
+			}
+			ts.Nanos = nanos
+		}
+	}
+	return nil
+}

app/vlinsert/loki/push_request.proto

@@ -1,38 +0,0 @@
-syntax = "proto3";
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package logproto;
-
-import "gogoproto/gogo.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
-
-message PushRequest {
-  repeated StreamAdapter streams = 1 [
-    (gogoproto.jsontag) = "streams",
-    (gogoproto.customtype) = "Stream"
-  ];
-}
-
-message StreamAdapter {
-  string labels = 1 [(gogoproto.jsontag) = "labels"];
-  repeated EntryAdapter entries = 2 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "entries"
-  ];
-  // hash contains the original hash of the stream.
-  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
-}
-
-message EntryAdapter {
-  google.protobuf.Timestamp timestamp = 1 [
-    (gogoproto.stdtime) = true,
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "ts"
-  ];
-  string line = 2 [(gogoproto.jsontag) = "line"];
-}

app/vlinsert/loki/timestamp.go

@@ -1,110 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"errors"
-	"strconv"
-	"time"
-
-	"github.com/gogo/protobuf/types"
-)
-
-const (
-	// Seconds field of the earliest valid Timestamp.
-	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	minValidSeconds = -62135596800
-	// Seconds field just after the latest valid Timestamp.
-	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	maxValidSeconds = 253402300800
-)
-
-// validateTimestamp determines whether a Timestamp is valid.
-// A valid timestamp represents a time in the range
-// [0001-01-01, 10000-01-01) and has a Nanos field
-// in the range [0, 1e9).
-//
-// If the Timestamp is valid, validateTimestamp returns nil.
-// Otherwise, it returns an error that describes
-// the problem.
-//
-// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
-func validateTimestamp(ts *types.Timestamp) error {
-	if ts == nil {
-		return errors.New("timestamp: nil Timestamp")
-	}
-	if ts.Seconds < minValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
-	}
-	if ts.Seconds >= maxValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
-	}
-	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
-		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
-	}
-	return nil
-}
-
-// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
-// but avoids the escape incurred by using fmt.Sprintf, eliminating
-// unnecessary heap allocations.
-func formatTimestamp(ts *types.Timestamp) string {
-	if ts == nil {
-		return "nil"
-	}
-
-	seconds := strconv.FormatInt(ts.Seconds, 10)
-	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
-	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
-}
-
-func sizeOfStdTime(t time.Time) int {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0
-	}
-	return ts.Size()
-}
-
-func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0, err
-	}
-	return ts.MarshalTo(data)
-}
-
-func stdTimeUnmarshal(t *time.Time, data []byte) error {
-	ts := &types.Timestamp{}
-	if err := ts.Unmarshal(data); err != nil {
-		return err
-	}
-	tt, err := timestampFromProto(ts)
-	if err != nil {
-		return err
-	}
-	*t = tt
-	return nil
-}
-
-func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
-	// Don't return the zero value on error, because corresponds to a valid
-	// timestamp. Instead return whatever time.Unix gives us.
-	var t time.Time
-	if ts == nil {
-		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
-	} else {
-		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	}
-	return t, validateTimestamp(ts)
-}
-
-func timestampProto(t time.Time) (types.Timestamp, error) {
-	ts := types.Timestamp{
-		Seconds: t.Unix(),
-		Nanos:   int32(t.Nanosecond()),
-	}
-	return ts, validateTimestamp(&ts)
-}

app/vlinsert/loki/types.go

@@ -1,481 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"fmt"
-	"io"
-	"time"
-)
-
-// Stream contains a unique labels set as a string and a set of entries for it.
-// We are not using the proto generated version but this custom one so that we
-// can improve serialization see benchmark.
-type Stream struct {
-	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
-	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
-	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
-}
-
-// Entry is a log entry with a timestamp.
-type Entry struct {
-	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
-	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Stream) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Hash != 0 {
-		i = encodeVarintPush(dAtA, i, m.Hash)
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Entries) > 0 {
-		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintPush(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x12
-		}
-	}
-	if len(m.Labels) > 0 {
-		i -= len(m.Labels)
-		copy(dAtA[i:], m.Labels)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Entry) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if len(m.Line) > 0 {
-		i -= len(m.Line)
-		copy(dAtA[i:], m.Line)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
-		i--
-		dAtA[i] = 0x12
-	}
-	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
-	if err7 != nil {
-		return 0, err7
-	}
-	i -= n7
-	i = encodeVarintPush(dAtA, i, uint64(n7))
-	i--
-	dAtA[i] = 0xa
-	return len(dAtA) - i, nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Stream) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Labels = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Entries = append(m.Entries, Entry{})
-			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
-			}
-			m.Hash = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Hash |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Entry) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Line = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Size returns the size of the serialized Stream.
-func (m *Stream) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Labels)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	if len(m.Entries) > 0 {
-		for _, e := range m.Entries {
-			l = e.Size()
-			n += 1 + l + sovPush(uint64(l))
-		}
-	}
-	if m.Hash != 0 {
-		n += 1 + sovPush(m.Hash)
-	}
-	return n
-}
-
-// Size returns the size of the serialized Entry
-func (m *Entry) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = sizeOfStdTime(m.Timestamp)
-	n += 1 + l + sovPush(uint64(l))
-	l = len(m.Line)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	return n
-}
-
-// Equal returns true if the two Streams are equal.
-func (m *Stream) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Stream)
-	if !ok {
-		that2, ok := that.(Stream)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if m.Labels != that1.Labels {
-		return false
-	}
-	if len(m.Entries) != len(that1.Entries) {
-		return false
-	}
-	for i := range m.Entries {
-		if !m.Entries[i].Equal(that1.Entries[i]) {
-			return false
-		}
-	}
-	return m.Hash == that1.Hash
-}
-
-// Equal returns true if the two Entries are equal.
-func (m *Entry) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Entry)
-	if !ok {
-		that2, ok := that.(Entry)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if !m.Timestamp.Equal(that1.Timestamp) {
-		return false
-	}
-	if m.Line != that1.Line {
-		return false
-	}
-	return true
-}

app/vlinsert/main.go

@@ -7,14 +7,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
 )
 
 // Init initializes vlinsert
 func Init() {
+	syslog.MustInit()
 }
 
 // Stop stops vlinsert
 func Stop() {
+	syslog.MustStop()
 }
 
 // RequestHandler handles insert requests for VictoriaLogs
@@ -28,7 +31,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path = strings.ReplaceAll(path, "//", "/")
 
 	if path == "/jsonline" {
-		return jsonline.RequestHandler(w, r)
+		jsonline.RequestHandler(w, r)
+		return true
 	}
 	switch {
 	case strings.HasPrefix(path, "/elasticsearch/"):

app/vlinsert/syslog/syslog.go

@@ -0,0 +1,531 @@
+package syslog
+
+import (
+	"bufio"
+	"crypto/tls"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/klauspost/compress/gzip"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
+		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+
+		"The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
+		"Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
+		"The provided key file is automatically re-read every second, so it can be dynamically updated. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+
+		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"Supported values: TLS10, TLS11, TLS12, TLS13. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+
+	compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+
+		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
+	compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+
+		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
+
+	useLocalTimestampTCP = flagutil.NewArrayBool("syslog.useLocalTimestamp.tcp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
+		"at the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
+	useLocalTimestampUDP = flagutil.NewArrayBool("syslog.useLocalTimestamp.udp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
+		"at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
+)
+
+// MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports
+//
+// This function must be called after flag.Parse().
+//
+// MustStop() must be called in order to free up resources occupied by the initialized syslog parser.
+func MustInit() {
+	if workersStopCh != nil {
+		logger.Panicf("BUG: MustInit() called twice without MustStop() call")
+	}
+	workersStopCh = make(chan struct{})
+
+	for argIdx, addr := range *listenAddrTCP {
+		workersWG.Add(1)
+		go func(addr string, argIdx int) {
+			runTCPListener(addr, argIdx)
+			workersWG.Done()
+		}(addr, argIdx)
+	}
+
+	for argIdx, addr := range *listenAddrUDP {
+		workersWG.Add(1)
+		go func(addr string, argIdx int) {
+			runUDPListener(addr, argIdx)
+			workersWG.Done()
+		}(addr, argIdx)
+	}
+
+	currentYear := time.Now().Year()
+	globalCurrentYear.Store(int64(currentYear))
+	workersWG.Add(1)
+	go func() {
+		ticker := time.NewTicker(time.Minute)
+		for {
+			select {
+			case <-workersStopCh:
+				ticker.Stop()
+				workersWG.Done()
+				return
+			case <-ticker.C:
+				currentYear := time.Now().Year()
+				globalCurrentYear.Store(int64(currentYear))
+			}
+		}
+	}()
+
+	if *syslogTimezone != "" {
+		tz, err := time.LoadLocation(*syslogTimezone)
+		if err != nil {
+			logger.Fatalf("cannot parse -syslog.timezone=%q: %s", *syslogTimezone, err)
+		}
+		globalTimezone = tz
+	} else {
+		globalTimezone = time.Local
+	}
+}
+
+var (
+	globalCurrentYear atomic.Int64
+	globalTimezone    *time.Location
+)
+
+var (
+	workersWG     sync.WaitGroup
+	workersStopCh chan struct{}
+)
+
+// MustStop stops syslog parser initialized via MustInit()
+func MustStop() {
+	close(workersStopCh)
+	workersWG.Wait()
+	workersStopCh = nil
+}
+
+func runUDPListener(addr string, argIdx int) {
+	ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr)
+	if err != nil {
+		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
+	}
+
+	tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
+	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
+	}
+
+	compressMethod := compressMethodUDP.GetOptionalArg(argIdx)
+	checkCompressMethod(compressMethod, addr, "udp")
+
+	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
+
+	doneCh := make(chan struct{})
+	go func() {
+		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
+		close(doneCh)
+	}()
+
+	logger.Infof("started accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
+	<-workersStopCh
+	if err := ln.Close(); err != nil {
+		logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err)
+	}
+	<-doneCh
+	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
+}
+
+func runTCPListener(addr string, argIdx int) {
+	var tlsConfig *tls.Config
+	if tlsEnable.GetOptionalArg(argIdx) {
+		certFile := tlsCertFile.GetOptionalArg(argIdx)
+		keyFile := tlsKeyFile.GetOptionalArg(argIdx)
+		tc, err := netutil.GetServerTLSConfig(certFile, keyFile, *tlsMinVersion, *tlsCipherSuites)
+		if err != nil {
+			logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s",
+				certFile, keyFile, *tlsMinVersion, *tlsCipherSuites, err)
+		}
+		tlsConfig = tc
+	}
+	ln, err := netutil.NewTCPListener("syslog", addr, false, tlsConfig)
+	if err != nil {
+		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
+	}
+
+	tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
+	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
+	}
+
+	compressMethod := compressMethodTCP.GetOptionalArg(argIdx)
+	checkCompressMethod(compressMethod, addr, "tcp")
+
+	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
+
+	doneCh := make(chan struct{})
+	go func() {
+		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
+		close(doneCh)
+	}()
+
+	logger.Infof("started accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
+	<-workersStopCh
+	if err := ln.Close(); err != nil {
+		logger.Fatalf("syslog: cannot close TCP listener at %s: %s", addr, err)
+	}
+	<-doneCh
+	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
+}
+
+func checkCompressMethod(compressMethod, addr, protocol string) {
+	switch compressMethod {
+	case "", "none", "gzip", "deflate":
+		return
+	default:
+		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
+	}
+}
+
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+	gomaxprocs := cgroup.AvailableCPUs()
+	var wg sync.WaitGroup
+	localAddr := ln.LocalAddr()
+	for i := 0; i < gomaxprocs; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			var bb bytesutil.ByteBuffer
+			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
+			for {
+				bb.Reset()
+				bb.B = bb.B[:cap(bb.B)]
+				n, remoteAddr, err := ln.ReadFrom(bb.B)
+				if err != nil {
+					udpErrorsTotal.Inc()
+					var ne net.Error
+					if errors.As(err, &ne) {
+						if ne.Temporary() {
+							logger.Errorf("syslog: temporary error when listening for UDP at %q: %s", localAddr, err)
+							time.Sleep(time.Second)
+							continue
+						}
+						if strings.Contains(err.Error(), "use of closed network connection") {
+							break
+						}
+					}
+					logger.Errorf("syslog: cannot read UDP data from %s at %s: %s", remoteAddr, localAddr, err)
+					continue
+				}
+				bb.B = bb.B[:n]
+				udpRequestsTotal.Inc()
+				if err := processStream(bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
+					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+	var cm ingestserver.ConnsMap
+	cm.Init("syslog")
+
+	var wg sync.WaitGroup
+	addr := ln.Addr()
+	for {
+		c, err := ln.Accept()
+		if err != nil {
+			var ne net.Error
+			if errors.As(err, &ne) {
+				if ne.Temporary() {
+					logger.Errorf("syslog: temporary error when listening for TCP addr %q: %s", addr, err)
+					time.Sleep(time.Second)
+					continue
+				}
+				if strings.Contains(err.Error(), "use of closed network connection") {
+					break
+				}
+				logger.Fatalf("syslog: unrecoverable error when accepting TCP connections at %q: %s", addr, err)
+			}
+			logger.Fatalf("syslog: unexpected error when accepting TCP connections at %q: %s", addr, err)
+		}
+		if !cm.Add(c) {
+			_ = c.Close()
+			break
+		}
+
+		wg.Add(1)
+		go func() {
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
+				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
+			}
+
+			cm.Delete(c)
+			_ = c.Close()
+			wg.Done()
+		}()
+	}
+
+	cm.CloseAll(0)
+	wg.Wait()
+}
+
+// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
+func processStream(r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
+	if err := vlstorage.CanWriteData(); err != nil {
+		return err
+	}
+
+	lmp := cp.NewLogMessageProcessor()
+	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
+	lmp.MustClose()
+
+	return err
+}
+
+func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	switch compressMethod {
+	case "", "none":
+	case "gzip":
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read gzipped data: %w", err)
+		}
+		r = zr
+	case "deflate":
+		zr, err := common.GetZlibReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read deflated data: %w", err)
+		}
+		r = zr
+	default:
+		logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
+	}
+
+	err := processUncompressedStream(r, useLocalTimestamp, lmp)
+
+	switch compressMethod {
+	case "gzip":
+		zr := r.(*gzip.Reader)
+		common.PutGzipReader(zr)
+	case "deflate":
+		zr := r.(io.ReadCloser)
+		common.PutZlibReader(zr)
+	}
+
+	return err
+}
+
+func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	wcr := writeconcurrencylimiter.GetReader(r)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	slr := getSyslogLineReader(wcr)
+	defer putSyslogLineReader(slr)
+
+	n := 0
+	for {
+		ok := slr.nextLine()
+		wcr.DecConcurrency()
+		if !ok {
+			break
+		}
+
+		currentYear := int(globalCurrentYear.Load())
+		err := processLine(slr.line, currentYear, globalTimezone, useLocalTimestamp, lmp)
+		if err != nil {
+			errorsTotal.Inc()
+			return fmt.Errorf("cannot read line #%d: %s", n, err)
+		}
+		n++
+		rowsIngestedTotal.Inc()
+	}
+	return slr.Error()
+}
+
+type syslogLineReader struct {
+	line []byte
+
+	br  *bufio.Reader
+	err error
+}
+
+func (slr *syslogLineReader) reset(r io.Reader) {
+	slr.line = slr.line[:0]
+	slr.br.Reset(r)
+	slr.err = nil
+}
+
+// Error returns the last error occurred in slr.
+func (slr *syslogLineReader) Error() error {
+	if slr.err == nil || slr.err == io.EOF {
+		return nil
+	}
+	return slr.err
+}
+
+// nextLine reads the next syslog line from slr and stores it at slr.line.
+//
+// false is returned if the next line cannot be read. Error() must be called in this case
+// in order to verify whether there is an error or just slr stream has been finished.
+func (slr *syslogLineReader) nextLine() bool {
+	if slr.err != nil {
+		return false
+	}
+
+again:
+	prefix, err := slr.br.ReadSlice(' ')
+	if err != nil {
+		if err != io.EOF {
+			slr.err = fmt.Errorf("cannot read message frame prefix: %w", err)
+			return false
+		}
+		if len(prefix) == 0 {
+			slr.err = err
+			return false
+		}
+	}
+	// skip empty lines
+	for len(prefix) > 0 && prefix[0] == '\n' {
+		prefix = prefix[1:]
+	}
+	if len(prefix) == 0 {
+		// An empty prefix or a prefix with empty lines - try reading yet another prefix.
+		goto again
+	}
+
+	if prefix[0] >= '0' && prefix[0] <= '9' {
+		// This is octet-counting method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#msgxfer
+		msgLenStr := bytesutil.ToUnsafeString(prefix[:len(prefix)-1])
+		msgLen, err := strconv.ParseUint(msgLenStr, 10, 64)
+		if err != nil {
+			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
+			return false
+		}
+		if maxMsgLen := insertutils.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
+			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
+			return false
+		}
+		slr.line = slicesutil.SetLength(slr.line, int(msgLen))
+		if _, err := io.ReadFull(slr.br, slr.line); err != nil {
+			slr.err = fmt.Errorf("cannot read message with size %d bytes: %w", msgLen, err)
+			return false
+		}
+		return true
+	}
+
+	// This is octet-stuffing method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#octet-stuffing-legacy
+	slr.line = append(slr.line[:0], prefix...)
+	for {
+		line, err := slr.br.ReadSlice('\n')
+		if err == nil {
+			slr.line = append(slr.line, line[:len(line)-1]...)
+			return true
+		}
+		if err == io.EOF {
+			slr.line = append(slr.line, line...)
+			return true
+		}
+		if err == bufio.ErrBufferFull {
+			slr.line = append(slr.line, line...)
+			continue
+		}
+		slr.err = fmt.Errorf("cannot read message in octet-stuffing method: %w", err)
+		return false
+	}
+}
+
+func getSyslogLineReader(r io.Reader) *syslogLineReader {
+	v := syslogLineReaderPool.Get()
+	if v == nil {
+		br := bufio.NewReaderSize(r, 64*1024)
+		return &syslogLineReader{
+			br: br,
+		}
+	}
+	slr := v.(*syslogLineReader)
+	slr.reset(r)
+	return slr
+}
+
+func putSyslogLineReader(slr *syslogLineReader) {
+	syslogLineReaderPool.Put(slr)
+}
+
+var syslogLineReaderPool sync.Pool
+
+func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	p := logstorage.GetSyslogParser(currentYear, timezone)
+	lineStr := bytesutil.ToUnsafeString(line)
+	p.Parse(lineStr)
+
+	var ts int64
+	if useLocalTimestamp {
+		ts = time.Now().UnixNano()
+	} else {
+		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
+		if err != nil {
+			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
+		}
+		ts = nsecs
+	}
+	logstorage.RenameField(p.Fields, "message", "_msg")
+	lmp.AddRow(ts, p.Fields)
+	logstorage.PutSyslogParser(p)
+
+	return nil
+}
+
+var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
+
+	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
+
+	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
+	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
+)

app/vlinsert/syslog/syslog_test.go

@@ -0,0 +1,130 @@
+package syslog
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+)
+
+func TestSyslogLineReader_Success(t *testing.T) {
+	f := func(data string, linesExpected []string) {
+		t.Helper()
+
+		r := bytes.NewBufferString(data)
+		slr := getSyslogLineReader(r)
+		defer putSyslogLineReader(slr)
+
+		var lines []string
+		for slr.nextLine() {
+			lines = append(lines, string(slr.line))
+		}
+		if err := slr.Error(); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if !reflect.DeepEqual(lines, linesExpected) {
+			t.Fatalf("unexpected lines read;\ngot\n%q\nwant\n%q", lines, linesExpected)
+		}
+	}
+
+	f("", nil)
+	f("\n", nil)
+	f("\n\n\n", nil)
+
+	f("foobar", []string{"foobar"})
+	f("foobar\n", []string{"foobar\n"})
+	f("\n\nfoo\n\nbar\n\n", []string{"foo\n\nbar\n\n"})
+
+	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...`, []string{"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches..."})
+
+	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
+
+48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
+
+`, []string{
+		"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...",
+		"<165>Jun  4 12:08:33 abcd systemd[345]: abc defg",
+		`<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`,
+	})
+}
+
+func TestSyslogLineReader_Failure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		r := bytes.NewBufferString(data)
+		slr := getSyslogLineReader(r)
+		defer putSyslogLineReader(slr)
+
+		if slr.nextLine() {
+			t.Fatalf("expecting failure to read the first line")
+		}
+		if err := slr.Error(); err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// invalid format for message size
+	f("12foo bar")
+
+	// too big message size
+	f("123 aa")
+	f("1233423432 abc")
+}
+
+func TestProcessStreamInternal_Success(t *testing.T) {
+	f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		MustInit()
+		defer MustStop()
+
+		globalTimezone = time.UTC
+		globalCurrentYear.Store(int64(currentYear))
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal(r, "", false, tlp); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	data := `Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
+
+48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
+`
+	currentYear := 2023
+	rowsExpected := 3
+	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
+	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
+}
+
+func TestProcessStreamInternal_Failure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		MustInit()
+		defer MustStop()
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal(r, "", false, tlp); err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// invalid format for message size
+	f("12foo bar")
+
+	// too big message size
+	f("123 foo")
+	f("123456789 bar")
+}

app/vlogsgenerator/README.md

@@ -50,7 +50,8 @@ Below is an example output:
   "u64_0": "4810489083243239145",
   "float_0": "1.868",
   "ip_0": "250.34.75.125",
-  "timestamp_0": "1799-03-16T01:34:18.311Z"
+  "timestamp_0": "1799-03-16T01:34:18.311Z",
+  "json_0": "{\"foo\":\"bar_3\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":32}"
 }
 {
   "_time": "2024-05-08T14:34:00.854Z",
@@ -70,7 +71,8 @@ Below is an example output:
   "u64_0": "6593354256620219850",
   "float_0": "1.085",
   "ip_0": "253.151.88.158",
-  "timestamp_0": "2042-10-05T16:42:57.082Z"
+  "timestamp_0": "2042-10-05T16:42:57.082Z",
+  "json_0": "{\"foo\":\"bar_5\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":27}"
 }
 ```
 
@@ -92,13 +94,13 @@ These flags can be inspected by running `vlogsgenerator -help`. Below are the mo
 
 * `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
 * `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to generate.
+* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to generate.
 * `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
 
 The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
 
 For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
-[log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields), where every logs stream contains `10_000` log entries,
+[log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields), where every logs stream contains `10_000` log entries,
 and writes them to `http://localhost:9428/insert/jsonline`:
 
 ```
@@ -111,7 +113,7 @@ bin/vlogsgenerator \
 
 ### Churn rate
 
-It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
+It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
 by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
 logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
 but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:

app/vlogsgenerator/main.go

@@ -27,30 +27,32 @@ var (
 
 	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
 	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
-	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields")
+	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
 	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
 		"during data generation")
 	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
 	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 
 	statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
 )
@@ -263,6 +265,9 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 			timestamp := toISO8601(int64(rand.Uint64()))
 			fmt.Fprintf(bw, `,"timestamp_%d":"%s"`, j, timestamp)
 		}
+		for j := 0; j < *jsonFieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"json_%d":"{\"foo\":\"bar_%d\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":%d}"`, j, rand.Intn(10), rand.Intn(100))
+		}
 		fmt.Fprintf(bw, "}\n")
 
 		logEntriesCount.Add(1)

app/vlselect/logsql/hits_response.qtpl

@@ -0,0 +1,70 @@
+{% import (
+	"slices"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+) %}
+
+{% stripspace %}
+
+// FieldsForHits formats labels for /select/logsql/hits response
+{% func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) %}
+{
+	{% if len(columns) > 0 %}
+		{%q= columns[0].Name %}:{%q= columns[0].Values[rowIdx] %}
+		{% for _, c := range columns[1:] %}
+			,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
+		{% endfor %}
+	{% endif %}
+}
+{% endfunc %}
+
+{% func HitsSeries(m map[string]*hitsSeries) %}
+{
+	{% code
+		sortedKeys := make([]string, 0, len(m))
+		for k := range m {
+			sortedKeys = append(sortedKeys, k)
+		}
+		slices.Sort(sortedKeys)
+	%}
+	"hits":[
+		{% if len(sortedKeys) > 0 %}
+			{%= hitsSeriesLine(m, sortedKeys[0]) %}
+			{% for _, k := range sortedKeys[1:] %}
+				,{%= hitsSeriesLine(m, k) %}
+			{% endfor %}
+		{% endif %}
+	]
+}
+{% endfunc %}
+
+{% func hitsSeriesLine(m map[string]*hitsSeries, k string) %}
+{
+	{% code
+		hs := m[k]
+		hs.sort()
+		timestamps := hs.timestamps
+		hits := hs.hits
+	%}
+	"fields":{%s= k %},
+	"timestamps":[
+		{% if len(timestamps) > 0 %}
+			{%q= timestamps[0] %}
+			{% for _, ts := range timestamps[1:] %}
+				,{%q= ts %}
+			{% endfor %}
+		{% endif %}
+	],
+	"values":[
+		{% if len(hits) > 0 %}
+			{%dul= hits[0] %}
+			{% for _, v := range hits[1:] %}
+				,{%dul= v %}
+			{% endfor %}
+		{% endif %}
+	],
+	"total":{%dul= hs.hitsTotal %}
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/hits_response.qtpl.go

@@ -0,0 +1,223 @@
+// Code generated by qtc from "hits_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlselect/logsql/hits_response.qtpl:1
+package logsql
+
+//line app/vlselect/logsql/hits_response.qtpl:1
+import (
+	"slices"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// FieldsForHits formats labels for /select/logsql/hits response
+
+//line app/vlselect/logsql/hits_response.qtpl:10
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/hits_response.qtpl:10
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/hits_response.qtpl:10
+func StreamFieldsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/hits_response.qtpl:10
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/hits_response.qtpl:12
+	if len(columns) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:13
+		qw422016.N().Q(columns[0].Name)
+//line app/vlselect/logsql/hits_response.qtpl:13
+		qw422016.N().S(`:`)
+//line app/vlselect/logsql/hits_response.qtpl:13
+		qw422016.N().Q(columns[0].Values[rowIdx])
+//line app/vlselect/logsql/hits_response.qtpl:14
+		for _, c := range columns[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:14
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:15
+			qw422016.N().Q(c.Name)
+//line app/vlselect/logsql/hits_response.qtpl:15
+			qw422016.N().S(`:`)
+//line app/vlselect/logsql/hits_response.qtpl:15
+			qw422016.N().Q(c.Values[rowIdx])
+//line app/vlselect/logsql/hits_response.qtpl:16
+		}
+//line app/vlselect/logsql/hits_response.qtpl:17
+	}
+//line app/vlselect/logsql/hits_response.qtpl:17
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/hits_response.qtpl:19
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:19
+func WriteFieldsForHits(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	StreamFieldsForHits(qw422016, columns, rowIdx)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/hits_response.qtpl:19
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:19
+func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) string {
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/hits_response.qtpl:19
+	WriteFieldsForHits(qb422016, columns, rowIdx)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	return qs422016
+//line app/vlselect/logsql/hits_response.qtpl:19
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:21
+func StreamHitsSeries(qw422016 *qt422016.Writer, m map[string]*hitsSeries) {
+//line app/vlselect/logsql/hits_response.qtpl:21
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/hits_response.qtpl:24
+	sortedKeys := make([]string, 0, len(m))
+	for k := range m {
+		sortedKeys = append(sortedKeys, k)
+	}
+	slices.Sort(sortedKeys)
+
+//line app/vlselect/logsql/hits_response.qtpl:29
+	qw422016.N().S(`"hits":[`)
+//line app/vlselect/logsql/hits_response.qtpl:31
+	if len(sortedKeys) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:32
+		streamhitsSeriesLine(qw422016, m, sortedKeys[0])
+//line app/vlselect/logsql/hits_response.qtpl:33
+		for _, k := range sortedKeys[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:33
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:34
+			streamhitsSeriesLine(qw422016, m, k)
+//line app/vlselect/logsql/hits_response.qtpl:35
+		}
+//line app/vlselect/logsql/hits_response.qtpl:36
+	}
+//line app/vlselect/logsql/hits_response.qtpl:36
+	qw422016.N().S(`]}`)
+//line app/vlselect/logsql/hits_response.qtpl:39
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:39
+func WriteHitsSeries(qq422016 qtio422016.Writer, m map[string]*hitsSeries) {
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	StreamHitsSeries(qw422016, m)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/hits_response.qtpl:39
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:39
+func HitsSeries(m map[string]*hitsSeries) string {
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/hits_response.qtpl:39
+	WriteHitsSeries(qb422016, m)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	return qs422016
+//line app/vlselect/logsql/hits_response.qtpl:39
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:41
+func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k string) {
+//line app/vlselect/logsql/hits_response.qtpl:41
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/hits_response.qtpl:44
+	hs := m[k]
+	hs.sort()
+	timestamps := hs.timestamps
+	hits := hs.hits
+
+//line app/vlselect/logsql/hits_response.qtpl:48
+	qw422016.N().S(`"fields":`)
+//line app/vlselect/logsql/hits_response.qtpl:49
+	qw422016.N().S(k)
+//line app/vlselect/logsql/hits_response.qtpl:49
+	qw422016.N().S(`,"timestamps":[`)
+//line app/vlselect/logsql/hits_response.qtpl:51
+	if len(timestamps) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:52
+		qw422016.N().Q(timestamps[0])
+//line app/vlselect/logsql/hits_response.qtpl:53
+		for _, ts := range timestamps[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:53
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:54
+			qw422016.N().Q(ts)
+//line app/vlselect/logsql/hits_response.qtpl:55
+		}
+//line app/vlselect/logsql/hits_response.qtpl:56
+	}
+//line app/vlselect/logsql/hits_response.qtpl:56
+	qw422016.N().S(`],"values":[`)
+//line app/vlselect/logsql/hits_response.qtpl:59
+	if len(hits) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:60
+		qw422016.N().DUL(hits[0])
+//line app/vlselect/logsql/hits_response.qtpl:61
+		for _, v := range hits[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:61
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:62
+			qw422016.N().DUL(v)
+//line app/vlselect/logsql/hits_response.qtpl:63
+		}
+//line app/vlselect/logsql/hits_response.qtpl:64
+	}
+//line app/vlselect/logsql/hits_response.qtpl:64
+	qw422016.N().S(`],"total":`)
+//line app/vlselect/logsql/hits_response.qtpl:66
+	qw422016.N().DUL(hs.hitsTotal)
+//line app/vlselect/logsql/hits_response.qtpl:66
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/hits_response.qtpl:68
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:68
+func writehitsSeriesLine(qq422016 qtio422016.Writer, m map[string]*hitsSeries, k string) {
+//line app/vlselect/logsql/hits_response.qtpl:68
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/hits_response.qtpl:68
+	streamhitsSeriesLine(qw422016, m, k)
+//line app/vlselect/logsql/hits_response.qtpl:68
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/hits_response.qtpl:68
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:68
+func hitsSeriesLine(m map[string]*hitsSeries, k string) string {
+//line app/vlselect/logsql/hits_response.qtpl:68
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/hits_response.qtpl:68
+	writehitsSeriesLine(qb422016, m, k)
+//line app/vlselect/logsql/hits_response.qtpl:68
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/hits_response.qtpl:68
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/hits_response.qtpl:68
+	return qs422016
+//line app/vlselect/logsql/hits_response.qtpl:68
+}

app/vlselect/logsql/logsql.go

@@ -5,52 +5,230 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-// ProcessQueryRequest handles /select/logsql/query request.
-func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+// ProcessHitsRequest handles /select/logsql/hits request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-hits-stats
+func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
 
-	// Parse query
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQuery(qStr)
+	// Obtain step
+	stepStr := r.FormValue("step")
+	if stepStr == "" {
+		stepStr = "1d"
+	}
+	step, err := promutils.ParseDuration(stepStr)
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse query [%s]: %s", qStr, err)
+		httpserver.Errorf(w, r, "cannot parse 'step' arg: %s", err)
+		return
+	}
+	if step <= 0 {
+		httpserver.Errorf(w, r, "'step' must be bigger than zero")
+	}
+
+	// Obtain offset
+	offsetStr := r.FormValue("offset")
+	if offsetStr == "" {
+		offsetStr = "0s"
+	}
+	offset, err := promutils.ParseDuration(offsetStr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse 'offset' arg: %s", err)
 		return
 	}
 
-	// Parse optional start and end args
-	start, okStart, err := getTimeNsec(r, "start")
+	// Obtain field entries
+	fields := r.Form["field"]
+
+	// Obtain limit on the number of top fields entries.
+	fieldsLimit, err := httputils.GetInt(r, "fields_limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	end, okEnd, err := getTimeNsec(r, "end")
+	if fieldsLimit < 0 {
+		fieldsLimit = 0
+	}
+
+	// Prepare the query for hits count.
+	q.Optimize()
+	q.DropAllPipes()
+	q.AddCountByTimePipe(int64(step), int64(offset), fields)
+
+	var mLock sync.Mutex
+	m := make(map[string]*hitsSeries)
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
+			return
+		}
+
+		timestampValues := columns[0].Values
+		hitsValues := columns[len(columns)-1].Values
+		columns = columns[1 : len(columns)-1]
+
+		bb := blockResultPool.Get()
+		for i := range timestamps {
+			timestampStr := strings.Clone(timestampValues[i])
+			hitsStr := strings.Clone(hitsValues[i])
+			hits, err := strconv.ParseUint(hitsStr, 10, 64)
+			if err != nil {
+				logger.Panicf("BUG: cannot parse hitsStr=%q: %s", hitsStr, err)
+			}
+
+			bb.Reset()
+			WriteFieldsForHits(bb, columns, i)
+
+			mLock.Lock()
+			hs, ok := m[string(bb.B)]
+			if !ok {
+				k := string(bb.B)
+				hs = &hitsSeries{}
+				m[k] = hs
+			}
+			hs.timestamps = append(hs.timestamps, timestampStr)
+			hs.hits = append(hs.hits, hits)
+			hs.hitsTotal += hits
+			mLock.Unlock()
+		}
+		blockResultPool.Put(bb)
+	}
+
+	// Execute the query
+	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
+		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
+		return
+	}
+
+	m = getTopHitsSeries(m, fieldsLimit)
+
+	// Write response
+	w.Header().Set("Content-Type", "application/json")
+	WriteHitsSeries(w, m)
+}
+
+func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
+	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
+		return m
+	}
+
+	type fieldsHits struct {
+		fieldsStr string
+		hs        *hitsSeries
+	}
+	a := make([]fieldsHits, 0, len(m))
+	for fieldsStr, hs := range m {
+		a = append(a, fieldsHits{
+			fieldsStr: fieldsStr,
+			hs:        hs,
+		})
+	}
+	sort.Slice(a, func(i, j int) bool {
+		return a[i].hs.hitsTotal > a[j].hs.hitsTotal
+	})
+
+	hitsOther := make(map[string]uint64)
+	for _, x := range a[fieldsLimit:] {
+		for i, timestampStr := range x.hs.timestamps {
+			hitsOther[timestampStr] += x.hs.hits[i]
+		}
+	}
+	var hsOther hitsSeries
+	for timestampStr, hits := range hitsOther {
+		hsOther.timestamps = append(hsOther.timestamps, timestampStr)
+		hsOther.hits = append(hsOther.hits, hits)
+		hsOther.hitsTotal += hits
+	}
+
+	mNew := make(map[string]*hitsSeries, fieldsLimit+1)
+	for _, x := range a[:fieldsLimit] {
+		mNew[x.fieldsStr] = x.hs
+	}
+	mNew["{}"] = &hsOther
+
+	return mNew
+}
+
+type hitsSeries struct {
+	hitsTotal  uint64
+	timestamps []string
+	hits       []uint64
+}
+
+func (hs *hitsSeries) sort() {
+	sort.Sort(hs)
+}
+
+func (hs *hitsSeries) Len() int {
+	return len(hs.timestamps)
+}
+
+func (hs *hitsSeries) Swap(i, j int) {
+	hs.timestamps[i], hs.timestamps[j] = hs.timestamps[j], hs.timestamps[i]
+	hs.hits[i], hs.hits[j] = hs.hits[j], hs.hits[i]
+}
+
+func (hs *hitsSeries) Less(i, j int) bool {
+	return hs.timestamps[i] < hs.timestamps[j]
+}
+
+// ProcessFieldNamesRequest handles /select/logsql/field_names request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-field-names
+func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	if okStart || okEnd {
-		if !okStart {
-			start = math.MinInt64
-		}
-		if !okEnd {
-			end = math.MaxInt64
-		}
-		q.AddTimeFilter(start, end)
+
+	// Obtain field names for the given query
+	q.Optimize()
+	fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain field names: %s", err)
+		return
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, fieldNames)
+}
+
+// ProcessFieldValuesRequest handles /select/logsql/field_values request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-field-values
+func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse fieldName query arg
+	fieldName := r.FormValue("field")
+	if fieldName == "" {
+		httpserver.Errorf(w, r, "missing 'field' query arg")
+		return
 	}
 
 	// Parse limit query arg
@@ -59,17 +237,378 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain unique values for the given field
+	q.Optimize()
+	values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain values for field %q: %s", fieldName, err)
+		return
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, values)
+}
+
+// ProcessStreamFieldNamesRequest processes /select/logsql/stream_field_names request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-names
+func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Obtain stream field names for the given query
+	q.Optimize()
+	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream field names: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, names)
+}
+
+// ProcessStreamFieldValuesRequest processes /select/logsql/stream_field_values request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-values
+func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse fieldName query arg
+	fieldName := r.FormValue("field")
+	if fieldName == "" {
+		httpserver.Errorf(w, r, "missing 'field' query arg")
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain stream field values for the given query and the given fieldName
+	q.Optimize()
+	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream field values: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, values)
+}
+
+// ProcessStreamIDsRequest processes /select/logsql/stream_ids request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream_ids
+func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain streamIDs for the given query
+	q.Optimize()
+	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream_ids: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, streamIDs)
+}
+
+// ProcessStreamsRequest processes /select/logsql/streams request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-streams
+func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain streams for the given query
+	q.Optimize()
+	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain streams: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, streams)
+}
+
+// ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
+func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	liveTailRequests.Inc()
+	defer liveTailRequests.Dec()
+
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if !q.CanLiveTail() {
+		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
+	}
+	q.Optimize()
+
+	refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
+
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	tp := newTailProcessor(cancel)
+
+	ticker := time.NewTicker(refreshInterval)
+	defer ticker.Stop()
+
+	end := time.Now().UnixNano()
+	doneCh := ctxWithCancel.Done()
+	flusher, ok := w.(http.Flusher)
+	if !ok {
+		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
+	}
+	for {
+		start := end - tailOffsetNsecs
+		end = time.Now().UnixNano()
+
+		qCopy := q.Clone()
+		qCopy.AddTimeFilter(start, end)
+		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
+			httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
+			return
+		}
+		resultRows, err := tp.getTailRows()
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot get tail results for query [%q]: %s", q, err)
+			return
+		}
+		if len(resultRows) > 0 {
+			WriteJSONRows(w, resultRows)
+			flusher.Flush()
+		}
+
+		select {
+		case <-doneCh:
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+var liveTailRequests = metrics.NewCounter(`vl_live_tailing_requests`)
+
+const tailOffsetNsecs = 5e9
+
+type logRow struct {
+	timestamp int64
+	fields    []logstorage.Field
+}
+
+func sortLogRows(rows []logRow) {
+	sort.SliceStable(rows, func(i, j int) bool {
+		return rows[i].timestamp < rows[j].timestamp
+	})
+}
+
+type tailProcessor struct {
+	cancel func()
+
+	mu sync.Mutex
+
+	perStreamRows  map[string][]logRow
+	lastTimestamps map[string]int64
+
+	err error
+}
+
+func newTailProcessor(cancel func()) *tailProcessor {
+	return &tailProcessor{
+		cancel: cancel,
+
+		perStreamRows:  make(map[string][]logRow),
+		lastTimestamps: make(map[string]int64),
+	}
+}
+
+func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	if len(timestamps) == 0 {
+		return
+	}
+
+	tp.mu.Lock()
+	defer tp.mu.Unlock()
+
+	if tp.err != nil {
+		return
+	}
+
+	// Make sure columns contain _time field, since it is needed for proper tail work.
+	hasTime := false
+	for _, c := range columns {
+		if c.Name == "_time" {
+			hasTime = true
+			break
+		}
+	}
+	if !hasTime {
+		tp.err = fmt.Errorf("missing _time field")
+		tp.cancel()
+		return
+	}
+
+	// Copy block rows to tp.perStreamRows
+	for i, timestamp := range timestamps {
+		streamID := ""
+		fields := make([]logstorage.Field, len(columns))
+		for j, c := range columns {
+			name := strings.Clone(c.Name)
+			value := strings.Clone(c.Values[i])
+
+			fields[j] = logstorage.Field{
+				Name:  name,
+				Value: value,
+			}
+
+			if name == "_stream_id" {
+				streamID = value
+			}
+		}
+
+		tp.perStreamRows[streamID] = append(tp.perStreamRows[streamID], logRow{
+			timestamp: timestamp,
+			fields:    fields,
+		})
+	}
+}
+
+func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
+	if tp.err != nil {
+		return nil, tp.err
+	}
+
+	var resultRows []logRow
+	for streamID, rows := range tp.perStreamRows {
+		sortLogRows(rows)
+
+		lastTimestamp, ok := tp.lastTimestamps[streamID]
+		if ok {
+			// Skip already written rows
+			for len(rows) > 0 && rows[0].timestamp <= lastTimestamp {
+				rows = rows[1:]
+			}
+		}
+		if len(rows) > 0 {
+			resultRows = append(resultRows, rows...)
+			tp.lastTimestamps[streamID] = rows[len(rows)-1].timestamp
+		}
+	}
+	clear(tp.perStreamRows)
+
+	sortLogRows(resultRows)
+
+	tailRows := make([][]logstorage.Field, len(resultRows))
+	for i, row := range resultRows {
+		tailRows[i] = row.fields
+	}
+
+	return tailRows, nil
+}
+
+// ProcessQueryRequest handles /select/logsql/query request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
+func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	bw := getBufferedWriter(w)
+	defer func() {
+		bw.FlushIgnoreErrors()
+		putBufferedWriter(bw)
+	}()
+	w.Header().Set("Content-Type", "application/stream+json")
+
 	if limit > 0 {
+		if q.CanReturnLastNResults() {
+			rows, err := getLastNQueryResults(ctx, tenantIDs, q, limit)
+			if err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+				return
+			}
+			bb := blockResultPool.Get()
+			b := bb.B
+			for i := range rows {
+				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
+				b = append(b, '\n')
+				bw.WriteIgnoreErrors(b)
+			}
+			bb.B = b
+			blockResultPool.Put(bb)
+			return
+		}
+
 		q.AddPipeLimit(uint64(limit))
 	}
 	q.Optimize()
 
-	tenantIDs := []logstorage.TenantID{tenantID}
-
-	bw := getBufferedWriter(w)
-
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
 			return
 		}
 
@@ -81,29 +620,152 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		blockResultPool.Put(bb)
 	}
 
-	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
-	err = vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock)
-
-	bw.FlushIgnoreErrors()
-	putBufferedWriter(bw)
-
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", qStr, err)
+	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
+		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
+		return
 	}
-
 }
 
 var blockResultPool bytesutil.ByteBufferPool
 
+type row struct {
+	timestamp int64
+	fields    []logstorage.Field
+}
+
+func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+	limitUpper := 2 * limit
+	q.AddPipeLimit(uint64(limitUpper))
+	q.Optimize()
+	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
+	if err != nil {
+		return nil, err
+	}
+	if len(rows) < limitUpper {
+		// Fast path - the requested time range contains up to limitUpper rows.
+		rows = getLastNRows(rows, limit)
+		return rows, nil
+	}
+
+	// Slow path - search for the time range containing up to limitUpper rows.
+	start, end := q.GetFilterTimeRange()
+	d := end/2 - start/2
+	start += d
+
+	qOrig := q
+	for {
+		q = qOrig.Clone()
+		q.AddTimeFilter(start, end)
+		rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
+			rows = getLastNRows(rows, limit)
+			return rows, nil
+		}
+
+		lastBit := d & 1
+		d /= 2
+		if len(rows) > limit {
+			start += d
+		} else {
+			start -= d + lastBit
+		}
+	}
+}
+
+func getLastNRows(rows []row, limit int) []row {
+	sort.Slice(rows, func(i, j int) bool {
+		return rows[i].timestamp < rows[j].timestamp
+	})
+	if len(rows) > limit {
+		rows = rows[len(rows)-limit:]
+	}
+	return rows
+}
+
+func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var rows []row
+	var rowsLock sync.Mutex
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		rowsLock.Lock()
+		defer rowsLock.Unlock()
+
+		for i, timestamp := range timestamps {
+			fields := make([]logstorage.Field, len(columns))
+			for j := range columns {
+				f := &fields[j]
+				f.Name = strings.Clone(columns[j].Name)
+				f.Value = strings.Clone(columns[j].Values[i])
+			}
+			rows = append(rows, row{
+				timestamp: timestamp,
+				fields:    fields,
+			})
+		}
+
+		if len(rows) >= limit {
+			cancel()
+		}
+	}
+	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
+		return nil, err
+	}
+
+	return rows, nil
+}
+
+func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {
+	// Extract tenantID
+	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot obtain tenanID: %w", err)
+	}
+	tenantIDs := []logstorage.TenantID{tenantID}
+
+	// Parse query
+	qStr := r.FormValue("query")
+	q, err := logstorage.ParseQuery(qStr)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
+	}
+
+	// Parse optional start and end args
+	start, okStart, err := getTimeNsec(r, "start")
+	if err != nil {
+		return nil, nil, err
+	}
+	end, okEnd, err := getTimeNsec(r, "end")
+	if err != nil {
+		return nil, nil, err
+	}
+	if okStart || okEnd {
+		if !okStart {
+			start = math.MinInt64
+		}
+		if !okEnd {
+			end = math.MaxInt64
+		}
+		q.AddTimeFilter(start, end)
+	}
+
+	return q, tenantIDs, nil
+}
+
 func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
 	s := r.FormValue(argName)
 	if s == "" {
 		return 0, false, nil
 	}
-	currentTimestamp := float64(time.Now().UnixNano()) / 1e9
-	secs, err := promutils.ParseTimeAt(s, currentTimestamp)
+	currentTimestamp := time.Now().UnixNano()
+	nsecs, err := promutils.ParseTimeAt(s, currentTimestamp)
 	if err != nil {
 		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
 	}
-	return int64(secs * 1e9), true, nil
+	return nsecs, true, nil
 }

app/vlselect/logsql/logsql.qtpl

@@ -0,0 +1,32 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+) %}
+
+{% stripspace %}
+
+// ValuesWithHitsJSON generates JSON from the given values.
+{% func ValuesWithHitsJSON(values []logstorage.ValueWithHits) %}
+{
+	"values":{%= valuesWithHitsJSONArray(values) %}
+}
+{% endfunc %}
+
+{% func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) %}
+[
+	{% if len(values) > 0 %}
+		{%= valueWithHitsJSON(values[0]) %}
+		{% for _, v := range values[1:] %}
+			,{%= valueWithHitsJSON(v) %}
+		{% endfor %}
+	{% endif %}
+]
+{% endfunc %}
+
+{% func valueWithHitsJSON(v logstorage.ValueWithHits) %}
+{
+	"value":{%q= v.Value %},
+	"hits":{%dul= v.Hits %}
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/logsql.qtpl.go

@@ -0,0 +1,152 @@
+// Code generated by qtc from "logsql.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlselect/logsql/logsql.qtpl:1
+package logsql
+
+//line app/vlselect/logsql/logsql.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ValuesWithHitsJSON generates JSON from the given values.
+
+//line app/vlselect/logsql/logsql.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/logsql.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/logsql.qtpl:8
+func StreamValuesWithHitsJSON(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:8
+	qw422016.N().S(`{"values":`)
+//line app/vlselect/logsql/logsql.qtpl:10
+	streamvaluesWithHitsJSONArray(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:10
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:12
+func WriteValuesWithHitsJSON(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:12
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+	StreamValuesWithHitsJSON(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:12
+func ValuesWithHitsJSON(values []logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:12
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:12
+	WriteValuesWithHitsJSON(qb422016, values)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:14
+func streamvaluesWithHitsJSONArray(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:14
+	qw422016.N().S(`[`)
+//line app/vlselect/logsql/logsql.qtpl:16
+	if len(values) > 0 {
+//line app/vlselect/logsql/logsql.qtpl:17
+		streamvalueWithHitsJSON(qw422016, values[0])
+//line app/vlselect/logsql/logsql.qtpl:18
+		for _, v := range values[1:] {
+//line app/vlselect/logsql/logsql.qtpl:18
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/logsql.qtpl:19
+			streamvalueWithHitsJSON(qw422016, v)
+//line app/vlselect/logsql/logsql.qtpl:20
+		}
+//line app/vlselect/logsql/logsql.qtpl:21
+	}
+//line app/vlselect/logsql/logsql.qtpl:21
+	qw422016.N().S(`]`)
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:23
+func writevaluesWithHitsJSONArray(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+	streamvaluesWithHitsJSONArray(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:23
+func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:23
+	writevaluesWithHitsJSONArray(qb422016, values)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:25
+func streamvalueWithHitsJSON(qw422016 *qt422016.Writer, v logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:25
+	qw422016.N().S(`{"value":`)
+//line app/vlselect/logsql/logsql.qtpl:27
+	qw422016.N().Q(v.Value)
+//line app/vlselect/logsql/logsql.qtpl:27
+	qw422016.N().S(`,"hits":`)
+//line app/vlselect/logsql/logsql.qtpl:28
+	qw422016.N().DUL(v.Hits)
+//line app/vlselect/logsql/logsql.qtpl:28
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/logsql.qtpl:30
+}
+
+//line app/vlselect/logsql/logsql.qtpl:30
+func writevalueWithHitsJSON(qq422016 qtio422016.Writer, v logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:30
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+	streamvalueWithHitsJSON(qw422016, v)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+}
+
+//line app/vlselect/logsql/logsql.qtpl:30
+func valueWithHitsJSON(v logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:30
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:30
+	writevalueWithHitsJSON(qb422016, v)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:30
+}

app/vlselect/main.go

@@ -1,6 +1,7 @@
 package vlselect
 
 import (
+	"context"
 	"embed"
 	"flag"
 	"fmt"
@@ -13,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -23,7 +23,7 @@ var (
 		"See also -search.maxQueueDuration")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
+	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden on a per-query basis via 'timeout' query arg")
 )
 
 func getDefaultMaxConcurrentRequests() int {
@@ -75,10 +75,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		// Skip requests, which do not start with /select/, since these aren't our requests.
 		return false
 	}
-	path = strings.TrimPrefix(path, "/select")
 	path = strings.ReplaceAll(path, "//", "/")
 
-	if path == "/vmui" {
+	if path == "/select/vmui" {
 		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
 		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
 		// is hidden behind vmauth or similar proxy.
@@ -87,65 +86,129 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.Redirect(w, newURL)
 		return true
 	}
-	if strings.HasPrefix(path, "/vmui/") {
-		if strings.HasPrefix(path, "/vmui/static/") {
+	if strings.HasPrefix(path, "/select/vmui/") {
+		if strings.HasPrefix(path, "/select/vmui/static/") {
 			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
 			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
 			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
 			w.Header().Set("Cache-Control", "max-age=31536000")
 		}
-		r.URL.Path = path
+		r.URL.Path = strings.TrimPrefix(path, "/select")
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
 	}
 
-	// Limit the number of concurrent queries, which can consume big amounts of CPU.
+	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
 	startTime := time.Now()
 	ctx := r.Context()
-	stopCh := ctx.Done()
+	d := getMaxQueryDuration(r)
+	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
+	defer cancel()
+
+	stopCh := ctxWithTimeout.Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
-		d := getMaxQueryDuration(r)
-		if d > *maxQueueDuration {
-			d = *maxQueueDuration
-		}
-		t := timerpool.Get(d)
 		select {
 		case concurrencyLimitCh <- struct{}{}:
-			timerpool.Put(t)
 			defer func() { <-concurrencyLimitCh }()
 		case <-stopCh:
-			timerpool.Put(t)
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-				time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			return true
-		case <-t.C:
-			timerpool.Put(t)
-			concurrencyLimitTimeout.Inc()
-			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
-					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusServiceUnavailable,
+			switch ctxWithTimeout.Err() {
+			case context.Canceled:
+				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+				requestURI := httpserver.GetRequestURI(r)
+				logger.Infof("client has canceled the pending request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+					time.Since(startTime).Seconds(), remoteAddr, requestURI)
+			case context.DeadlineExceeded:
+				concurrencyLimitTimeout.Inc()
+				err := &httpserver.ErrorWithStatusCode{
+					Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
+						"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
+						"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
+						"to pass bigger value to 'timeout' query arg",
+						d.Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
+					StatusCode: http.StatusServiceUnavailable,
+				}
+				httpserver.Errorf(w, r, "%s", err)
 			}
-			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 	}
 
-	switch {
-	case path == "/logsql/query":
+	if path == "/select/logsql/tail" {
+		logsqlTailRequests.Inc()
+		// Process live tailing request without timeout (e.g. use ctx instead of ctxWithTimeout),
+		// since it is OK to run live tailing requests for very long time.
+		logsql.ProcessLiveTailRequest(ctx, w, r)
+		return true
+	}
+
+	ok := processSelectRequest(ctxWithTimeout, w, r, path)
+	if !ok {
+		return false
+	}
+
+	err := ctxWithTimeout.Err()
+	switch err {
+	case nil:
+		// nothing to do
+	case context.Canceled:
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Infof("client has canceled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+			time.Since(startTime).Seconds(), remoteAddr, requestURI)
+	case context.DeadlineExceeded:
+		err = &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
+				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", d.Seconds(), maxQueryDuration),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+		httpserver.Errorf(w, r, "%s", err)
+	default:
+		httpserver.Errorf(w, r, "unexpected error: %s", err)
+	}
+
+	return true
+}
+
+func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
+	httpserver.EnableCORS(w, r)
+	switch path {
+	case "/select/logsql/field_names":
+		logsqlFieldNamesRequests.Inc()
+		logsql.ProcessFieldNamesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/field_values":
+		logsqlFieldValuesRequests.Inc()
+		logsql.ProcessFieldValuesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/hits":
+		logsqlHitsRequests.Inc()
+		logsql.ProcessHitsRequest(ctx, w, r)
+		return true
+	case "/select/logsql/query":
 		logsqlQueryRequests.Inc()
-		httpserver.EnableCORS(w, r)
 		logsql.ProcessQueryRequest(ctx, w, r)
 		return true
+	case "/select/logsql/stream_field_names":
+		logsqlStreamFieldNamesRequests.Inc()
+		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stream_field_values":
+		logsqlStreamFieldValuesRequests.Inc()
+		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stream_ids":
+		logsqlStreamIDsRequests.Inc()
+		logsql.ProcessStreamIDsRequest(ctx, w, r)
+		return true
+	case "/select/logsql/streams":
+		logsqlStreamsRequests.Inc()
+		logsql.ProcessStreamsRequest(ctx, w, r)
+		return true
 	default:
 		return false
 	}
@@ -165,5 +228,13 @@ func getMaxQueryDuration(r *http.Request) time.Duration {
 }
 
 var (
-	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
+	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
+	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
+	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
+	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
+	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
+	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
+	logsqlTailRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
 )

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.bc07cc78.css",
-    "main.js": "./static/js/main.8e7757ef.js",
+    "main.css": "./static/css/main.1041c3d4.css",
+    "main.js": "./static/js/main.62a82f4a.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.da86c2db4f0b05e286b0.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.bc07cc78.css",
-    "static/js/main.8e7757ef.js"
+    "static/css/main.1041c3d4.css",
+    "static/js/main.62a82f4a.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8e7757ef.js"></script><link href="./static/css/main.bc07cc78.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.62a82f4a.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.62a82f4a.js.LICENSE.txt

@@ -5,7 +5,7 @@
 */
 
 /**
- * @remix-run/router v1.15.1
+ * @remix-run/router v1.17.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.22.1
+ * React Router DOM v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +27,7 @@
  */
 
 /**
- * React Router v6.22.1
+ * React Router v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlselect/vmui/static/media/MetricsQL.aaabf95f2c9bf356bde4.md

@@ -107,7 +107,7 @@ The list of MetricsQL features on top of PromQL:
 * Trailing commas on all the lists are allowed - label filters, function args and with expressions.
   For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
   This simplifies maintenance of multi-line queries.
-* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a value MetricsQL expression.
+* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a valid MetricsQL expression.
 * Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
   It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
   Additionally, the following escape sequences are supported:
@@ -328,6 +328,7 @@ See also [increases_over_time](#increases_over_time).
 
 `default_rollup(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 value on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+Compared to [last_over_time](#last_over_time) it accounts for [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) to detect stale series.
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),

app/vlstorage/main.go

@@ -20,19 +20,21 @@ import (
 var (
 	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
 		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
+		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
+	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
+		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
 	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory with the VictoriaLogs data; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#storage")
+		"see https://docs.victoriametrics.com/victorialogs/#retention")
+	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
+		"see https://docs.victoriametrics.com/victorialogs/#storage")
 	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
 		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
 		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
 		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
 	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows")
 	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/ ; see also -logNewStreams")
+		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
 		"the storage stops accepting new data")
 )
@@ -49,12 +51,13 @@ func Init() {
 		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
 	}
 	cfg := &logstorage.StorageConfig{
-		Retention:             retentionPeriod.Duration(),
-		FlushInterval:         *inmemoryDataFlushInterval,
-		FutureRetention:       futureRetention.Duration(),
-		LogNewStreams:         *logNewStreams,
-		LogIngestedRows:       *logIngestedRows,
-		MinFreeDiskSpaceBytes: minFreeDiskSpaceBytes.N,
+		Retention:              retentionPeriod.Duration(),
+		MaxDiskSpaceUsageBytes: maxDiskSpaceUsageBytes.N,
+		FlushInterval:          *inmemoryDataFlushInterval,
+		FutureRetention:        futureRetention.Duration(),
+		LogNewStreams:          *logNewStreams,
+		LogIngestedRows:        *logIngestedRows,
+		MinFreeDiskSpaceBytes:  minFreeDiskSpaceBytes.N,
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
@@ -77,7 +80,7 @@ func Init() {
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics)
+	metrics.UnregisterSet(storageMetrics, true)
 	storageMetrics = nil
 
 	strg.MustClose()
@@ -107,10 +110,48 @@ func MustAddRows(lr *logstorage.LogRows) {
 }
 
 // RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock func(workerID uint, timestamps []int64, columns []logstorage.BlockColumn)) error {
+func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
 	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
 }
 
+// GetFieldNames executes q and returns field names seen in results.
+func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return strg.GetFieldNames(ctx, tenantIDs, q)
+}
+
+// GetFieldValues executes q and returns unique values for the fieldName seen in results.
+//
+// If limit > 0, then up to limit unique values are returned.
+func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+}
+
+// GetStreamFieldNames executes q and returns stream field names seen in results.
+func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
+}
+
+// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
+//
+// If limit > 0, then up to limit unique stream field values are returned.
+func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+}
+
+// GetStreams executes q and returns streams seen in query results.
+//
+// If limit > 0, then up to limit unique streams are returned.
+func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreams(ctx, tenantIDs, q, limit)
+}
+
+// GetStreamIDs executes q and returns streamIDs seen in query results.
+//
+// If limit > 0, then up to limit unique streamIDs are returned.
+func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
+}
+
 func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	var ss logstorage.StorageStats
 	strg.UpdateStats(&ss)

app/vmagent/main.go

@@ -24,7 +24,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -37,7 +36,6 @@ import (
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
-	statsdserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
@@ -63,10 +61,6 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	statsdListenAddr = flag.String("statsdListenAddr", "", "TCP and UDP address to listen for Statsd plaintext data. Usually :8125 must be set. Doesn't work if empty. "+
-		"See also -statsdListenAddr.useProxyProtocol")
-	statsdUseProxyProtocol = flag.Bool("statsdListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -statsdListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
@@ -76,8 +70,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -86,7 +80,6 @@ var (
 var (
 	influxServer       *influxserver.Server
 	graphiteServer     *graphiteserver.Server
-	statsdServer       *statsdserver.Server
 	opentsdbServer     *opentsdbserver.Server
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
@@ -121,7 +114,7 @@ func main() {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
+			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
@@ -144,9 +137,6 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
 	}
-	if len(*statsdListenAddr) > 0 {
-		statsdServer = statsdserver.MustStart(*statsdListenAddr, *statsdUseProxyProtocol, statsd.InsertHandler)
-	}
 	if len(*opentsdbListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
 		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
@@ -182,9 +172,6 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer.MustStop()
 	}
-	if len(*statsdListenAddr) > 0 {
-		statsdServer.MustStop()
-	}
 	if len(*opentsdbListenAddr) > 0 {
 		opentsdbServer.MustStop()
 	}
@@ -447,7 +434,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -456,7 +443,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -466,7 +453,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
 		}
 		promscrapeConfigReloadRequests.Inc()

app/vmagent/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmagent/prometheusimport/request_handler_test.go

@@ -26,11 +26,11 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Errorf("unxepected error %s", err)
+		t.Fatalf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 

app/vmagent/remotewrite/client.go

@@ -11,16 +11,18 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -36,7 +38,7 @@ var (
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
+	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -118,7 +120,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
-		DialContext:         statDial,
+		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
 		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
@@ -298,6 +300,11 @@ func (c *client) runWorker() {
 		if !ok {
 			return
 		}
+		if len(block) == 0 {
+			// skip empty data blocks from sending
+			// see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241
+			continue
+		}
 		go func() {
 			startTime := time.Now()
 			ch <- c.sendBlock(block)

app/vmagent/remotewrite/pendingseries.go

@@ -109,10 +109,9 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss       []prompbmarshal.TimeSeries
-	labels    []prompbmarshal.Label
-	samples   []prompbmarshal.Sample
-	exemplars []prompbmarshal.Exemplar
+	tss     []prompbmarshal.TimeSeries
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
 
 	// buf holds labels data
 	buf []byte
@@ -130,7 +129,6 @@ func (wr *writeRequest) reset() {
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
-	wr.exemplars = wr.exemplars[:0]
 	wr.buf = wr.buf[:0]
 }
 
@@ -202,7 +200,6 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	labelsDst := wr.labels
 	labelsLen := len(wr.labels)
 	samplesDst := wr.samples
-	exemplarsDst := wr.exemplars
 	buf := wr.buf
 	for i := range src.Labels {
 		labelsDst = append(labelsDst, prompbmarshal.Label{})
@@ -219,12 +216,8 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	samplesDst = append(samplesDst, src.Samples...)
 	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
 
-	exemplarsDst = append(exemplarsDst, src.Exemplars...)
-	dst.Exemplars = exemplarsDst[len(exemplarsDst)-len(src.Exemplars):]
-
 	wr.samples = samplesDst
 	wr.labels = labelsDst
-	wr.exemplars = exemplarsDst
 	wr.buf = buf
 }
 
@@ -236,6 +229,7 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 		// Nothing to push
 		return true
 	}
+
 	marshalConcurrencyCh <- struct{}{}
 
 	bb := writeRequestBufPool.Get()
@@ -272,8 +266,6 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 	if len(wr.Timeseries) == 1 {
 		// A single time series left. Recursively split its samples into smaller parts if possible.
 		samples := wr.Timeseries[0].Samples
-		exemplars := wr.Timeseries[0].Exemplars
-
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
 			return true
@@ -285,16 +277,11 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 			return false
 		}
 		wr.Timeseries[0].Samples = samples[n:]
-		// We do not want to send exemplars twice
-		wr.Timeseries[0].Exemplars = nil
-
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
-			wr.Timeseries[0].Exemplars = exemplars
 			return false
 		}
 		wr.Timeseries[0].Samples = samples
-		wr.Timeseries[0].Exemplars = exemplars
 		return true
 	}
 	timeseries := wr.Timeseries

app/vmagent/remotewrite/pendingseries_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestPushWriteRequest(t *testing.T) {
 	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
-	expectedBlockLensProm := []int{248, 1952, 17433, 180381, 1861994}
-	expectedBlockLensVM := []int{170, 575, 4748, 44936, 367096}
+	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
+	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
 	for i, rowsCount := range rowsCounts {
 		expectedBlockLenProm := expectedBlockLensProm[i]
 		expectedBlockLenVM := expectedBlockLensVM[i]
@@ -34,7 +34,7 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 			return true
 		}
 		if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
-			t.Fatalf("cannot push data to to remote storage")
+			t.Fatalf("cannot push data to remote storage")
 		}
 		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
 			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
@@ -59,20 +59,6 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 				Value: fmt.Sprintf("value_%d_%d", i, j),
 			})
 		}
-		exemplar := prompbmarshal.Exemplar{
-			Labels: []prompbmarshal.Label{
-				{
-					Name:  "trace_id",
-					Value: "123456",
-				},
-				{
-					Name:  "log_id",
-					Value: "987654",
-				},
-			},
-			Value:     float64(i),
-			Timestamp: 1000 * int64(i),
-		}
 		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
 			Labels: labels,
 			Samples: []prompbmarshal.Sample{
@@ -81,10 +67,6 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 					Timestamp: 1000 * int64(i),
 				},
 			},
-
-			Exemplars: []prompbmarshal.Exemplar{
-				exemplar,
-			},
 		})
 	}
 	return &wr

app/vmagent/remotewrite/relabel.go

@@ -181,7 +181,7 @@ func (rctx *relabelCtx) reset() {
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return &relabelCtx{}
 	},
 }

app/vmagent/remotewrite/remotewrite.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -87,34 +88,16 @@ var (
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
-	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
-		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
-	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
-	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current aggregation interval "+
-		"for the corresponding -remoteWrite.streamAggr.config . See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
-	streamAggrDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
-
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence ."+
-		"See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flagutil.NewArrayBool("remoteWrite.dropSamplesOnOverload", "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
+		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
+	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
 )
 
 var (
-	// rwctxs contains statically populated entries when -remoteWrite.url is specified.
-	rwctxs []*remoteWriteCtx
+	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
+	rwctxsGlobal []*remoteWriteCtx
 
 	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
 	defaultAuthToken = &auth.Token{}
@@ -127,8 +110,11 @@ var (
 		StatusCode: http.StatusTooManyRequests,
 	}
 
-	// disableOnDiskQueueAll is set to true if all remoteWrite.urls were configured to disable persistent queue via disableOnDiskQueue
-	disableOnDiskQueueAll bool
+	// disableOnDiskQueueAny is set to true if at least a single -remoteWrite.url is configured with -remoteWrite.disableOnDiskQueue
+	disableOnDiskQueueAny bool
+
+	// dropSamplesOnFailureGlobal is set to true if -remoteWrite.dropSamplesOnOverload is set or if multiple -remoteWrite.disableOnDiskQueue options are set.
+	dropSamplesOnFailureGlobal bool
 )
 
 // MultitenancyEnabled returns true if -enableMultitenantHandlers is specified.
@@ -139,6 +125,12 @@ func MultitenancyEnabled() bool {
 // Contains the current relabelConfigs.
 var allRelabelConfigs atomic.Pointer[relabelConfigs]
 
+// Contains the current global stream aggregators.
+var sasGlobal atomic.Pointer[streamaggr.Aggregators]
+
+// Contains the current global deduplicator.
+var deduplicatorGlobal *streamaggr.Deduplicator
+
 // maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
 // since it may lead to high memory usage due to big number of buffers.
 var maxQueues = cgroup.AvailableCPUs() * 16
@@ -215,17 +207,18 @@ func Init() {
 	relabelConfigSuccess.Set(1)
 	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
-	if len(*remoteWriteURLs) > 0 {
-		rwctxs = newRemoteWriteCtxs(nil, *remoteWriteURLs)
-	}
+	initStreamAggrConfigGlobal()
 
-	disableOnDiskQueueAll = true
-	for _, v := range *disableOnDiskQueue {
-		if !v {
-			disableOnDiskQueueAll = false
-			break
-		}
-	}
+	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+
+	disableOnDiskQueues := []bool(*disableOnDiskQueue)
+	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
+
+	// Samples must be dropped if multiple -remoteWrite.disableOnDiskQueue options are configured and at least a single is set to true.
+	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
+	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
+	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
+	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
 
 	dropDanglingQueues()
 
@@ -235,9 +228,9 @@ func Init() {
 		defer configReloaderWG.Done()
 		for {
 			select {
-			case <-sighupCh:
 			case <-configReloaderStopCh:
 				return
+			case <-sighupCh:
 			}
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
@@ -256,8 +249,8 @@ func dropDanglingQueues() {
 	// In case if there were many persistent queues with identical *remoteWriteURLs
 	// the queue with the last index will be dropped.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxs))
-	for _, rwctx := range rwctxs {
+	existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
 		existingQueues[rwctx.fq.Dirname()] = struct{}{}
 	}
 
@@ -274,7 +267,7 @@ func dropDanglingQueues() {
 		}
 	}
 	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
 	}
 }
 
@@ -301,12 +294,6 @@ var (
 	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
-func reloadStreamAggrConfigs() {
-	for _, rwctx := range rwctxs {
-		rwctx.reinitStreamAggr()
-	}
-}
-
 func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
@@ -342,8 +329,10 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 	return rwctxs
 }
 
-var configReloaderStopCh = make(chan struct{})
-var configReloaderWG sync.WaitGroup
+var (
+	configReloaderStopCh = make(chan struct{})
+	configReloaderWG     sync.WaitGroup
+)
 
 // StartIngestionRateLimiter starts ingestion rate limiter.
 //
@@ -381,10 +370,16 @@ func Stop() {
 	close(configReloaderStopCh)
 	configReloaderWG.Wait()
 
-	for _, rwctx := range rwctxs {
+	sasGlobal.Load().MustStop()
+	if deduplicatorGlobal != nil {
+		deduplicatorGlobal.MustStop()
+		deduplicatorGlobal = nil
+	}
+
+	for _, rwctx := range rwctxsGlobal {
 		rwctx.MustStop()
 	}
-	rwctxs = nil
+	rwctxsGlobal = nil
 
 	if sl := hourlySeriesLimiter; sl != nil {
 		sl.MustStop()
@@ -396,6 +391,8 @@ func Stop() {
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
 //
+// PushDropSamplesOnFailure drops wr samples if they cannot be sent to -remoteWrite.url by any reason.
+//
 // PushDropSamplesOnFailure can modify wr contents.
 func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
@@ -408,7 +405,7 @@ func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
 func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, false)
+	return tryPush(at, wr, dropSamplesOnFailureGlobal)
 }
 
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
@@ -425,24 +422,20 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		tenantRctx = getRelabelCtx()
 		defer putRelabelCtx(tenantRctx)
 	}
-	rowsCount := getRowsCount(tss)
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
 	// if some of remote storage systems cannot keep up with the data ingestion rate.
-	// this shortcut is only applicable if all remote writes have disableOnDiskQueue = true
-	if disableOnDiskQueueAll {
-		for _, rwctx := range rwctxs {
-			if rwctx.fq.IsWriteBlocked() {
-				rwctx.pushFailures.Inc()
-				if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
-					// Just drop samples
-					rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-					continue
-				}
-				return false
-			}
-		}
+	rwctxs, ok := getEligibleRemoteWriteCtxs(tss, forceDropSamplesOnFailure)
+	if !ok {
+		// At least a single remote write queue is blocked and dropSamplesOnFailure isn't set.
+		// Return false to the caller, so it could re-send samples again.
+		return false
+	}
+	if len(rwctxs) == 0 {
+		// All the remote write queues are skipped because they are blocked and dropSamplesOnFailure is set to true.
+		// Return true to the caller, so it doesn't re-send the samples again.
+		return true
 	}
 
 	var rctx *relabelCtx
@@ -452,11 +445,14 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}
+	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 
+	sas := sasGlobal.Load()
+
 	for len(tss) > 0 {
 		// Process big tss in smaller blocks in order to reduce the maximum memory usage
 		samplesCount := 0
@@ -491,14 +487,57 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if !tryPushBlockToRemoteStorages(tssBlock, forceDropSamplesOnFailure) {
+		if sas.IsEnabled() {
+			matchIdxs := matchIdxsPool.Get()
+			matchIdxs.B = sas.Push(tssBlock, matchIdxs.B)
+			if !*streamAggrGlobalKeepInput {
+				tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
+			}
+			matchIdxsPool.Put(matchIdxs)
+		} else if deduplicatorGlobal != nil {
+			deduplicatorGlobal.Push(tssBlock)
+			tssBlock = tssBlock[:0]
+		}
+		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
 	return true
 }
 
-func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
+	if !disableOnDiskQueueAny {
+		return rwctxsGlobal, true
+	}
+
+	// This code is applicable if at least a single remote storage has -disableOnDiskQueue
+	rwctxs := make([]*remoteWriteCtx, 0, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
+		if !rwctx.fq.IsWriteBlocked() {
+			rwctxs = append(rwctxs, rwctx)
+		} else {
+			rwctx.pushFailures.Inc()
+			if !forceDropSamplesOnFailure {
+				return nil, false
+			}
+			rowsCount := getRowsCount(tss)
+			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+		}
+	}
+	return rwctxs, true
+}
+
+func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
+	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
+	if len(rwctxs) == 0 {
+		return
+	}
+	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
+		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
+	}
+}
+
+func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -517,7 +556,7 @@ func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDrop
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingBlockAmongRemoteStorages(tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
@@ -538,7 +577,7 @@ func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDrop
 	return !anyPushFailed.Load()
 }
 
-func tryShardingBlockAmongRemoteStorages(tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -725,10 +764,8 @@ type remoteWriteCtx struct {
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
 
-	streamAggrKeepInput   bool
-	streamAggrDropInput   bool
-	disableOnDiskQueue    bool
-	dropSamplesOnOverload bool
+	streamAggrKeepInput bool
+	streamAggrDropInput bool
 
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
@@ -753,6 +790,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
+
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -797,46 +835,20 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		c:   c,
 		pss: pss,
 
-		dropSamplesOnOverload: dropSamplesOnOverload.GetOptionalArg(argIdx),
-		disableOnDiskQueue:    isPQDisabled,
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 
-		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-	}
-
-	// Initialize sas
-	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
-	ignoreOldSamples := streamAggrIgnoreOldSamples.GetOptionalArg(argIdx)
-	if sasFile != "" {
-		opts := &streamaggr.Options{
-			DedupInterval:        dedupInterval,
-			DropInputLabels:      *streamAggrDropInputLabels,
-			IgnoreOldSamples:     ignoreOldSamples,
-			IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
-		}
-		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
-		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
-		}
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
-	} else if dedupInterval > 0 {
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels)
+		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
+	rwctx.initStreamAggrConfig()
 
 	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
 	// sas and deduplicator must be stopped before rwctx is closed
-	// because sas can write pending series to rwctx.pss if there are any
+	// because they can write pending series to rwctx.pss if there are any
 	sas := rwctx.sas.Swap(nil)
 	sas.MustStop()
 
@@ -863,12 +875,20 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 // TryPush sends tss series to the configured remote write endpoint
 //
-// TryPush can be called concurrently for multiple remoteWriteCtx,
-// so it shouldn't modify tss entries.
+// TryPush doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
 func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
-	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
+	defer func() {
+		if rctx == nil {
+			return
+		}
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
+	}()
+
+	// Apply relabeling
 	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
 	if pcs.Len() > 0 {
@@ -889,7 +909,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 
 	// Apply stream aggregation or deduplication if they are configured
 	sas := rwctx.sas.Load()
-	if sas != nil {
+	if sas.IsEnabled() {
 		matchIdxs := matchIdxsPool.Get()
 		matchIdxs.B = sas.Push(tss, matchIdxs.B)
 		if !rwctx.streamAggrKeepInput {
@@ -904,28 +924,22 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 		matchIdxsPool.Put(matchIdxs)
 	} else if rwctx.deduplicator != nil {
 		rwctx.deduplicator.Push(tss)
-		tss = tss[:0]
+		return true
 	}
 
-	// Try pushing the data to remote storage
-	ok := rwctx.tryPushInternal(tss)
-
-	// Return back relabeling contexts to the pool
-	if rctx != nil {
-		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
+	// Try pushing tss to remote storage
+	if rwctx.tryPushInternal(tss) {
+		return true
 	}
 
-	if !ok {
-		rwctx.pushFailures.Inc()
-		if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
-			rwctx.rowsDroppedOnPushFailure.Add(len(tss))
-			return true
-		}
+	// Couldn't push tss to remote storage
+	rwctx.pushFailures.Inc()
+	if forceDropSamplesOnFailure {
+		rowsCount := getRowsCount(tss)
+		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+		return true
 	}
-
-	return ok
+	return false
 }
 
 var matchIdxsPool bytesutil.ByteBufferPool
@@ -949,19 +963,26 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSe
 	if rwctx.tryPushInternal(tss) {
 		return
 	}
-	if !rwctx.disableOnDiskQueue {
+	if !rwctx.fq.IsPersistentQueueDisabled() {
 		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
 	}
 	rwctx.pushFailures.Inc()
-	if dropSamplesOnOverload.GetOptionalArg(rwctx.idx) {
-		rowsCount := getRowsCount(tss)
-		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-	}
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 }
 
 func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
+	defer func() {
+		if rctx == nil {
+			return
+		}
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
+	}()
+
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels in order to prevent
 		// from affecting time series for other remoteWrite.url configs.
@@ -974,53 +995,11 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
-	ok := pss[idx].TryPush(tss)
-
-	if rctx != nil {
-		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
-	}
-
-	return ok
-}
-
-func (rwctx *remoteWriteCtx) reinitStreamAggr() {
-	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if sasFile == "" {
-		// There is no stream aggregation for rwctx
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
-	opts := &streamaggr.Options{
-		DedupInterval:    streamAggrDedupInterval.GetOptionalArg(rwctx.idx),
-		DropInputLabels:  *streamAggrDropInputLabels,
-		IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(rwctx.idx),
-	}
-	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternalTrackDropped, opts)
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(0)
-		logger.Errorf("cannot reload stream aggregation config from -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", sasFile, err)
-		return
-	}
-	sas := rwctx.sas.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := rwctx.sas.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded stream aggregation configs at -remoteWrite.streamAggr.config=%q", sasFile)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("the config at -remoteWrite.streamAggr.config=%q wasn't changed", sasFile)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
+	return pss[idx].TryPush(tss)
 }
 
 var tssPool = &sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		a := []prompbmarshal.TimeSeries{}
 		return &a
 	},
@@ -1034,27 +1013,6 @@ func getRowsCount(tss []prompbmarshal.TimeSeries) int {
 	return rowsCount
 }
 
-// CheckStreamAggrConfigs checks configs pointed by -remoteWrite.streamAggr.config
-func CheckStreamAggrConfigs() error {
-	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-	for idx, sasFile := range *streamAggrConfig {
-		if sasFile == "" {
-			continue
-		}
-		opts := &streamaggr.Options{
-			DedupInterval:    streamAggrDedupInterval.GetOptionalArg(idx),
-			DropInputLabels:  *streamAggrDropInputLabels,
-			IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		}
-		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, opts)
-		if err != nil {
-			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)
-		}
-		sas.MustStop()
-	}
-	return nil
-}
-
 func newMapFromStrings(a []string) map[string]struct{} {
 	m := make(map[string]struct{}, len(a))
 	for _, s := range a {

app/vmagent/remotewrite/remotewrite_test.go

@@ -3,14 +3,12 @@ package remotewrite
 import (
 	"fmt"
 	"math"
-	"os"
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -79,24 +77,28 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil)
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
 		}
 
-		if len(streamAggrConfig) > 0 {
-			f := createFile(t, []byte(streamAggrConfig))
-			sas, err := streamaggr.LoadFromFile(f.Name(), nil, nil)
+		if streamAggrConfig != "" {
+			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
+			defer sas.MustStop()
 			rwctx.sas.Store(sas)
 		}
 
-		inputTss := mustParsePromMetrics(input)
+		offsetMsecs := time.Now().UnixMilli()
+		inputTss := prompbmarshal.MustParsePromMetrics(input, offsetMsecs)
 		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
 
 		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		rwctx.TryPush(inputTss, false)
+		if !rwctx.TryPush(inputTss, false) {
+			t.Fatalf("cannot push samples to rwctx")
+		}
 
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
@@ -165,51 +167,3 @@ metric{env="dev"} 15
 metric{env="bar"} 25
 `)
 }
-
-func mustParsePromMetrics(s string) []prompbmarshal.TimeSeries {
-	var rows prometheus.Rows
-	errLogger := func(s string) {
-		panic(fmt.Errorf("unexpected error when parsing Prometheus metrics: %s", s))
-	}
-	rows.UnmarshalWithErrLogger(s, errLogger)
-	var tss []prompbmarshal.TimeSeries
-	samples := make([]prompbmarshal.Sample, 0, len(rows.Rows))
-	for _, row := range rows.Rows {
-		labels := make([]prompbmarshal.Label, 0, len(row.Tags)+1)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: row.Metric,
-		})
-		for _, tag := range row.Tags {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  tag.Key,
-				Value: tag.Value,
-			})
-		}
-		samples = append(samples, prompbmarshal.Sample{
-			Value:     row.Value,
-			Timestamp: row.Timestamp,
-		})
-		ts := prompbmarshal.TimeSeries{
-			Labels:  labels,
-			Samples: samples[len(samples)-1:],
-		}
-		tss = append(tss, ts)
-	}
-	return tss
-}
-
-func createFile(t *testing.T, data []byte) *os.File {
-	t.Helper()
-	f, err := os.CreateTemp("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := os.WriteFile(f.Name(), data, 0644); err != nil {
-		t.Fatal(err)
-	}
-	if err := f.Sync(); err != nil {
-		t.Fatal(err)
-	}
-	return f
-}

app/vmagent/remotewrite/statconn.go

@@ -1,73 +0,0 @@
-package remotewrite
-
-import (
-	"context"
-	"net"
-	"sync/atomic"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-func statDial(ctx context.Context, _, addr string) (conn net.Conn, err error) {
-	network := netutil.GetTCPNetwork()
-	conn, err = netutil.DialMaybeSRV(ctx, network, addr)
-	dialsTotal.Inc()
-	if err != nil {
-		dialErrors.Inc()
-		return nil, err
-	}
-	conns.Inc()
-	sc := &statConn{
-		Conn: conn,
-	}
-	return sc, nil
-}
-
-var (
-	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
-	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
-	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
-)
-
-type statConn struct {
-	closed atomic.Int32
-	net.Conn
-}
-
-func (sc *statConn) Read(p []byte) (int, error) {
-	n, err := sc.Conn.Read(p)
-	connReadsTotal.Inc()
-	if err != nil {
-		connReadErrors.Inc()
-	}
-	connBytesRead.Add(n)
-	return n, err
-}
-
-func (sc *statConn) Write(p []byte) (int, error) {
-	n, err := sc.Conn.Write(p)
-	connWritesTotal.Inc()
-	if err != nil {
-		connWriteErrors.Inc()
-	}
-	connBytesWritten.Add(n)
-	return n, err
-}
-
-func (sc *statConn) Close() error {
-	err := sc.Conn.Close()
-	if sc.closed.Add(1) == 1 {
-		conns.Dec()
-	}
-	return err
-}
-
-var (
-	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
-	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
-	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
-	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
-	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
-	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
-)

app/vmagent/remotewrite/streamaggr.go

@@ -0,0 +1,240 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	// Global config
+	streamAggrGlobalConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
+		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
+	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation "+
+		"with -streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
+	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
+	streamAggrGlobalDedupInterval = flagutil.NewDuration("streamAggr.dedupInterval", "0s", "Input samples are de-duplicated with this interval on "+
+		"aggregator before optional aggregation with -streamAggr.config . "+
+		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
+	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
+		"current aggregation interval for aggregator. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
+	streamAggrGlobalIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start for "+
+		"aggregator. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from "+
+		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
+	streamAggrGlobalDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples for aggregator "+
+		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+
+	// Per URL config
+	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
+		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
+	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
+	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
+	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
+	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
+		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
+	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
+		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
+	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
+		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+)
+
+// CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
+func CheckStreamAggrConfigs() error {
+	// Check global config
+	sas, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		return err
+	}
+	sas.MustStop()
+
+	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
+		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
+	}
+
+	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+	for idx := range *streamAggrConfig {
+		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
+		if err != nil {
+			return err
+		}
+		sas.MustStop()
+	}
+	return nil
+}
+
+func reloadStreamAggrConfigs() {
+	reloadStreamAggrConfigGlobal()
+	for _, rwctx := range rwctxsGlobal {
+		rwctx.reloadStreamAggrConfig()
+	}
+}
+
+func reloadStreamAggrConfigGlobal() {
+	path := *streamAggrGlobalConfig
+	if path == "" {
+		return
+	}
+
+	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
+
+	sasNew, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
+		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		return
+	}
+
+	sas := sasGlobal.Load()
+	if !sasNew.Equal(sas) {
+		sasOld := sasGlobal.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
+	} else {
+		sasNew.MustStop()
+		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
+	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
+}
+
+func initStreamAggrConfigGlobal() {
+	sas, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
+	}
+	if sas != nil {
+		filePath := sas.FilePath()
+		sasGlobal.Store(sas)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
+	} else {
+		dedupInterval := streamAggrGlobalDedupInterval.Duration()
+		if dedupInterval > 0 {
+			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
+		}
+	}
+}
+
+func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
+	idx := rwctx.idx
+
+	sas, err := rwctx.newStreamAggrConfig()
+	if err != nil {
+		logger.Fatalf("cannot initialize stream aggregators: %s", err)
+	}
+	if sas != nil {
+		filePath := sas.FilePath()
+		rwctx.sas.Store(sas)
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
+		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
+	} else {
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+		if dedupInterval > 0 {
+			alias := fmt.Sprintf("dedup-%d", idx+1)
+			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
+		}
+	}
+}
+
+func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
+	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
+	if path == "" {
+		return
+	}
+
+	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
+
+	sasNew, err := rwctx.newStreamAggrConfig()
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
+		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		return
+	}
+
+	sas := rwctx.sas.Load()
+	if !sasNew.Equal(sas) {
+		sasOld := rwctx.sas.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
+	} else {
+		sasNew.MustStop()
+		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
+	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
+}
+
+func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
+	path := *streamAggrGlobalConfig
+	if path == "" {
+		return nil, nil
+	}
+
+	opts := &streamaggr.Options{
+		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
+		DropInputLabels:      *streamAggrGlobalDropInputLabels,
+		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
+		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+	}
+
+	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
+	if err != nil {
+		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
+	}
+	return sas, nil
+}
+
+func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
+	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
+}
+
+func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
+	path := streamAggrConfig.GetOptionalArg(idx)
+	if path == "" {
+		return nil, nil
+	}
+
+	alias := fmt.Sprintf("%d:secret-url", idx+1)
+	if *showRemoteWriteURL {
+		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
+	}
+	opts := &streamaggr.Options{
+		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
+		DropInputLabels:      *streamAggrDropInputLabels,
+		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
+		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+	}
+
+	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
+	if err != nil {
+		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
+	}
+	return sas, nil
+}

app/vmagent/statsd/request_handler.go

@@ -1,68 +0,0 @@
-package statsd
-
-import (
-	"io"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd/stream"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="statsd"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="statsd"}`)
-)
-
-// InsertHandler processes remote write for statsd plaintext protocol.
-//
-// See https://github.com/statsd/statsd/blob/master/docs/metric_types.md
-func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
-		return insertRows(nil, rows)
-	})
-}
-
-func insertRows(at *auth.Token, rows []parser.Row) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		labelsLen := len(labels)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: r.Metric,
-		})
-		for j := range r.Tags {
-			tag := &r.Tags[j]
-			labels = append(labels, prompbmarshal.Label{
-				Name:  tag.Key,
-				Value: tag.Value,
-			})
-		}
-		samples = append(samples, prompbmarshal.Sample{
-			Value:     r.Value,
-			Timestamp: r.Timestamp,
-		})
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[len(samples)-1:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	rowsPerInsert.Update(float64(len(rows)))
-	return nil
-}

app/vmalert-tool/Makefile

@@ -81,6 +81,9 @@ vmalert-tool-linux-ppc64le:
 vmalert-tool-linux-s390x:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
+vmalert-tool-linux-loong64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+
 vmalert-tool-linux-386:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert-tool/main.go

@@ -26,8 +26,14 @@ func main() {
 				UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html#Unit-testing-for-rules",
 				Flags: []cli.Flag{
 					&cli.StringSliceFlag{
-						Name:     "files",
-						Usage:    "files to run unittest with. Supports an array of values separated by comma or specified via multiple flags.",
+						Name: "files",
+						Usage: `File path or http url with test files. Supports an array of values separated by comma or specified via multiple flags.
+						Supports hierarchical patterns and regexpes.
+Examples:
+ -files="/path/to/file". Path to a single test file.
+ -files="http://<some-server-addr>/path/to/test.yaml". HTTP URL to a test file.
+ -files="dir/**/*.yaml". Includes all the .yaml files in "dir" subfolders recursively.
+ `,
 						Required: true,
 					},
 					&cli.BoolFlag{

app/vmalert-tool/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmalert-tool/unittest/input.go

@@ -18,6 +18,8 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
+var numReg = regexp.MustCompile(`\D?\d*\.?\d*\D?`)
+
 // series holds input_series defined in the test file
 type series struct {
 	Series string `yaml:"series"`
@@ -72,10 +74,7 @@ func writeInputSeries(input []series, interval *promutils.Duration, startStamp t
 		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
 	}
 
-	data, err := testutil.Compress(r)
-	if err != nil {
-		return fmt.Errorf("failed to compress data: %v", err)
-	}
+	data := testutil.Compress(r)
 	// write input series to vm
 	httpWrite(dst, bytes.NewBuffer(data))
 	vmstorage.Storage.DebugFlush()
@@ -86,13 +85,12 @@ func writeInputSeries(input []series, interval *promutils.Duration, startStamp t
 func parseInputValue(input string, origin bool) ([]sequenceValue, error) {
 	var res []sequenceValue
 	items := strings.Split(input, " ")
-	reg := regexp.MustCompile(`\D?\d*\D?`)
 	for _, item := range items {
 		if item == "stale" {
 			res = append(res, sequenceValue{Value: decimal.StaleNaN})
 			continue
 		}
-		vals := reg.FindAllString(item, -1)
+		vals := numReg.FindAllString(item, -1)
 		switch len(vals) {
 		case 1:
 			if vals[0] == "_" {

app/vmalert-tool/unittest/input_test.go

@@ -6,88 +6,61 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )
 
-func TestParseInputValue(t *testing.T) {
-	testCases := []struct {
-		input  string
-		exp    []sequenceValue
-		failed bool
-	}{
-		{
-			"",
-			nil,
-			true,
-		},
-		{
-			"testfailed",
-			nil,
-			true,
-		},
-		// stale doesn't support operations
-		{
-			"stalex3",
-			nil,
-			true,
-		},
-		{
-			"-4",
-			[]sequenceValue{{Value: -4}},
-			false,
-		},
-		{
-			"_",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"stale",
-			[]sequenceValue{{Value: decimal.StaleNaN}},
-			false,
-		},
-		{
-			"-4x1",
-			[]sequenceValue{{Value: -4}, {Value: -4}},
-			false,
-		},
-		{
-			"_x1",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"1+1x4",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 4}, {Value: 5}},
-			false,
-		},
-		{
-			"2-1x4",
-			[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
-			false,
-		},
-		{
-			"1+1x1 _ -4 stale 3+20x1",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
-			false,
-		},
+func TestParseInputValue_Failure(t *testing.T) {
+	f := func(input string) {
+		t.Helper()
+
+		_, err := parseInputValue(input, true)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
 	}
 
-	for _, tc := range testCases {
-		output, err := parseInputValue(tc.input, true)
-		if err != nil != tc.failed {
-			t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
+	f("")
+	f("testfailed")
+
+	// stale doesn't support operations
+	f("stalex3")
+}
+
+func TestParseInputValue_Success(t *testing.T) {
+	f := func(input string, outputExpected []sequenceValue) {
+		t.Helper()
+
+		output, err := parseInputValue(input, true)
+		if err != nil {
+			t.Fatalf("unexpected error in parseInputValue: %s", err)
 		}
-		if len(tc.exp) != len(output) {
-			t.Fatalf("expect %v, got %v", tc.exp, output)
+
+		if len(outputExpected) != len(output) {
+			t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
 		}
-		for i := 0; i < len(tc.exp); i++ {
-			if tc.exp[i].Omitted != output[i].Omitted {
-				t.Fatalf("expect %v, got %v", tc.exp, output)
+		for i := 0; i < len(outputExpected); i++ {
+			if outputExpected[i].Omitted != output[i].Omitted {
+				t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}
-			if tc.exp[i].Value != output[i].Value {
-				if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
+			if outputExpected[i].Value != output[i].Value {
+				if decimal.IsStaleNaN(outputExpected[i].Value) && decimal.IsStaleNaN(output[i].Value) {
 					continue
 				}
-				t.Fatalf("expect %v, got %v", tc.exp, output)
+				t.Fatalf("unexpeccted Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}
 		}
 	}
+
+	f("-4", []sequenceValue{{Value: -4}})
+
+	f("_", []sequenceValue{{Omitted: true}})
+
+	f("stale", []sequenceValue{{Value: decimal.StaleNaN}})
+
+	f("-4x1", []sequenceValue{{Value: -4}, {Value: -4}})
+
+	f("_x1", []sequenceValue{{Omitted: true}})
+
+	f("1+1x2 0.1 0.1+0.3x2 3.14", []sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}})
+
+	f("2-1x4", []sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}})
+
+	f("1+1x1 _ -4 stale 3+20x1", []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}})
 }

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -37,3 +37,6 @@ groups:
     rules:
       - record: t3
         expr: t1
+
+  - name: emptyGroup
+    rules: []

app/vmalert-tool/unittest/unittest.go

@@ -9,10 +9,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"sort"
+	"strings"
 	"time"
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -66,36 +68,38 @@ func UnitTest(files []string, disableGroupLabel bool) bool {
 	defer vminsert.Stop()
 	defer vmselect.Stop()
 	disableAlertgroupLabel = disableGroupLabel
-	return rulesUnitTest(files)
-}
 
-func rulesUnitTest(files []string) bool {
+	testfiles, err := config.ReadFromFS(files)
+	if err != nil {
+		fmt.Println("  FAILED")
+		fmt.Printf("\nfailed to read test files: \n%v", err)
+	}
 	var failed bool
-	for _, f := range files {
-		if err := ruleUnitTest(f); err != nil {
+	for fileName, file := range testfiles {
+		if err := ruleUnitTest(fileName, file); err != nil {
 			fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", f, err)
+			fmt.Printf("\nfailed to run unit test for file %q: \n%v", file, err)
 			failed = true
 		} else {
 			fmt.Println("  SUCCESS")
 		}
 	}
+
 	return failed
 }
 
-func ruleUnitTest(filename string) []error {
+func ruleUnitTest(filename string, content []byte) []error {
 	fmt.Println("\nUnit Testing: ", filename)
-	b, err := os.ReadFile(filename)
-	if err != nil {
-		return []error{fmt.Errorf("failed to read file: %w", err)}
-	}
-
 	var unitTestInp unitTestFile
-	if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
+	if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
 		return []error{fmt.Errorf("failed to unmarshal file: %w", err)}
 	}
-	if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
-		return []error{fmt.Errorf("failed to resolve path for `rule_files`: %w", err)}
+
+	// add file directory for rule files if needed
+	for i, rf := range unitTestInp.RuleFiles {
+		if rf != "" && !filepath.IsAbs(rf) && !strings.HasPrefix(rf, "http") {
+			unitTestInp.RuleFiles[i] = filepath.Join(filepath.Dir(filename), rf)
+		}
 	}
 
 	if unitTestInp.EvaluationInterval.Duration() == 0 {
@@ -139,24 +143,24 @@ func verifyTestGroup(group testGroup) error {
 	}
 	for _, at := range group.AlertRuleTests {
 		if at.Alertname == "" {
-			return fmt.Errorf("\n%s    missing required filed \"alertname\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"alertname\"", testGroupName)
 		}
 		if !disableAlertgroupLabel && at.GroupName == "" {
-			return fmt.Errorf("\n%s    missing required filed \"groupname\" when flag \"disableAlertGroupLabel\" is false", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"groupname\" when flag \"disableAlertgroupLabel\" is false", testGroupName)
 		}
 		if disableAlertgroupLabel && at.GroupName != "" {
-			return fmt.Errorf("\n%s    shouldn't set filed \"groupname\" when flag \"disableAlertGroupLabel\" is true", testGroupName)
+			return fmt.Errorf("\n%s    shouldn't set field \"groupname\" when flag \"disableAlertgroupLabel\" is true", testGroupName)
 		}
 		if at.EvalTime == nil {
-			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"eval_time\"", testGroupName)
 		}
 	}
 	for _, et := range group.MetricsqlExprTests {
 		if et.Expr == "" {
-			return fmt.Errorf("\n%s    missing required filed \"expr\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"expr\"", testGroupName)
 		}
 		if et.EvalTime == nil {
-			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"eval_time\"", testGroupName)
 		}
 	}
 	return nil
@@ -235,30 +239,6 @@ func tearDown() {
 	fs.MustRemoveAll(storagePath)
 }
 
-// resolveAndGlobFilepaths joins all relative paths in a configuration
-// with a given base directory and replaces all globs with matching files.
-func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
-	for i, rf := range utf.RuleFiles {
-		if rf != "" && !filepath.IsAbs(rf) {
-			utf.RuleFiles[i] = filepath.Join(baseDir, rf)
-		}
-	}
-
-	var globbedFiles []string
-	for _, rf := range utf.RuleFiles {
-		m, err := filepath.Glob(rf)
-		if err != nil {
-			return err
-		}
-		if len(m) == 0 {
-			fmt.Fprintln(os.Stderr, "  WARNING: no file match pattern", rf)
-		}
-		globbedFiles = append(globbedFiles, m...)
-	}
-	utf.RuleFiles = globbedFiles
-	return nil
-}
-
 func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
 	// set up vmstorage and http server for ingest and read queries
 	setUp()
@@ -316,11 +296,15 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	maxEvalTime := testStartTime.Add(tg.maxEvalTime())
 	for ts := testStartTime; ts.Before(maxEvalTime) || ts.Equal(maxEvalTime); ts = ts.Add(evalInterval) {
 		for _, g := range groups {
+			if len(g.Rules) == 0 {
+				continue
+			}
 			errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts)
 			for err := range errs {
 				if err != nil {
 					checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name,
 						ts, err))
+					return
 				}
 			}
 			// flush series after each group evaluation
@@ -374,7 +358,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 						if expAlert.ExpLabels == nil {
 							expAlert.ExpLabels = make(map[string]string)
 						}
-						// alertGroupNameLabel is added as additional labels when `disableAlertGroupLabel` is false
+						// alertGroupNameLabel is added as additional labels when `disableAlertgroupLabel` is false
 						if !disableAlertgroupLabel {
 							expAlert.ExpLabels["alertgroup"] = groupname
 						}

app/vmalert-tool/unittest/unittest_test.go

@@ -14,34 +14,33 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestUnitRule(t *testing.T) {
-	testCases := []struct {
-		name              string
-		disableGroupLabel bool
-		files             []string
-		failed            bool
-	}{
-		{
-			name:   "run multi files",
-			files:  []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
-			failed: false,
-		},
-		{
-			name:              "disable group label",
-			disableGroupLabel: true,
-			files:             []string{"./testdata/disable-group-label.yaml"},
-			failed:            false,
-		},
-		{
-			name:   "failing test",
-			files:  []string{"./testdata/failed-test.yaml"},
-			failed: true,
-		},
-	}
-	for _, tc := range testCases {
-		fail := UnitTest(tc.files, tc.disableGroupLabel)
-		if fail != tc.failed {
-			t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
+func TestUnitTest_Failure(t *testing.T) {
+	f := func(files []string) {
+		t.Helper()
+
+		failed := UnitTest(files, false)
+		if !failed {
+			t.Fatalf("expecting failed test")
 		}
 	}
+
+	// failing test
+	f([]string{"./testdata/failed-test.yaml"})
+}
+
+func TestUnitTest_Success(t *testing.T) {
+	f := func(disableGroupLabel bool, files []string) {
+		t.Helper()
+
+		failed := UnitTest(files, disableGroupLabel)
+		if failed {
+			t.Fatalf("unexpected failed test")
+		}
+	}
+
+	// run multi files
+	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"})
+
+	// disable group label
+	f(true, []string{"./testdata/disable-group-label.yaml"})
 }

app/vmalert/Makefile

@@ -101,8 +101,7 @@ replay-vmalert: vmalert
 		-remoteWrite.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \
 		-external.label=replica=a \
-		-replay.timeFrom=2021-05-11T07:21:43Z \
-		-replay.timeTo=2021-05-29T18:40:43Z
+		-replay.timeFrom=2024-06-01T00:00:00Z
 
 vmalert-linux-amd64:
 	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch

app/vmalert/config/config.go

@@ -45,11 +45,11 @@ type Group struct {
 	// EvalAlignment will make the timestamp of group query requests be aligned with interval
 	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]interface{} `yaml:",inline"`
+	XXX map[string]any `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
 	type group Group
 	if err := unmarshal((*group)(g)); err != nil {
 		return err
@@ -142,11 +142,11 @@ type Rule struct {
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]interface{} `yaml:",inline"`
+	XXX map[string]any `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (r *Rule) UnmarshalYAML(unmarshal func(any) error) error {
 	type rule Rule
 	if err := unmarshal((*rule)(r)); err != nil {
 		return err
@@ -234,7 +234,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 	cLogger.Suppress(true)
 	defer cLogger.Suppress(false)
 
-	files, err := readFromFS(pathPatterns)
+	files, err := ReadFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
@@ -243,7 +243,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns)
+	files, err := ReadFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
@@ -301,7 +301,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.
-		XXX map[string]interface{} `yaml:",inline"`
+		XXX map[string]any `yaml:",inline"`
 	}{}
 	err = yaml.Unmarshal(data, &g)
 	if err != nil {
@@ -310,7 +310,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	return g.Groups, checkOverflow(g.XXX, "config")
 }
 
-func checkOverflow(m map[string]interface{}, ctx string) error {
+func checkOverflow(m map[string]any, ctx string) error {
 	if len(m) > 0 {
 		var keys []string
 		for k := range m {

app/vmalert/config/config_test.go

@@ -23,12 +23,6 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
-		t.Errorf("error parsing files %s", err)
-	}
-}
-
 func TestParseFromURL(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/bad", func(w http.ResponseWriter, _ *http.Request) {
@@ -55,438 +49,353 @@ groups:
 	defer srv.Close()
 
 	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
-		t.Errorf("error parsing URLs %s", err)
+		t.Fatalf("error parsing URLs %s", err)
 	}
 
 	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
-		t.Errorf("expected parsing error: %s", err)
+		t.Fatalf("expected parsing error: %s", err)
 	}
 }
 
-func TestParseBad(t *testing.T) {
-	testCases := []struct {
-		path   []string
-		expErr string
-	}{
-		{
-			[]string{"testdata/rules/rules_interval_bad.rules"},
-			"eval_offset should be smaller than interval",
-		},
-		{
-			[]string{"testdata/rules/rules0-bad.rules"},
-			"unexpected token",
-		},
-		{
-			[]string{"testdata/dir/rules0-bad.rules"},
-			"error parsing annotation",
-		},
-		{
-			[]string{"testdata/dir/rules1-bad.rules"},
-			"duplicate in file",
-		},
-		{
-			[]string{"testdata/dir/rules2-bad.rules"},
-			"function \"unknown\" not defined",
-		},
-		{
-			[]string{"testdata/dir/rules3-bad.rules"},
-			"either `record` or `alert` must be set",
-		},
-		{
-			[]string{"testdata/dir/rules4-bad.rules"},
-			"either `record` or `alert` must be set",
-		},
-		{
-			[]string{"testdata/rules/rules1-bad.rules"},
-			"bad graphite expr",
-		},
-		{
-			[]string{"testdata/dir/rules6-bad.rules"},
-			"missing ':' in header",
-		},
-		{
-			[]string{"http://unreachable-url"},
-			"failed to",
-		},
+func TestParse_Success(t *testing.T) {
+	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
+	if err != nil {
+		t.Fatalf("error parsing files %s", err)
 	}
-	for _, tc := range testCases {
-		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
+}
+
+func TestParse_Failure(t *testing.T) {
+	f := func(paths []string, errStrExpected string) {
+		t.Helper()
+
+		_, err := Parse(paths, notifier.ValidateTemplates, true)
 		if err == nil {
-			t.Errorf("expected to get error")
-			return
+			t.Fatalf("expected to get error")
 		}
-		if !strings.Contains(err.Error(), tc.expErr) {
-			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		if !strings.Contains(err.Error(), errStrExpected) {
+			t.Fatalf("expected err to contain %q; got %q instead", errStrExpected, err)
 		}
 	}
+
+	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
+	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
+	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
+	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
+	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
+	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
+	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
+	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
+	f([]string{"http://unreachable-url"}, "failed to")
 }
 
-func TestRule_Validate(t *testing.T) {
+func TestRuleValidate(t *testing.T) {
 	if err := (&Rule{}).Validate(); err == nil {
-		t.Errorf("expected empty name error")
+		t.Fatalf("expected empty name error")
 	}
 	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
-		t.Errorf("expected empty expr error")
+		t.Fatalf("expected empty expr error")
 	}
 	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
-		t.Errorf("expected valid rule; got %s", err)
+		t.Fatalf("expected valid rule; got %s", err)
 	}
 }
 
-func TestGroup_Validate(t *testing.T) {
-	testCases := []struct {
-		group               *Group
-		rules               []Rule
-		validateAnnotations bool
-		validateExpressions bool
-		expErr              string
-	}{
-		{
-			group:  &Group{},
-			expErr: "group name must be set",
-		},
-		{
-			group: &Group{
-				Name:     "negative interval",
-				Interval: promutils.NewDuration(-1),
+func TestGroupValidate_Failure(t *testing.T) {
+	f := func(group *Group, validateExpressions bool, errStrExpected string) {
+		t.Helper()
+
+		err := group.Validate(nil, validateExpressions)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		errStr := err.Error()
+		if !strings.Contains(errStr, errStrExpected) {
+			t.Fatalf("missing %q in the returned error %q", errStrExpected, errStr)
+		}
+	}
+
+	f(&Group{}, false, "group name must be set")
+
+	f(&Group{
+		Name:     "negative interval",
+		Interval: promutils.NewDuration(-1),
+	}, false, "interval shouldn't be lower than 0")
+
+	f(&Group{
+		Name:       "wrong eval_offset",
+		Interval:   promutils.NewDuration(time.Minute),
+		EvalOffset: promutils.NewDuration(2 * time.Minute),
+	}, false, "eval_offset should be smaller than interval")
+
+	f(&Group{
+		Name:  "wrong limit",
+		Limit: -1,
+	}, false, "invalid limit")
+
+	f(&Group{
+		Name:        "wrong concurrency",
+		Concurrency: -1,
+	}, false, "invalid concurrency")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
 			},
-			expErr: "interval shouldn't be lower than 0",
-		},
-		{
-			group: &Group{
-				Name:       "wrong eval_offset",
-				Interval:   promutils.NewDuration(time.Minute),
-				EvalOffset: promutils.NewDuration(2 * time.Minute),
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
 			},
-			expErr: "eval_offset should be smaller than interval",
 		},
-		{
-			group: &Group{
-				Name:  "wrong limit",
-				Limit: -1,
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Record: "record", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Record: "record", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test graphite prometheus bad expr",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Expr: "sum(up == 0 ) by (host)",
+				For:  promutils.NewDuration(10 * time.Millisecond),
 			},
-			expErr: "invalid limit",
-		},
-		{
-			group: &Group{
-				Name:        "wrong concurrency",
-				Concurrency: -1,
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
 			},
-			expErr: "invalid concurrency",
 		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Record: "record",
-						Expr:   "up | 0",
-					},
+	}, false, "invalid rule")
+
+	f(&Group{
+		Name: "test graphite inherit",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
+				For:  promutils.NewDuration(10 * time.Millisecond),
+			},
+			{
+				Expr: "sum(up == 0 ) by (host)",
+			},
+		},
+	}, false, "either `record` or `alert` must be set")
+
+	// validate expressions
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, true, "invalid expression")
+
+	f(&Group{
+		Name: "test thanos",
+		Type: NewRawType("thanos"),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, true, "unknown datasource type")
+
+	f(&Group{
+		Name: "test graphite",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "some-description",
+			}},
+		},
+	}, true, "bad graphite expr")
+}
+
+func TestGroupValidate_Success(t *testing.T) {
+	f := func(group *Group, validateAnnotations, validateExpressions bool) {
+		t.Helper()
+
+		var validateTplFn ValidateTplFn
+		if validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := group.Validate(validateTplFn, validateExpressions)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	}
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, false, false)
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
+				Labels: map[string]string{
+					"summary": "{{ value|query }}",
 				},
 			},
-			expErr: "",
 		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Record: "record",
-						Expr:   "up | 0",
-					},
-				},
-			},
-			expErr:              "invalid expression",
-			validateExpressions: true,
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-						Labels: map[string]string{
-							"summary": "{{ value|query }}",
-						},
-					},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-						Labels: map[string]string{
-							"summary": `
+	}, false, false)
+
+	// validate annotiations
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
+				Labels: map[string]string{
+					"summary": `
 {{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
   {{ . | first | value | humanize1024 }}B
 {{ end }}`,
-						},
-					},
-				},
-			},
-			validateAnnotations: true,
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-					},
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-					},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Record: "record", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Record: "record", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test thanos",
-				Type: NewRawType("thanos"),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "unknown datasource type",
-		},
-		{
-			group: &Group{
-				Name: "test graphite",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "some-description",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "",
-		},
-		{
-			group: &Group{
-				Name: "test prometheus",
-				Type: NewPrometheusType(),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "",
-		},
-		{
-			group: &Group{
-				Name: "test graphite inherit",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{
-						Expr: "sumSeries(time('foo.bar',10))",
-						For:  promutils.NewDuration(10 * time.Millisecond),
-					},
-					{
-						Expr: "sum(up == 0 ) by (host)",
-					},
 				},
 			},
 		},
-		{
-			group: &Group{
-				Name: "test graphite prometheus bad expr",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{
-						Expr: "sum(up == 0 ) by (host)",
-						For:  promutils.NewDuration(10 * time.Millisecond),
-					},
-					{
-						Expr: "sumSeries(time('foo.bar',10))",
-					},
-				},
-			},
-			expErr: "invalid rule",
-		},
-	}
+	}, true, false)
 
-	for _, tc := range testCases {
-		var validateTplFn ValidateTplFn
-		if tc.validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
-		}
-		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
-		if err == nil {
-			if tc.expErr != "" {
-				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
-			}
-			continue
-		}
-		if !strings.Contains(err.Error(), tc.expErr) {
-			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
-		}
-	}
+	// validate expressions
+	f(&Group{
+		Name: "test prometheus",
+		Type: NewPrometheusType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, false, true)
 }
 
-func TestHashRule(t *testing.T) {
-	testCases := []struct {
-		a, b  Rule
-		equal bool
-	}{
-		{
-			Rule{Record: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1"},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-				"foo": "bar",
-			}},
-			true,
-		},
-		{
-			Rule{Alert: "record", Expr: "up == 1"},
-			Rule{Alert: "record", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 1"},
-			false,
-		},
-		{
-			Rule{Record: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 2"},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-				"foo": "baz",
-			}},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-			}},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			false,
-		},
-	}
-	for i, tc := range testCases {
-		aID, bID := HashRule(tc.a), HashRule(tc.b)
-		if tc.equal != (aID == bID) {
-			t.Fatalf("missmatch for rule %d", i)
+func TestHashRule_NotEqual(t *testing.T) {
+	f := func(a, b Rule) {
+		t.Helper()
+
+		aID, bID := HashRule(a), HashRule(b)
+		if aID == bID {
+			t.Fatalf("rule hashes mustn't be equal; got %d", aID)
 		}
 	}
+
+	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
+
+	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 2"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+		"foo": "baz",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1"})
+}
+
+func TestHashRule_Equal(t *testing.T) {
+	f := func(a, b Rule) {
+		t.Helper()
+
+		aID, bID := HashRule(a), HashRule(b)
+		if aID != bID {
+			t.Fatalf("rule hashes must be equal; got %d and %d", aID, bID)
+		}
+	}
+
+	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1"}, Rule{Alert: "alert", Expr: "up == 1"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+		"foo": "bar",
+	}})
+
+	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Alert: "record", Expr: "up == 1"})
+
+	f(Rule{
+		Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute),
+	}, Rule{Alert: "alert", Expr: "up == 1"})
 }
 
 func TestGroupChecksum(t *testing.T) {

app/vmalert/config/fs.go

@@ -32,14 +32,14 @@ var (
 	fsRegistry   = make(map[string]FS)
 )
 
-// readFromFS parses the given path list and inits FS for each item.
-// Once initialed, readFromFS will try to read and return files from each FS.
-// readFromFS returns an error if at least one FS failed to init.
+// ReadFromFS parses the given path list and inits FS for each item.
+// Once initialed, ReadFromFS will try to read and return files from each FS.
+// ReadFromFS returns an error if at least one FS failed to init.
 // The function can be called multiple times but each unique path
 // will be initialed only once.
 //
 // It is allowed to mix different FS types in path list.
-func readFromFS(paths []string) (map[string][]byte, error) {
+func ReadFromFS(paths []string) (map[string][]byte, error) {
 	var err error
 	result := make(map[string][]byte)
 	for _, path := range paths {

app/vmalert/config/log/logger.go

@@ -29,7 +29,7 @@ func (l *Logger) isDisabled() bool {
 }
 
 // Errorf logs error message.
-func (l *Logger) Errorf(format string, args ...interface{}) {
+func (l *Logger) Errorf(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -37,7 +37,7 @@ func (l *Logger) Errorf(format string, args ...interface{}) {
 }
 
 // Warnf logs warning message.
-func (l *Logger) Warnf(format string, args ...interface{}) {
+func (l *Logger) Warnf(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -45,7 +45,7 @@ func (l *Logger) Warnf(format string, args ...interface{}) {
 }
 
 // Infof logs info message.
-func (l *Logger) Infof(format string, args ...interface{}) {
+func (l *Logger) Infof(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -54,6 +54,6 @@ func (l *Logger) Infof(format string, args ...interface{}) {
 
 // Panicf logs panic message and panics.
 // Panicf can't be suppressed
-func (l *Logger) Panicf(format string, args ...interface{}) {
+func (l *Logger) Panicf(format string, args ...any) {
 	logger.Panicf(format, args...)
 }

app/vmalert/config/log/logger_test.go

@@ -18,14 +18,14 @@ func TestOutput(t *testing.T) {
 
 	mustMatch := func(exp string) {
 		t.Helper()
+
 		if exp == "" {
 			if testOutput.String() != "" {
-				t.Errorf("expected output to be empty; got %q", testOutput.String())
-				return
+				t.Fatalf("expected output to be empty; got %q", testOutput.String())
 			}
 		}
 		if !strings.Contains(testOutput.String(), exp) {
-			t.Errorf("output %q should contain %q", testOutput.String(), exp)
+			t.Fatalf("output %q should contain %q", testOutput.String(), exp)
 		}
 		fmt.Println(testOutput.String())
 		testOutput.Reset()

app/vmalert/config/types.go

@@ -69,7 +69,7 @@ func (t *Type) ValidateExpr(expr string) error {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err
@@ -87,7 +87,7 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 // MarshalYAML implements the yaml.Unmarshaler interface.
-func (t Type) MarshalYAML() (interface{}, error) {
+func (t Type) MarshalYAML() (any, error) {
 	return t.Name, nil
 }
 
@@ -98,7 +98,7 @@ type Header struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (h *Header) UnmarshalYAML(unmarshal func(any) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err

app/vmalert/datasource/init.go

@@ -12,11 +12,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
-	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
-		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend and -datasource.showURL")
+	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
@@ -56,9 +58,10 @@ var (
 		`Whether to align "time" parameter with evaluation interval. `+
 		"Alignment supposed to produce deterministic results despite number of vmalert replicas or time they were started. "+
 		"See more details at https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
-	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
-	disableKeepAlive   = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
-		`If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.`)
+	maxIdleConnections    = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
+	idleConnectionTimeout = flag.Duration("datasource.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
+	disableKeepAlive      = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
+		`If true, disables HTTP keep-alive and will only use the connection to the server for a single HTTP request.`)
 	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
@@ -98,11 +101,13 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
+	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
+	tr.IdleConnTimeout = *idleConnectionTimeout
 
 	if extraParams == nil {
 		extraParams = url.Values{}

app/vmalert/datasource/vm.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -140,7 +141,7 @@ func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Resu
 	}
 	resp, err := s.do(req)
 	if err != nil {
-		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
 			// Return unexpected error to the caller.
 			return Result{}, nil, err
 		}
@@ -185,7 +186,7 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	}
 	resp, err := s.do(req)
 	if err != nil {
-		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
 			// Return unexpected error to the caller.
 			return res, err
 		}

app/vmalert/datasource/vm_prom_api.go

@@ -7,6 +7,10 @@ import (
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+
+	"github.com/valyala/fastjson"
 )
 
 var (
@@ -31,33 +35,91 @@ type promResponse struct {
 	} `json:"stats,omitempty"`
 }
 
+// see https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 type promInstant struct {
-	Result []struct {
-		Labels map[string]string `json:"metric"`
-		TV     [2]interface{}    `json:"value"`
-	} `json:"result"`
+	// ms is populated after Unmarshal call
+	ms []Metric
 }
 
-func (r promInstant) metrics() ([]Metric, error) {
-	result := make([]Metric, len(r.Result))
-	for i, res := range r.Result {
-		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
-		if err != nil {
-			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
-		}
-		var m Metric
-		m.SetLabels(res.Labels)
-		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
-		m.Values = append(m.Values, f)
-		result[i] = m
+// metrics returned parsed Metric slice
+// Must be called only after Unmarshal
+func (pi *promInstant) metrics() ([]Metric, error) {
+	return pi.ms, nil
+}
+
+var jsonParserPool fastjson.ParserPool
+
+// Unmarshal unmarshals the given byte slice into promInstant
+// It is using fastjson to reduce number of allocations compared to
+// standard json.Unmarshal function.
+// Response example:
+//
+//	[{"metric":{"__name__":"up","job":"prometheus"},value": [ 1435781451.781,"1"]},
+//	{"metric":{"__name__":"up","job":"node"},value": [ 1435781451.781,"0"]}]
+func (pi *promInstant) Unmarshal(b []byte) error {
+	p := jsonParserPool.Get()
+	defer jsonParserPool.Put(p)
+
+	v, err := p.ParseBytes(b)
+	if err != nil {
+		return err
 	}
-	return result, nil
+
+	rows, err := v.Array()
+	if err != nil {
+		return fmt.Errorf("cannot find the top-level array of result objects: %w", err)
+	}
+	pi.ms = make([]Metric, len(rows))
+	for i, row := range rows {
+		metric := row.Get("metric")
+		if metric == nil {
+			return fmt.Errorf("can't find `metric` object in %q", row)
+		}
+		labels := metric.GetObject()
+
+		r := &pi.ms[i]
+		r.Labels = make([]Label, 0, labels.Len())
+		labels.Visit(func(key []byte, v *fastjson.Value) {
+			lv, errLocal := v.StringBytes()
+			if errLocal != nil {
+				err = fmt.Errorf("error when parsing label value %q: %s", v, errLocal)
+				return
+			}
+			r.Labels = append(r.Labels, Label{
+				Name:  string(key),
+				Value: string(lv),
+			})
+		})
+		if err != nil {
+			return fmt.Errorf("error when parsing `metric` object in %q: %w", row, err)
+		}
+
+		value := row.Get("value")
+		if value == nil {
+			return fmt.Errorf("can't find `value` object in %q", row)
+		}
+		sample := value.GetArray()
+		if len(sample) != 2 {
+			return fmt.Errorf("object `value` in %q should contain 2 values, but contains %d instead", row, len(sample))
+		}
+		r.Timestamps = []int64{sample[0].GetInt64()}
+		val, err := sample[1].StringBytes()
+		if err != nil {
+			return fmt.Errorf("error when parsing `value` object %q: %s", sample[1], err)
+		}
+		f, err := strconv.ParseFloat(bytesutil.ToUnsafeString(val), 64)
+		if err != nil {
+			return fmt.Errorf("error when parsing float64 from %s in %q: %w", sample[1], row, err)
+		}
+		r.Values = []float64{f}
+	}
+	return nil
 }
 
 type promRange struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TVs    [][2]interface{}  `json:"values"`
+		TVs    [][2]any          `json:"values"`
 	} `json:"result"`
 }
 
@@ -85,7 +147,7 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
-type promScalar [2]interface{}
+type promScalar [2]any
 
 func (r promScalar) metrics() ([]Metric, error) {
 	var m Metric
@@ -118,7 +180,7 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result
 	switch r.Data.ResultType {
 	case rtVector:
 		var pi promInstant
-		if err := json.Unmarshal(r.Data.Result, &pi.Result); err != nil {
+		if err := pi.Unmarshal(r.Data.Result); err != nil {
 			return res, fmt.Errorf("unmarshal err %w; \n %#v", err, string(r.Data.Result))
 		}
 		parseFn = pi.metrics

app/vmalert/datasource/vm_prom_api_test.go

@@ -1,20 +1,73 @@
 package datasource
 
 import (
-	"encoding/json"
+	"reflect"
 	"testing"
 )
 
-func BenchmarkMetrics(b *testing.B) {
-	payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`)
-
-	var pi promInstant
-	if err := json.Unmarshal(payload, &pi.Result); err != nil {
-		b.Fatalf(err.Error())
-	}
-	b.Run("Instant", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			_, _ = pi.metrics()
+func TestPromInstant_UnmarshalPositive(t *testing.T) {
+	f := func(data string, exp []Metric) {
+		t.Helper()
+		var pi promInstant
+		err := pi.Unmarshal([]byte(data))
+		if err != nil {
+			t.Fatalf("unexpected unmarshal err %v; \n %v", err, string(data))
 		}
+		got, _ := pi.metrics()
+		if !reflect.DeepEqual(got, exp) {
+			t.Fatalf("expected to get:\n%v\ngot instead:\n%v", exp, got)
+		}
+	}
+
+	f(`[{"metric":{"__name__":"up"},"value":[1583780000,"42"]}]`, []Metric{
+		{
+			Labels:     []Label{{Name: "__name__", Value: "up"}},
+			Timestamps: []int64{1583780000},
+			Values:     []float64{42},
+		},
+	})
+	f(`[
+{"metric":{"__name__":"up"},"value":[1583780000,"42"]},
+{"metric":{"__name__":"foo"},"value":[1583780001,"7"]},
+{"metric":{"__name__":"baz", "instance":"bar"},"value":[1583780002,"8"]}]`, []Metric{
+		{
+			Labels:     []Label{{Name: "__name__", Value: "up"}},
+			Timestamps: []int64{1583780000},
+			Values:     []float64{42},
+		},
+		{
+			Labels:     []Label{{Name: "__name__", Value: "foo"}},
+			Timestamps: []int64{1583780001},
+			Values:     []float64{7},
+		},
+		{
+			Labels:     []Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
+			Timestamps: []int64{1583780002},
+			Values:     []float64{8},
+		},
 	})
 }
+
+func TestPromInstant_UnmarshalNegative(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+		var pi promInstant
+		err := pi.Unmarshal([]byte(data))
+		if err == nil {
+			t.Fatalf("expected to get an error; got nil instead")
+		}
+	}
+	f(``)
+	f(`foo`)
+	f(`[{"metric":{"__name__":"up"},"value":[1583780000,"42"]},`)
+	f(`[{"metric":{"__name__"},"value":[1583780000,"42"]},`)
+	// no `metric` object
+	f(`[{"value":[1583780000,"42"]}]`)
+	// no `value` object
+	f(`[{"metric":{"__name__":"up"}}]`)
+	// less than 2 values in `value` object
+	f(`[{"metric":{"__name__":"up"},"value":["42"]}]`)
+	f(`[{"metric":{"__name__":"up"},"value":[1583780000]}]`)
+	// non-numeric sample value
+	f(`[{"metric":{"__name__":"up"},"value":[1583780000,"foo"]}]`)
+}

app/vmalert/datasource/vm_prom_api_timing_test.go

@@ -0,0 +1,43 @@
+package datasource
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"os"
+	"testing"
+)
+
+func BenchmarkMetrics(b *testing.B) {
+	payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`)
+
+	var pi promInstant
+	if err := pi.Unmarshal(payload); err != nil {
+		b.Fatalf(err.Error())
+	}
+	b.Run("Instant", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, _ = pi.metrics()
+		}
+	})
+}
+
+func BenchmarkParsePrometheusResponse(b *testing.B) {
+	req, _ := http.NewRequest("GET", "", nil)
+	resp := &http.Response{StatusCode: http.StatusOK}
+	data, err := os.ReadFile("testdata/instant_response.json")
+	if err != nil {
+		b.Fatalf("error while reading file: %s", err)
+	}
+	resp.Body = io.NopCloser(bytes.NewReader(data))
+
+	b.Run("Instant", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, err := parsePrometheusResponse(req, resp)
+			if err != nil {
+				b.Fatalf("unexpected parse err: %s", err)
+			}
+			resp.Body = io.NopCloser(bytes.NewReader(data))
+		}
+	})
+}

app/vmalert/datasource/vm_test.go

@@ -31,26 +31,26 @@ var (
 func TestVMInstantQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Errorf("expected POST method got %s", r.Method)
+			t.Fatalf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		timeParam := r.URL.Query().Get("time")
 		if timeParam == "" {
-			t.Errorf("expected 'time' in query param, got nil instead")
+			t.Fatalf("expected 'time' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, timeParam); err != nil {
-			t.Errorf("failed to parse 'time' query param %q: %s", timeParam, err)
+			t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
 		case 0:
@@ -197,13 +197,13 @@ func TestVMInstantQuery(t *testing.T) {
 func TestVMInstantQueryWithRetry(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.URL.Query().Get("query") != query {
-			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		switch c {
 		case 0:
@@ -289,37 +289,37 @@ func metricsEqual(t *testing.T, gotM, expectedM []Metric) {
 func TestVMRangeQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query_range", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Errorf("expected POST method got %s", r.Method)
+			t.Fatalf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		startTS := r.URL.Query().Get("start")
 		if startTS == "" {
-			t.Errorf("expected 'start' in query param, got nil instead")
+			t.Fatalf("expected 'start' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, startTS); err != nil {
-			t.Errorf("failed to parse 'start' query param: %s", err)
+			t.Fatalf("failed to parse 'start' query param: %s", err)
 		}
 		endTS := r.URL.Query().Get("end")
 		if endTS == "" {
-			t.Errorf("expected 'end' in query param, got nil instead")
+			t.Fatalf("expected 'end' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, endTS); err != nil {
-			t.Errorf("failed to parse 'end' query param: %s", err)
+			t.Fatalf("failed to parse 'end' query param: %s", err)
 		}
 		step := r.URL.Query().Get("step")
 		if step != "15s" {
-			t.Errorf("expected 'step' query param to be 15s; got %q instead", step)
+			t.Fatalf("expected 'step' query param to be 15s; got %q instead", step)
 		}
 		switch c {
 		case 0:
@@ -370,368 +370,299 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
-	authCfg, err := baCfg.NewConfig(".")
-	if err != nil {
-		t.Fatalf("unexpected: %s", err)
-	}
 	query := "up"
 	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
+
+	f := func(isQueryRange bool, vm *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
+		t.Helper()
+
+		req, err := vm.newRequest(ctx)
+		if err != nil {
+			t.Fatalf("error in newRequest: %s", err)
+		}
+
+		switch vm.dataSourceType {
+		case "", datasourcePrometheus:
+			if isQueryRange {
+				vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
+			} else {
+				vm.setPrometheusInstantReqParams(req, query, timestamp)
+			}
+		case datasourceGraphite:
+			vm.setGraphiteReqParams(req, query)
+		}
+
+		checkFn(t, req)
+	}
+
+	authCfg, err := baCfg.NewConfig(".")
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
 	storage := VMStorage{
 		extraParams: url.Values{"round_digits": {"10"}},
 	}
-	testCases := []struct {
-		name       string
-		queryRange bool
-		vm         *VMStorage
-		checkFn    func(t *testing.T, r *http.Request)
-	}{
-		{
-			"prometheus path",
-			false,
-			&VMStorage{
-				dataSourceType: datasourcePrometheus,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/api/v1/query", r.URL.Path)
-			},
-		},
-		{
-			"prometheus prefix",
-			false,
-			&VMStorage{
-				dataSourceType:   datasourcePrometheus,
-				appendTypePrefix: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
-			},
-		},
-		{
-			"prometheus range path",
-			true,
-			&VMStorage{
-				dataSourceType: datasourcePrometheus,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
-			},
-		},
-		{
-			"prometheus range prefix",
-			true,
-			&VMStorage{
-				dataSourceType:   datasourcePrometheus,
-				appendTypePrefix: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
-			},
-		},
-		{
-			"graphite path",
-			false,
-			&VMStorage{
-				dataSourceType: datasourceGraphite,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, graphitePath, r.URL.Path)
-			},
-		},
-		{
-			"graphite prefix",
-			false,
-			&VMStorage{
-				dataSourceType:   datasourceGraphite,
-				appendTypePrefix: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
-			},
-		},
-		{
-			"default params",
-			false,
-			&VMStorage{},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"default range params",
-			true,
-			&VMStorage{},
-			func(t *testing.T, r *http.Request) {
-				ts := timestamp.Format(time.RFC3339)
-				exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"basic auth",
-			false,
-			&VMStorage{authCfg: authCfg},
-			func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "foo", u)
-				checkEqualString(t, "bar", p)
-			},
-		},
-		{
-			"basic auth range",
-			true,
-			&VMStorage{authCfg: authCfg},
-			func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "foo", u)
-				checkEqualString(t, "bar", p)
-			},
-		},
-		{
-			"evaluation interval",
-			false,
-			&VMStorage{
-				evaluationInterval: 15 * time.Second,
-			},
-			func(t *testing.T, r *http.Request) {
-				evalInterval := 15 * time.Second
-				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"step override",
-			false,
-			&VMStorage{
-				queryStep: time.Minute,
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{
-					"query": {query},
-					"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
-					"time":  {timestamp.Format(time.RFC3339)},
-				}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"step to seconds",
-			false,
-			&VMStorage{
-				evaluationInterval: 3 * time.Hour,
-			},
-			func(t *testing.T, r *http.Request) {
-				evalInterval := 3 * time.Hour
-				exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"prometheus extra params",
-			false,
-			&VMStorage{
-				extraParams: url.Values{"round_digits": {"10"}},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"prometheus extra params range",
-			true,
-			&VMStorage{
-				extraParams: url.Values{
-					"nocache":      {"1"},
-					"max_lookback": {"1h"},
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{
-					"query":        {query},
-					"end":          {timestamp.Format(time.RFC3339)},
-					"start":        {timestamp.Format(time.RFC3339)},
-					"nocache":      {"1"},
-					"max_lookback": {"1h"},
-				}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"custom params overrides the original params",
-			false,
-			storage.Clone().ApplyParams(QuerierParams{
-				QueryParams: url.Values{"round_digits": {"2"}},
-			}),
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"allow duplicates in query params",
-			false,
-			storage.Clone().ApplyParams(QuerierParams{
-				QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
-			}),
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"graphite extra params",
-			false,
-			&VMStorage{
-				dataSourceType: datasourceGraphite,
-				extraParams: url.Values{
-					"nocache":      {"1"},
-					"max_lookback": {"1h"},
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
-				checkEqualString(t, exp, r.URL.RawQuery)
-			},
-		},
-		{
-			"graphite extra params allows to override from",
-			false,
-			&VMStorage{
-				dataSourceType: datasourceGraphite,
-				extraParams: url.Values{
-					"from": {"-10m"},
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
-				checkEqualString(t, exp, r.URL.RawQuery)
-			},
-		},
-	}
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			req, err := tc.vm.newRequest(ctx)
-			if err != nil {
-				t.Fatal(err)
-			}
-			switch tc.vm.dataSourceType {
-			case "", datasourcePrometheus:
-				if tc.queryRange {
-					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
-				} else {
-					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
-				}
-			case datasourceGraphite:
-				tc.vm.setGraphiteReqParams(req, query)
-			}
-			tc.checkFn(t, req)
-		})
-	}
+	// prometheus path
+	f(false, &VMStorage{
+		dataSourceType: datasourcePrometheus,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/api/v1/query", r.URL.Path)
+	})
+
+	// prometheus prefix
+	f(false, &VMStorage{
+		dataSourceType:   datasourcePrometheus,
+		appendTypePrefix: true,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
+	})
+
+	// prometheus range path
+	f(true, &VMStorage{
+		dataSourceType: datasourcePrometheus,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/api/v1/query_range", r.URL.Path)
+	})
+
+	// prometheus range prefix
+	f(true, &VMStorage{
+		dataSourceType:   datasourcePrometheus,
+		appendTypePrefix: true,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
+	})
+
+	// graphite path
+	f(false, &VMStorage{
+		dataSourceType: datasourceGraphite,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, graphitePath, r.URL.Path)
+	})
+
+	// graphite prefix
+	f(false, &VMStorage{
+		dataSourceType:   datasourceGraphite,
+		appendTypePrefix: true,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
+	})
+
+	// default params
+	f(false, &VMStorage{}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// default range params
+	f(true, &VMStorage{}, func(t *testing.T, r *http.Request) {
+		ts := timestamp.Format(time.RFC3339)
+		exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// basic auth
+	f(false, &VMStorage{
+		authCfg: authCfg,
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "foo", u)
+		checkEqualString(t, "bar", p)
+	})
+
+	// basic auth range
+	f(true, &VMStorage{
+		authCfg: authCfg,
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "foo", u)
+		checkEqualString(t, "bar", p)
+	})
+
+	// evaluation interval
+	f(false, &VMStorage{
+		evaluationInterval: 15 * time.Second,
+	}, func(t *testing.T, r *http.Request) {
+		evalInterval := 15 * time.Second
+		exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// step override
+	f(false, &VMStorage{
+		queryStep: time.Minute,
+	}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{
+			"query": {query},
+			"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
+			"time":  {timestamp.Format(time.RFC3339)},
+		}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	//  step to seconds
+	f(false, &VMStorage{
+		evaluationInterval: 3 * time.Hour,
+	}, func(t *testing.T, r *http.Request) {
+		evalInterval := 3 * time.Hour
+		exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// prometheus extra params
+	f(false, &VMStorage{
+		extraParams: url.Values{"round_digits": {"10"}},
+	}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// prometheus extra params range
+	f(true, &VMStorage{
+		extraParams: url.Values{
+			"nocache":      {"1"},
+			"max_lookback": {"1h"},
+		},
+	}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{
+			"query":        {query},
+			"end":          {timestamp.Format(time.RFC3339)},
+			"start":        {timestamp.Format(time.RFC3339)},
+			"nocache":      {"1"},
+			"max_lookback": {"1h"},
+		}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// custom params overrides the original params
+	f(false, storage.Clone().ApplyParams(QuerierParams{
+		QueryParams: url.Values{"round_digits": {"2"}},
+	}), func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// allow duplicates in query params
+	f(false, storage.Clone().ApplyParams(QuerierParams{
+		QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
+	}), func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// graphite extra params
+	f(false, &VMStorage{
+		dataSourceType: datasourceGraphite,
+		extraParams: url.Values{
+			"nocache":      {"1"},
+			"max_lookback": {"1h"},
+		},
+	}, func(t *testing.T, r *http.Request) {
+		exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
+		checkEqualString(t, exp, r.URL.RawQuery)
+	})
+
+	// graphite extra params allows to override from
+	f(false, &VMStorage{
+		dataSourceType: datasourceGraphite,
+		extraParams: url.Values{
+			"from": {"-10m"},
+		},
+	}, func(t *testing.T, r *http.Request) {
+		exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
+		checkEqualString(t, exp, r.URL.RawQuery)
+	})
 }
 
 func TestHeaders(t *testing.T) {
-	testCases := []struct {
-		name    string
-		vmFn    func() *VMStorage
-		checkFn func(t *testing.T, r *http.Request)
-	}{
-		{
-			name: "basic auth",
-			vmFn: func() *VMStorage {
-				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-				if err != nil {
-					t.Errorf("Error get auth config: %s", err)
-				}
-				return &VMStorage{authCfg: cfg}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "foo", u)
-				checkEqualString(t, "bar", p)
-			},
-		},
-		{
-			name: "bearer auth",
-			vmFn: func() *VMStorage {
-				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
-				if err != nil {
-					t.Errorf("Error get auth config: %s", err)
-				}
-				return &VMStorage{authCfg: cfg}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				reqToken := r.Header.Get("Authorization")
-				splitToken := strings.Split(reqToken, "Bearer ")
-				if len(splitToken) != 2 {
-					t.Errorf("expected two items got %d", len(splitToken))
-				}
-				token := splitToken[1]
-				checkEqualString(t, "foo", token)
-			},
-		},
-		{
-			name: "custom extraHeaders",
-			vmFn: func() *VMStorage {
-				return &VMStorage{extraHeaders: []keyValue{
-					{key: "Foo", value: "bar"},
-					{key: "Baz", value: "qux"},
-				}}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				h1 := r.Header.Get("Foo")
-				checkEqualString(t, "bar", h1)
-				h2 := r.Header.Get("Baz")
-				checkEqualString(t, "qux", h2)
-			},
-		},
-		{
-			name: "custom header overrides basic auth",
-			vmFn: func() *VMStorage {
-				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-				if err != nil {
-					t.Errorf("Error get auth config: %s", err)
-				}
-				return &VMStorage{
-					authCfg: cfg,
-					extraHeaders: []keyValue{
-						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
-					},
-				}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "Aladdin", u)
-				checkEqualString(t, "open sesame", p)
-			},
-		},
-	}
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			vm := tt.vmFn()
-			req, err := vm.newQueryRequest(ctx, "foo", time.Now())
-			if err != nil {
-				t.Fatal(err)
-			}
-			tt.checkFn(t, req)
-		})
+	f := func(vmFn func() *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
+		t.Helper()
+
+		vm := vmFn()
+		req, err := vm.newQueryRequest(ctx, "foo", time.Now())
+		if err != nil {
+			t.Fatalf("error in newQueryRequest: %s", err)
+		}
+		checkFn(t, req)
 	}
+
+	// basic auth
+	f(func() *VMStorage {
+		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+		if err != nil {
+			t.Fatalf("Error get auth config: %s", err)
+		}
+		return &VMStorage{authCfg: cfg}
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "foo", u)
+		checkEqualString(t, "bar", p)
+	})
+
+	// bearer auth
+	f(func() *VMStorage {
+		cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+		if err != nil {
+			t.Fatalf("Error get auth config: %s", err)
+		}
+		return &VMStorage{authCfg: cfg}
+	}, func(t *testing.T, r *http.Request) {
+		reqToken := r.Header.Get("Authorization")
+		splitToken := strings.Split(reqToken, "Bearer ")
+		if len(splitToken) != 2 {
+			t.Fatalf("expected two items got %d", len(splitToken))
+		}
+		token := splitToken[1]
+		checkEqualString(t, "foo", token)
+	})
+
+	// custom extraHeaders
+	f(func() *VMStorage {
+		return &VMStorage{extraHeaders: []keyValue{
+			{key: "Foo", value: "bar"},
+			{key: "Baz", value: "qux"},
+		}}
+	}, func(t *testing.T, r *http.Request) {
+		h1 := r.Header.Get("Foo")
+		checkEqualString(t, "bar", h1)
+		h2 := r.Header.Get("Baz")
+		checkEqualString(t, "qux", h2)
+	})
+
+	// custom header overrides basic auth
+	f(func() *VMStorage {
+		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+		if err != nil {
+			t.Fatalf("Error get auth config: %s", err)
+		}
+		return &VMStorage{
+			authCfg: cfg,
+			extraHeaders: []keyValue{
+				{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+			},
+		}
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "Aladdin", u)
+		checkEqualString(t, "open sesame", p)
+	})
 }
 
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
+
 	if got != exp {
-		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
+		t.Fatalf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }
 
 func expectError(t *testing.T, err error, exp string) {
 	t.Helper()
+
 	if err == nil {
-		t.Errorf("expected non-nil error")
+		t.Fatalf("expected non-nil error")
 	}
 	if !strings.Contains(err.Error(), exp) {
-		t.Errorf("expected error %q to contain %q", err, exp)
+		t.Fatalf("expected error %q to contain %q", err, exp)
 	}
 }

app/vmalert/main.go

@@ -72,7 +72,7 @@ absolute path to all .tpl files in root.
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
 		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
 		`Supports templating - see https://docs.victoriametrics.com/vmalert/#templating . `+
-		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
+		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
 		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
@@ -132,7 +132,7 @@ func main() {
 		validateTplFn = notifier.ValidateTemplates
 	}
 
-	if *replayFrom != "" || *replayTo != "" {
+	if *replayFrom != "" {
 		rw, err := remotewrite.Init(context.Background())
 		if err != nil {
 			logger.Fatalf("failed to init remoteWrite: %s", err)
@@ -153,6 +153,7 @@ func main() {
 		if err := replay(groupsCfg, q, rw); err != nil {
 			logger.Fatalf("replay failed: %s", err)
 		}
+		logger.Infof("replay succeed!")
 		return
 	}
 

app/vmalert/main_test.go

@@ -25,26 +25,26 @@ func TestGetExternalURL(t *testing.T) {
 	invalidURL := "victoriametrics.com/path"
 	_, err := getExternalURL(invalidURL)
 	if err == nil {
-		t.Errorf("expected error, got nil")
+		t.Fatalf("expected error, got nil")
 	}
 
 	expURL := "https://victoriametrics.com/path"
 	u, err := getExternalURL(expURL)
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url: want %q, got %s", expURL, u.String())
+		t.Fatalf("unexpected url: want %q, got %s", expURL, u.String())
 	}
 
 	h, _ := os.Hostname()
 	expURL = fmt.Sprintf("http://%s:8880", h)
 	u, err = getExternalURL("")
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url: want %s, got %s", expURL, u.String())
+		t.Fatalf("unexpected url: want %s, got %s", expURL, u.String())
 	}
 }
 
@@ -53,22 +53,22 @@ func TestGetAlertURLGenerator(t *testing.T) {
 	u, _ := url.Parse("https://victoriametrics.com/path")
 	fn, err := getAlertURLGenerator(u, "", false)
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
 	if exp != fn(testAlert) {
-		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
 	if err == nil {
-		t.Errorf("expected template validation error got nil")
+		t.Fatalf("expected template validation error got nil")
 	}
 	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
-		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
 

app/vmalert/manager_test.go

@@ -82,9 +82,8 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	wg.Wait()
 }
 
-// TestManagerUpdate tests sequential configuration
-// updates.
-func TestManagerUpdate(t *testing.T) {
+// TestManagerUpdate tests sequential configuration updates.
+func TestManagerUpdate_Success(t *testing.T) {
 	const defaultEvalInterval = time.Second * 30
 	currentEvalInterval := *evaluationInterval
 	*evaluationInterval = defaultEvalInterval
@@ -120,145 +119,127 @@ func TestManagerUpdate(t *testing.T) {
 		}
 	)
 
-	testCases := []struct {
-		name       string
-		initPath   string
-		updatePath string
-		want       []*rule.Group
-	}{
-		{
-			name:       "update good rules",
-			initPath:   "config/testdata/rules/rules0-good.rules",
-			updatePath: "config/testdata/dir/rules1-good.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/dir/rules1-good.rules",
-					Name:     "duplicatedGroupDiffFiles",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules: []rule.Rule{
-						&rule.AlertingRule{
-							Name:   "VMRows",
-							Expr:   "vm_rows > 0",
-							For:    5 * time.Minute,
-							Labels: map[string]string{"dc": "gcp", "label": "bar"},
-							Annotations: map[string]string{
-								"summary":     "{{ $value }}",
-								"description": "{{$labels}}",
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			name:       "update good rules from 1 to 2 groups",
-			initPath:   "config/testdata/dir/rules/rules1-good.rules",
-			updatePath: "config/testdata/rules/rules0-good.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Name:     "groupGorSingleAlert",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules:    []rule.Rule{VMRows},
-				},
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     config.NewPrometheusType(),
-					Name:     "TestGroup",
-					Rules: []rule.Rule{
-						Conns,
-						ExampleAlertAlwaysFiring,
-					},
-				},
-			},
-		},
-		{
-			name:       "update with one bad rule file",
-			initPath:   "config/testdata/rules/rules0-good.rules",
-			updatePath: "config/testdata/dir/rules2-bad.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Name:     "groupGorSingleAlert",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules:    []rule.Rule{VMRows},
-				},
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Interval: defaultEvalInterval,
-					Name:     "TestGroup",
-					Type:     config.NewPrometheusType(),
-					Rules: []rule.Rule{
-						Conns,
-						ExampleAlertAlwaysFiring,
-					},
-				},
-			},
-		},
-		{
-			name:       "update empty dir rules from 0 to 2 groups",
-			initPath:   "config/testdata/empty/*",
-			updatePath: "config/testdata/rules/rules0-good.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Name:     "groupGorSingleAlert",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules:    []rule.Rule{VMRows},
-				},
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     config.NewPrometheusType(),
-					Name:     "TestGroup",
-					Rules: []rule.Rule{
-						Conns,
-						ExampleAlertAlwaysFiring,
-					},
-				},
-			},
-		},
+	f := func(initPath, updatePath string, groupsExpected []*rule.Group) {
+		t.Helper()
+
+		ctx, cancel := context.WithCancel(context.TODO())
+		m := &manager{
+			groups:         make(map[uint64]*rule.Group),
+			querierBuilder: &datasource.FakeQuerier{},
+			notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
+		}
+
+		cfgInit := loadCfg(t, []string{initPath}, true, true)
+		if err := m.update(ctx, cfgInit, false); err != nil {
+			t.Fatalf("failed to complete initial rules update: %s", err)
+		}
+
+		cfgUpdate, err := config.Parse([]string{updatePath}, notifier.ValidateTemplates, true)
+		if err == nil { // update can fail and that's expected
+			_ = m.update(ctx, cfgUpdate, false)
+		}
+		if len(groupsExpected) != len(m.groups) {
+			t.Fatalf("unexpected number of groups; got %d; want %d", len(m.groups), len(groupsExpected))
+		}
+
+		for _, wantG := range groupsExpected {
+			gotG, ok := m.groups[wantG.ID()]
+			if !ok {
+				t.Fatalf("expected to have group %q", wantG.Name)
+			}
+			compareGroups(t, wantG, gotG)
+		}
+
+		cancel()
+		m.close()
 	}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{
-				groups:         make(map[uint64]*rule.Group),
-				querierBuilder: &datasource.FakeQuerier{},
-				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
-			}
 
-			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
-			if err := m.update(ctx, cfgInit, false); err != nil {
-				t.Fatalf("failed to complete initial rules update: %s", err)
-			}
+	// update good rules
+	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules1-good.rules", []*rule.Group{
+		{
+			File:     "config/testdata/dir/rules1-good.rules",
+			Name:     "duplicatedGroupDiffFiles",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules: []rule.Rule{
+				&rule.AlertingRule{
+					Name:   "VMRows",
+					Expr:   "vm_rows > 0",
+					For:    5 * time.Minute,
+					Labels: map[string]string{"dc": "gcp", "label": "bar"},
+					Annotations: map[string]string{
+						"summary":     "{{ $value }}",
+						"description": "{{$labels}}",
+					},
+				},
+			},
+		},
+	})
 
-			cfgUpdate, err := config.Parse([]string{tc.updatePath}, notifier.ValidateTemplates, true)
-			if err == nil { // update can fail and that's expected
-				_ = m.update(ctx, cfgUpdate, false)
-			}
-			if len(tc.want) != len(m.groups) {
-				t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
-			}
+	// update good rules from 1 to 2 groups
+	f("config/testdata/dir/rules/rules1-good.rules", "config/testdata/rules/rules0-good.rules", []*rule.Group{
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Name:     "groupGorSingleAlert",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules:    []rule.Rule{VMRows},
+		},
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Interval: defaultEvalInterval,
+			Type:     config.NewPrometheusType(),
+			Name:     "TestGroup",
+			Rules: []rule.Rule{
+				Conns,
+				ExampleAlertAlwaysFiring,
+			},
+		},
+	})
 
-			for _, wantG := range tc.want {
-				gotG, ok := m.groups[wantG.ID()]
-				if !ok {
-					t.Fatalf("expected to have group %q", wantG.Name)
-				}
-				compareGroups(t, wantG, gotG)
-			}
+	// update with one bad rule file
+	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules2-bad.rules", []*rule.Group{
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Name:     "groupGorSingleAlert",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules:    []rule.Rule{VMRows},
+		},
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Interval: defaultEvalInterval,
+			Name:     "TestGroup",
+			Type:     config.NewPrometheusType(),
+			Rules: []rule.Rule{
+				Conns,
+				ExampleAlertAlwaysFiring,
+			},
+		},
+	})
 
-			cancel()
-			m.close()
-		})
-	}
+	// update empty dir rules from 0 to 2 groups
+	f("config/testdata/empty/*", "config/testdata/rules/rules0-good.rules", []*rule.Group{
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Name:     "groupGorSingleAlert",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules:    []rule.Rule{VMRows},
+		},
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Interval: defaultEvalInterval,
+			Type:     config.NewPrometheusType(),
+			Name:     "TestGroup",
+			Rules: []rule.Rule{
+				Conns,
+				ExampleAlertAlwaysFiring,
+			},
+		},
+	})
 }
+
 func compareGroups(t *testing.T, a, b *rule.Group) {
 	t.Helper()
 	if a.Name != b.Name {
@@ -285,82 +266,59 @@ func compareGroups(t *testing.T, a, b *rule.Group) {
 	}
 }
 
-func TestManagerUpdateNegative(t *testing.T) {
-	testCases := []struct {
-		notifiers []notifier.Notifier
-		rw        remotewrite.RWClient
-		cfg       config.Group
-		expErr    string
-	}{
-		{
-			nil,
-			nil,
-			config.Group{
-				Name: "Recording rule only",
-				Rules: []config.Rule{
-					{Record: "record", Expr: "max(up)"},
-				},
-			},
-			"contains recording rules",
-		},
-		{
-			nil,
-			nil,
-			config.Group{
-				Name: "Alerting rule only",
-				Rules: []config.Rule{
-					{Alert: "alert", Expr: "up > 0"},
-				},
-			},
-			"contains alerting rules",
-		},
-		{
-			[]notifier.Notifier{&notifier.FakeNotifier{}},
-			nil,
-			config.Group{
-				Name: "Recording and alerting rules",
-				Rules: []config.Rule{
-					{Alert: "alert1", Expr: "up > 0"},
-					{Alert: "alert2", Expr: "up > 0"},
-					{Record: "record", Expr: "max(up)"},
-				},
-			},
-			"contains recording rules",
-		},
-		{
-			nil,
-			&remotewrite.Client{},
-			config.Group{
-				Name: "Recording and alerting rules",
-				Rules: []config.Rule{
-					{Record: "record1", Expr: "max(up)"},
-					{Record: "record2", Expr: "max(up)"},
-					{Alert: "alert", Expr: "up > 0"},
-				},
-			},
-			"contains alerting rules",
-		},
+func TestManagerUpdate_Failure(t *testing.T) {
+	f := func(notifiers []notifier.Notifier, rw remotewrite.RWClient, cfg config.Group, errStrExpected string) {
+		t.Helper()
+
+		m := &manager{
+			groups:         make(map[uint64]*rule.Group),
+			querierBuilder: &datasource.FakeQuerier{},
+			rw:             rw,
+		}
+		if notifiers != nil {
+			m.notifiers = func() []notifier.Notifier { return notifiers }
+		}
+		err := m.update(context.Background(), []config.Group{cfg}, false)
+		if err == nil {
+			t.Fatalf("expected to get error; got nil")
+		}
+		errStr := err.Error()
+		if !strings.Contains(errStr, errStrExpected) {
+			t.Fatalf("missing %q in the error %q", errStrExpected, errStr)
+		}
 	}
 
-	for _, tc := range testCases {
-		t.Run(tc.cfg.Name, func(t *testing.T) {
-			m := &manager{
-				groups:         make(map[uint64]*rule.Group),
-				querierBuilder: &datasource.FakeQuerier{},
-				rw:             tc.rw,
-			}
-			if tc.notifiers != nil {
-				m.notifiers = func() []notifier.Notifier { return tc.notifiers }
-			}
-			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
-			if err == nil {
-				t.Fatalf("expected to get error; got nil")
-			}
-			if !strings.Contains(err.Error(), tc.expErr) {
-				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
-			}
-		})
-	}
+	f(nil, nil, config.Group{
+		Name: "Recording rule only",
+		Rules: []config.Rule{
+			{Record: "record", Expr: "max(up)"},
+		},
+	}, "contains recording rules")
+
+	f(nil, nil, config.Group{
+		Name: "Alerting rule only",
+		Rules: []config.Rule{
+			{Alert: "alert", Expr: "up > 0"},
+		},
+	}, "contains alerting rules")
+
+	f([]notifier.Notifier{&notifier.FakeNotifier{}}, nil, config.Group{
+		Name: "Recording and alerting rules",
+		Rules: []config.Rule{
+			{Alert: "alert1", Expr: "up > 0"},
+			{Alert: "alert2", Expr: "up > 0"},
+			{Record: "record", Expr: "max(up)"},
+		},
+	}, "contains recording rules")
+
+	f(nil, &remotewrite.Client{}, config.Group{
+		Name: "Recording and alerting rules",
+		Rules: []config.Rule{
+			{Record: "record1", Expr: "max(up)"},
+			{Record: "record2", Expr: "max(up)"},
+			{Alert: "alert", Expr: "up > 0"},
+		},
+	}, "contains alerting rules")
 }
 
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {

app/vmalert/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmalert/notifier/alert_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-func TestAlert_ExecTemplate(t *testing.T) {
+func TestAlertExecTemplate(t *testing.T) {
 	extLabels := make(map[string]string)
 	const (
 		extCluster = "prod"
@@ -23,201 +23,164 @@ func TestAlert_ExecTemplate(t *testing.T) {
 	_, err := Init(nil, extLabels, extURL)
 	checkErr(t, err)
 
-	testCases := []struct {
-		name        string
-		alert       *Alert
-		annotations map[string]string
-		expTpl      map[string]string
-	}{
-		{
-			name:        "empty-alert",
-			alert:       &Alert{},
-			annotations: map[string]string{},
-			expTpl:      map[string]string{},
-		},
-		{
-			name: "no-template",
-			alert: &Alert{
-				Value: 1e4,
-				Labels: map[string]string{
-					"instance": "localhost",
+	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
+		t.Helper()
+
+		if err := ValidateTemplates(annotations); err != nil {
+			t.Fatalf("cannot validate annotations: %s", err)
+		}
+
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return []datasource.Metric{
+				{
+					Labels: []datasource.Label{
+						{Name: "foo", Value: "bar"},
+						{Name: "baz", Value: "qux"},
+					},
+					Values:     []float64{1},
+					Timestamps: []int64{1},
 				},
-			},
-			annotations: map[string]string{},
-			expTpl:      map[string]string{},
-		},
-		{
-			name: "label-template",
-			alert: &Alert{
-				Value: 1e4,
-				Labels: map[string]string{
-					"job":      "staging",
-					"instance": "localhost",
+				{
+					Labels: []datasource.Label{
+						{Name: "foo", Value: "garply"},
+						{Name: "baz", Value: "fred"},
+					},
+					Values:     []float64{2},
+					Timestamps: []int64{1},
 				},
-				For: 5 * time.Minute,
-			},
-			annotations: map[string]string{
-				"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
-				"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
-			},
-			expTpl: map[string]string{
-				"summary":     "Too high connection number for localhost for job staging",
-				"description": "It is 10000 connections for localhost for more than 5m0s",
-			},
-		},
-		{
-			name: "expression-template",
-			alert: &Alert{
-				Expr: `vm_rows{"label"="bar"}<0`,
-			},
-			annotations: map[string]string{
-				"exprEscapedQuery":  "{{ $expr|queryEscape }}",
-				"exprEscapedPath":   "{{ $expr|pathEscape }}",
-				"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
-				"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
-				"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
-			},
-			expTpl: map[string]string{
-				"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
-				"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
-				"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
-				"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
-				"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
-			},
-		},
-		{
-			name:  "query",
-			alert: &Alert{Expr: `vm_rows{"label"="bar"}>0`},
-			annotations: map[string]string{
-				"summary": `{{ query "foo" | first | value }}`,
-				"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
-			},
-			expTpl: map[string]string{
-				"summary": "1",
-				"desc":    "bar 1;garply 2;",
-			},
-		},
-		{
-			name: "external",
-			alert: &Alert{
-				Value: 1e4,
-				Labels: map[string]string{
-					"job":      "staging",
-					"instance": "localhost",
-				},
-			},
-			annotations: map[string]string{
-				"url":         "{{ $externalURL }}",
-				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
-				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
-			},
-			expTpl: map[string]string{
-				"url":         extURL,
-				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
-				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
-			},
-		},
-		{
-			name: "alert and group IDs",
-			alert: &Alert{
-				ID:      42,
-				GroupID: 24,
-			},
-			annotations: map[string]string{
-				"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
-			},
-			expTpl: map[string]string{
-				"url": "/api/v1/alert?alertID=42&groupID=24",
-			},
-		},
-		{
-			name: "ActiveAt time",
-			alert: &Alert{
-				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-			},
-			annotations: map[string]string{
-				"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
-			},
-			expTpl: map[string]string{
-				"diagram": "![](http://example.com?render=1660941298",
-			},
-		},
-		{
-			name:  "ActiveAt time is nil",
-			alert: &Alert{},
-			annotations: map[string]string{
-				"default_time": "{{$activeAt}}",
-			},
-			expTpl: map[string]string{
-				"default_time": "0001-01-01 00:00:00 +0000 UTC",
-			},
-		},
-		{
-			name: "ActiveAt custom format",
-			alert: &Alert{
-				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-			},
-			annotations: map[string]string{
-				"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
-			},
-			expTpl: map[string]string{
-				"fire_time": "2022/08/19 20:34:58",
-			},
-		},
-		{
-			name: "ActiveAt query range",
-			alert: &Alert{
-				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-			},
-			annotations: map[string]string{
-				"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
-			},
-			expTpl: map[string]string{
-				"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
-			},
-		},
+			}, nil
+		}
+
+		tpl, err := alert.ExecTemplate(qFn, alert.Labels, annotations)
+		if err != nil {
+			t.Fatalf("cannot execute template: %s", err)
+		}
+		if len(tpl) != len(tplExpected) {
+			t.Fatalf("unexpected number of elements; got %d; want %d", len(tpl), len(tplExpected))
+		}
+		for k := range tplExpected {
+			got, exp := tpl[k], tplExpected[k]
+			if got != exp {
+				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
+			}
+		}
 	}
 
-	qFn := func(_ string) ([]datasource.Metric, error) {
-		return []datasource.Metric{
-			{
-				Labels: []datasource.Label{
-					{Name: "foo", Value: "bar"},
-					{Name: "baz", Value: "qux"},
-				},
-				Values:     []float64{1},
-				Timestamps: []int64{1},
-			},
-			{
-				Labels: []datasource.Label{
-					{Name: "foo", Value: "garply"},
-					{Name: "baz", Value: "fred"},
-				},
-				Values:     []float64{2},
-				Timestamps: []int64{1},
-			},
-		}, nil
-	}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			if err := ValidateTemplates(tc.annotations); err != nil {
-				t.Fatal(err)
-			}
-			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if len(tpl) != len(tc.expTpl) {
-				t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
-			}
-			for k := range tc.expTpl {
-				got, exp := tpl[k], tc.expTpl[k]
-				if got != exp {
-					t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
-				}
-			}
-		})
-	}
+	// empty-alert
+	f(&Alert{}, map[string]string{}, map[string]string{})
+
+	// no-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"instance": "localhost",
+		},
+	}, map[string]string{}, map[string]string{})
+
+	// label-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+		For: 5 * time.Minute,
+	}, map[string]string{
+		"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
+	}, map[string]string{
+		"summary":     "Too high connection number for localhost for job staging",
+		"description": "It is 10000 connections for localhost for more than 5m0s",
+	})
+
+	// expression-template
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}<0`,
+	}, map[string]string{
+		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
+		"exprEscapedPath":   "{{ $expr|pathEscape }}",
+		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
+		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
+		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
+	}, map[string]string{
+		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
+		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
+		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
+		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
+		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
+	})
+
+	// query
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}>0`,
+	}, map[string]string{
+		"summary": `{{ query "foo" | first | value }}`,
+		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+	}, map[string]string{
+		"summary": "1",
+		"desc":    "bar 1;garply 2;",
+	})
+
+	// external
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"url":         "{{ $externalURL }}",
+		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+	}, map[string]string{
+		"url":         extURL,
+		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+	})
+
+	// alert and group IDs
+	f(&Alert{
+		ID:      42,
+		GroupID: 24,
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID=42&groupID=24",
+	})
+
+	// ActiveAt time
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+	}, map[string]string{
+		"diagram": "![](http://example.com?render=1660941298",
+	})
+
+	// ActiveAt time is nil
+	f(&Alert{}, map[string]string{
+		"default_time": "{{$activeAt}}",
+	}, map[string]string{
+		"default_time": "0001-01-01 00:00:00 +0000 UTC",
+	})
+
+	// ActiveAt custom format
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+	}, map[string]string{
+		"fire_time": "2022/08/19 20:34:58",
+	})
+
+	// ActiveAt query range
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+	}, map[string]string{
+		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+	})
 }
 
 func TestAlert_toPromLabels(t *testing.T) {

app/vmalert/notifier/alertmanager.go

@@ -145,7 +145,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecretFile, oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/notifier/alertmanager_test.go

@@ -16,10 +16,10 @@ func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
 	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
-		t.Errorf("unexpected error: %s", err)
+		t.Fatalf("unexpected error: %s", err)
 	}
 	if am.Addr() != addr {
-		t.Errorf("expected to have %q; got %q", addr, am.Addr())
+		t.Fatalf("expected to have %q; got %q", addr, am.Addr())
 	}
 }
 
@@ -28,21 +28,20 @@ func TestAlertManager_Send(t *testing.T) {
 	const headerKey, headerValue = "TenantID", "foo"
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
 		user, pass, ok := r.BasicAuth()
 		if !ok {
-			t.Errorf("unauthorized request")
+			t.Fatalf("unauthorized request")
 		}
 		if user != baUser || pass != baPass {
-			t.Errorf("wrong creds %q:%q; expected %q:%q",
-				user, pass, baUser, baPass)
+			t.Fatalf("wrong creds %q:%q; expected %q:%q", user, pass, baUser, baPass)
 		}
 		c++
 		if r.Method != http.MethodPost {
-			t.Errorf("expected POST method got %s", r.Method)
+			t.Fatalf("expected POST method got %s", r.Method)
 		}
 		switch c {
 		case 0:
@@ -59,25 +58,23 @@ func TestAlertManager_Send(t *testing.T) {
 				GeneratorURL string            `json:"generatorURL"`
 			}
 			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
-				t.Errorf("can not unmarshal data into alert %s", err)
-				t.FailNow()
+				t.Fatalf("can not unmarshal data into alert %s", err)
 			}
 			if len(a) != 1 {
-				t.Errorf("expected 1 alert in array got %d", len(a))
+				t.Fatalf("expected 1 alert in array got %d", len(a))
 			}
 			if a[0].GeneratorURL != "0/0" {
-				t.Errorf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
+				t.Fatalf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
 			}
 			if a[0].StartsAt.IsZero() {
-				t.Errorf("expected non-zero start time")
+				t.Fatalf("expected non-zero start time")
 			}
 			if a[0].EndAt.IsZero() {
-				t.Errorf("expected non-zero end time")
+				t.Fatalf("expected non-zero end time")
 			}
 		case 3:
 			if r.Header.Get(headerKey) != headerValue {
-				t.Errorf("expected header %q to be set to %q; got %q instead",
-					headerKey, headerValue, r.Header.Get(headerKey))
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
 			}
 		}
 	})
@@ -94,13 +91,13 @@ func TestAlertManager_Send(t *testing.T) {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
 	}, aCfg, nil, 0)
 	if err != nil {
-		t.Errorf("unexpected error: %s", err)
+		t.Fatalf("unexpected error: %s", err)
 	}
 	if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
-		t.Error("expected connection error got nil")
+		t.Fatalf("expected connection error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
-		t.Error("expected wrong http code error got nil")
+		t.Fatalf("expected wrong http code error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{{
 		GroupID:     0,
@@ -109,12 +106,12 @@ func TestAlertManager_Send(t *testing.T) {
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
 	}}, nil); err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	if c != 2 {
-		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
+		t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
 	}
 	if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 }