debug: [vmagent] add debug log for max_scrape_size

Signed-off-by: hagen1778 <roman@victoriametrics.com>

.github/workflows/build.yml

@@ -0,0 +1,54 @@
+name: build
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Makefile'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Makefile'
+
+permissions:
+  contents: read
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
+
+      - name: Run crossbuild
+        run: make crossbuild

.github/workflows/codeql-analysis-go.yml

@@ -0,0 +1,62 @@
+name: 'CodeQL Go'
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version: stable
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: go
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: 'language:go'

.github/workflows/codeql-analysis-js-ts.yml

@@ -1,22 +1,27 @@
-name: "CodeQL - JS"
+name: 'CodeQL JS/TS'
 
 on:
   push:
-    branches: [master, cluster]
+    branches:
+      - cluster
+      - master
     paths:
-      - "**.js"
+      - '**.js'
+      - '**.ts'
+      - '**.tsx'
   pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master, cluster]
+    branches:
+      - cluster
+      - master
     paths:
-      - "**.js"
-  schedule:
-    - cron: "30 18 * * 2"
+      - '**.js'
+      - '**.ts'
+      - '**.tsx'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
-  
+
 jobs:
   analyze:
     name: Analyze
@@ -26,11 +31,6 @@ jobs:
       contents: read
       security-events: write
 
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["javascript"]
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -38,9 +38,9 @@ jobs:
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with:
-          languages: ${{ matrix.language }}
+          languages: javascript-typescript
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v3
         with:
-          category: "javascript"
+          category: 'language:js/ts'

.github/workflows/codeql-analysis.yml

@@ -1,103 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
-  schedule:
-    - cron: "30 18 * * 2"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["go"]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-        if: ${{ matrix.language == 'go' }}
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-        if: ${{ matrix.language == 'go' }}
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
-
-      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-      #    and modify them (or add more) to build your code if your project
-      #    uses a compiled language
-
-      #- run: |
-      #   make bootstrap
-      #   make release
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3

.github/workflows/main.yml

@@ -1,29 +1,25 @@
 name: main
+
 on:
   push:
     branches:
-      - master
       - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - master
+    paths:
+      - '**.go'
   pull_request:
     branches:
-      - master
       - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - master
+    paths:
+      - '**.go'
+
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
 
 jobs:
   lint:
@@ -37,16 +33,16 @@ jobs:
         id: go
         uses: actions/setup-go@v5
         with:
-          go-version: stable
           cache: false
+          go-version: stable
 
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
-            ~/go/pkg/mod
             ~/go/bin
+            ~/go/pkg/mod
           key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-check-all-
 
@@ -55,41 +51,18 @@ jobs:
           make check-all
           git diff --exit-code
 
-  build:
-    needs: lint
-    name: build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
-      - name: Build
-        run: make crossbuild
-
   test:
+    name: test
     needs: lint
+    runs-on: ubuntu-latest
+
     strategy:
       matrix:
-        scenario: ["test-full", "test-pure", "test-full-386"]
-    name: test
-    runs-on: ubuntu-latest
+        scenario:
+          - 'test-full'
+          - 'test-full-386'
+          - 'test-pure'
+
     steps:
       - name: Code checkout
         uses: actions/checkout@v4
@@ -98,20 +71,20 @@ jobs:
         id: go
         uses: actions/setup-go@v5
         with:
-          go-version: stable
           cache: false
+          go-version: stable
 
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
-            ~/go/pkg/mod
             ~/go/bin
+            ~/go/pkg/mod
           key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
 
-      - name: run tests
+      - name: Run tests
         run: make ${{ matrix.scenario}}
 
       - name: Publish coverage

.github/workflows/sync-docs.yml

@@ -25,7 +25,7 @@ jobs:
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: docs
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v5
+        uses: crazy-max/ghaction-import-gpg@v6
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}

.gitignore

@@ -7,6 +7,7 @@
 .vscode
 *.test
 *.swp
+/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
@@ -22,4 +23,6 @@ Gemfile.lock
 /_site
 _site
 *.tmp
-/docs/.jekyll-metadata
+/docs/.jekyll-metadata
+coverage.txt
+cspell.json

.golangci.yml

@@ -16,4 +16,7 @@ issues:
 
 linters-settings:
   errcheck:
-    exclude: ./errcheck_excludes.txt
+    exclude-functions:
+      - "fmt.Fprintf"
+      - "fmt.Fprint"
+      - "(net/http.ResponseWriter).Write"

CODE_OF_CONDUCT_RU.md

@@ -1,120 +0,0 @@
-
-# Кодекс Поведения участника
-
-## Наши обязательства
-
-Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
-свободным от притеснений для всех, независимо от возраста, телосложения,
-видимых или невидимых ограничений способности, этнической принадлежности,
-половых признаков, гендерной идентичности и выражения, уровня опыта,
-образования, социо-экономического статуса, национальности, внешности,
-расы, религии, или сексуальной идентичности и ориентации.
-
-Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
-дружелюбное, многообразное, инклюзивное и здоровое сообщество.
-
-## Наши стандарты
-
-Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
-
-* Проявление доброты и эмпатии к другим участникам проекта
-* Уважение к чужой точке зрения и опыту
-* Конструктивная критика и принятие конструктивной критики
-* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
-  и извлечение уроков из опыта
-* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
-
-Примеры неприемлемого поведения участников включают в себя:
-
-* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
-* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
-* Публичное или приватное домогательство
-* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
-* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
-
-## Обязанности
-
-Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
-поведения и будут предпринимать соответствующие и честные меры по исправлению положения
-в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
-
-Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
-комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
-с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
-
-## Область применения
-
-Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
-а также когда человек официально представляет сообщество в публичных местах.
-Примеры представления проекта или сообщества включают использование официальной электронной почты,
-публикации в официальном аккаунте в социальных сетях,
-или упоминания как представителя в онлайн или оффлайн мероприятии.
-
-## Приведение в исполнение
-
-О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
-поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
-Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
-
-Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
-неприкосновенность автора сообщения.
-
-## Руководство по исполнению
-
-Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
-чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
-
-### 1. Исправление
-
-**Общественное влияние**: Использование недопустимой лексики или другое поведение,
-считающиеся непрофессиональным или нежелательным в сообществе.
-
-**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
-объясняющее суть нарушения и почему такое поведение
-было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
-
-### 2. Предупреждение
-
-**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
-
-**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
-На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
-включая незапрошенное взаимодействие
-с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
-в публичных пространствах, а так же во внешних каналах,
-таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
-
-### 3. Временный бан
-
-**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
-включая продолжительное неуместное поведение.
-
-**Последствия**: Временный запрет (бан) на любое взаимодействие
-или публичное общение с сообществом на определенный период времени.
-На этот период не допускается публичное или личное взаимодействие с людьми,
-вовлеченными в инцидент, включая незапрошенное взаимодействие
-с теми, кто обеспечивает соблюдение Кодекса.
-Нарушение этих правил влечет за собой вечный бан.
-
-### 4. Вечный бан
-
-**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
-включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
-или проявление агрессии либо пренебрежительного отношения к категориям лиц.
-
-**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
-
-## Атрибуция
-
-Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
-версии 2.0, доступной по адресу
-<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
-
-Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
-
-[homepage]: https://www.contributor-covenant.org
-
-Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
-<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
-<https://www.contributor-covenant.org/translations>.

Makefile

@@ -13,14 +13,15 @@ PKG_TAG := $(BUILDINFO_TAG)
 endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
+TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
+include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
-include snap/local/Makefile
 include package/release/Makefile
 
 all: \
@@ -245,7 +246,7 @@ release-victoria-metrics-windows-amd64:
 
 release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
@@ -302,7 +303,7 @@ release-victoria-logs-windows-amd64:
 
 release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
@@ -366,7 +367,7 @@ release-vmutils-goos-goarch: \
 	vmrestore-$(GOOS)-$(GOARCH)-prod \
 	vmctl-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
 			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
@@ -440,6 +441,8 @@ vet:
 
 check-all: fmt vet golangci-lint govulncheck
 
+clean-checkers: remove-golangci-lint remove-govulncheck
+
 test:
 	go test ./lib/... ./app/...
 
@@ -492,7 +495,10 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.57.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
+
+remove-golangci-lint:
+	rm -rf `which golangci-lint`
 
 govulncheck: install-govulncheck
 	govulncheck ./...
@@ -500,38 +506,11 @@ govulncheck: install-govulncheck
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 
+remove-govulncheck:
+	rm -rf `which govulncheck`
+
 install-wwhrd:
 	which wwhrd || go install github.com/frapposelli/wwhrd@latest
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
-
-copy-docs:
-# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
-# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
-	echo "---" > ${DST}
-	@if [ ${ORDER} -ne 0 ]; then \
-		echo "sort: ${ORDER}" >> ${DST}; \
-		echo "weight: ${ORDER}" >> ${DST}; \
-		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
-	fi
-
-	echo "title: ${TITLE}" >> ${DST}
-	@if [ ${OLD_URL} ]; then \
-		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
-	fi
-	echo "---" >> ${DST}
-	cat ${SRC} >> ${DST}
-	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
-	rm -rf docs/*.tmp
-
-# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
-# For ORDER=0 it adds no order tag/weight tag.
-# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
-# Images starting with <img src="docs/ are replaced with <img src="
-# Cluster docs are supposed to be ordered as 2nd.
-# The rest of docs is ordered manually.
-docs-sync:
-	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
-	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
+| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to security@victoriametrics.com
+Please report any security issues to <security@victoriametrics.com>

app/victoria-logs/Makefile

@@ -81,6 +81,9 @@ victoria-logs-linux-ppc64le:
 victoria-logs-linux-s390x:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
+victoria-logs-linux-loong64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+
 victoria-logs-linux-386:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
@@ -101,3 +104,10 @@ victoria-logs-windows-amd64:
 
 victoria-logs-pure:
 	APP_NAME=victoria-logs $(MAKE) app-local-pure
+
+run-victoria-logs:
+	mkdir -p victoria-logs-data
+	DOCKER_OPTS='-v $(shell pwd)/victoria-logs-data:/victoria-logs-data' \
+	APP_NAME=victoria-logs \
+	ARGS='' \
+	$(MAKE) run-via-docker

app/victoria-logs/main.go

@@ -47,7 +47,7 @@ func main() {
 	vlinsert.Init()
 
 	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
-	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
+	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
@@ -77,7 +77,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/VictoriaLogs/'>https://docs.victoriametrics.com/VictoriaLogs/</a></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/'>https://docs.victoriametrics.com/victorialogs/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"select/vmui", "Web UI for VictoriaLogs"},
@@ -99,7 +99,7 @@ func usage() {
 	const s = `
 victoria-logs is a log management and analytics service.
 
-See the docs at https://docs.victoriametrics.com/VictoriaLogs/
+See the docs at https://docs.victoriametrics.com/victorialogs/
 `
 	flagutil.Usage(s)
 }

app/victoria-logs/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/victoria-metrics/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -117,7 +118,7 @@ func (q *Query) metrics() ([]Metric, error) {
 type QueryInstant struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TV     [2]interface{}    `json:"value"`
+		TV     [2]any            `json:"value"`
 	} `json:"result"`
 }
 
@@ -140,7 +141,7 @@ func (q QueryInstant) metrics() ([]Metric, error) {
 type QueryRange struct {
 	Result []struct {
 		Metric map[string]string `json:"metric"`
-		Values [][]interface{}   `json:"values"`
+		Values [][]any           `json:"values"`
 	} `json:"result"`
 }
 
@@ -249,24 +250,24 @@ func TestWriteRead(t *testing.T) {
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
-		for _, test := range readIn("prometheus", t, insertionTime) {
+		for _, test := range readIn("prometheus", insertionTime) {
 			if test.Data == nil {
 				continue
 			}
-			s := newSuite(t)
 			r := testutil.WriteRequest{}
-			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
-			data, err := testutil.Compress(r)
-			s.greaterThan(len(r.Timeseries), 0)
-			if err != nil {
-				t.Errorf("error compressing %v %s", r, err)
-				t.Fail()
+			testData := strings.Join(test.Data, "\n")
+			if err := json.Unmarshal([]byte(testData), &r.Timeseries); err != nil {
+				panic(fmt.Errorf("BUG: cannot unmarshal TimeSeries: %s\ntest data\n%s", err, testData))
 			}
+			if n := len(r.Timeseries); n <= 0 {
+				panic(fmt.Errorf("BUG: expecting non-empty Timeseries in test data:\n%s", testData))
+			}
+			data := testutil.Compress(r)
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
 	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", t, insertionTime) {
+		for _, test := range readIn("csv", insertionTime) {
 			if test.Data == nil {
 				continue
 			}
@@ -275,7 +276,7 @@ func testWrite(t *testing.T) {
 	})
 
 	t.Run("influxdb", func(t *testing.T) {
-		for _, x := range readIn("influxdb", t, insertionTime) {
+		for _, x := range readIn("influxdb", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -284,7 +285,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("graphite", func(t *testing.T) {
-		for _, x := range readIn("graphite", t, insertionTime) {
+		for _, x := range readIn("graphite", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -293,7 +294,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdb", func(t *testing.T) {
-		for _, x := range readIn("opentsdb", t, insertionTime) {
+		for _, x := range readIn("opentsdb", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -302,7 +303,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdbhttp", func(t *testing.T) {
-		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
+		for _, x := range readIn("opentsdbhttp", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -316,7 +317,7 @@ func testWrite(t *testing.T) {
 func testRead(t *testing.T) {
 	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
-			for _, x := range readIn(engine, t, insertionTime) {
+			for _, x := range readIn(engine, insertionTime) {
 				test := x
 				t.Run(test.Name, func(t *testing.T) {
 					t.Parallel()
@@ -365,11 +366,10 @@ func testRead(t *testing.T) {
 	}
 }
 
-func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
-	t.Helper()
-	s := newSuite(t)
+func readIn(readFor string, insertTime time.Time) []test {
+	testDir := filepath.Join(testFixturesDir, readFor)
 	var tt []test
-	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
+	err := filepath.Walk(testDir, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -377,84 +377,129 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 			return nil
 		}
 		b, err := os.ReadFile(path)
-		s.noError(err)
+		if err != nil {
+			panic(fmt.Errorf("BUG: cannot read %s: %s", path, err))
+		}
 		item := test{}
-		s.noError(json.Unmarshal(b, &item))
+		if err := json.Unmarshal(b, &item); err != nil {
+			panic(fmt.Errorf("cannot parse %T from %s: %s; data:\n%s", &item, path, err, b))
+		}
 		for i := range item.Data {
 			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
 		}
 		tt = append(tt, item)
 		return nil
-	}))
+	})
+	if err != nil {
+		panic(fmt.Errorf("BUG: cannot read test data at %s: %w", testDir, err))
+	}
 	if len(tt) == 0 {
-		t.Fatalf("no test found in %s", filepath.Join(testFixturesDir, readFor))
+		panic(fmt.Errorf("BUG: no tests found in %s", testDir))
 	}
 	return tt
 }
 
 func httpWrite(t *testing.T, address, query string, r io.Reader) {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Post(address+query, "", r)
-	s.noError(err)
-	s.noError(resp.Body.Close())
-	s.equalInt(resp.StatusCode, 204)
+
+	requestURL := address + query
+	resp, err := http.Post(requestURL, "", r)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != 204 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 204", requestURL, resp.StatusCode)
+	}
 }
 
-func tcpWrite(t *testing.T, address string, data string) {
+func tcpWrite(t *testing.T, address, data string) {
 	t.Helper()
-	s := newSuite(t)
+
 	conn, err := net.Dial("tcp", address)
-	s.noError(err)
+	if err != nil {
+		t.Fatalf("cannot dial %s: %s", address, err)
+	}
 	defer func() {
 		_ = conn.Close()
 	}()
 	n, err := conn.Write([]byte(data))
-	s.noError(err)
-	s.equalInt(n, len(data))
+	if err != nil {
+		t.Fatalf("cannot write %d bytes to %s: %s", len(data), address, err)
+	}
+	if n != len(data) {
+		panic(fmt.Errorf("BUG: conn.Write() returned unexpected number of written bytes to %s; got %d; want %d", address, n, len(data)))
+	}
 }
 
 func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+
 	var rows []Metric
-	for dec := json.NewDecoder(resp.Body); dec.More(); {
+	dec := json.NewDecoder(resp.Body)
+	for {
 		var row Metric
-		s.noError(dec.Decode(&row))
+		err := dec.Decode(&row)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return rows
+			}
+			t.Fatalf("cannot decode %T from response received from %s: %s", &row, requestURL, err)
+		}
 		rows = append(rows, row)
 	}
-	return rows
 }
 
-func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
+func httpReadStruct(t *testing.T, address, query string, dst any) {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
-	s.noError(json.NewDecoder(resp.Body).Decode(dst))
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+	err = json.NewDecoder(resp.Body).Decode(dst)
+	if err != nil {
+		t.Fatalf("cannot decode %T from response received from %s: %s", dst, requestURL, err)
+	}
 }
 
 func httpReadData(t *testing.T, address, query string) []byte {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
 	data, err := io.ReadAll(resp.Body)
-	s.noError(err)
+	if err != nil {
+		t.Fatalf("cannot read response from %s: %s", requestURL, err)
+	}
 	return data
 }
 
@@ -503,34 +548,6 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
-type suite struct{ t *testing.T }
-
-func newSuite(t *testing.T) *suite { return &suite{t: t} }
-
-func (s *suite) noError(err error) {
-	s.t.Helper()
-	if err != nil {
-		s.t.Errorf("unexpected error %v", err)
-		s.t.FailNow()
-	}
-}
-
-func (s *suite) equalInt(a, b int) {
-	s.t.Helper()
-	if a != b {
-		s.t.Errorf("%d not equal %d", a, b)
-		s.t.FailNow()
-	}
-}
-
-func (s *suite) greaterThan(a, b int) {
-	s.t.Helper()
-	if a <= b {
-		s.t.Errorf("%d less or equal then %d", a, b)
-		s.t.FailNow()
-	}
-}
-
 func TestImportJSONLines(t *testing.T) {
 	f := func(labelsCount, labelLen int) {
 		t.Helper()

app/victoria-metrics/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/victoria-metrics/test/prom_writer.go

@@ -1,12 +1,16 @@
 package test
 
-import "github.com/golang/snappy"
+import (
+	"fmt"
+
+	"github.com/golang/snappy"
+)
 
 // Compress marshals and compresses wr.
-func Compress(wr WriteRequest) ([]byte, error) {
+func Compress(wr WriteRequest) []byte {
 	data, err := wr.Marshal()
 	if err != nil {
-		return nil, err
+		panic(fmt.Errorf("BUG: cannot compress WriteRequest: %s", err))
 	}
-	return snappy.Encode(nil, data), nil
+	return snappy.Encode(nil, data)
 }

app/vlinsert/elasticsearch/elasticsearch.go

@@ -57,6 +57,12 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
+	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
+		// Return fake response for Logstash APIs requests.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
 	switch path {
 	case "/":
 		switch r.Method {
@@ -97,12 +103,10 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-		processLogMessage := cp.GetProcessLogMessageFunc(lr)
+		lmp := cp.NewLogMessageProcessor()
 		isGzip := r.Header.Get("Content-Encoding") == "gzip"
-		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
-		vlstorage.MustAddRows(lr)
-		logstorage.PutLogRows(lr)
+		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, lmp)
+		lmp.MustClose()
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
 			return true
@@ -131,9 +135,7 @@ var (
 	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (int, error) {
+func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp insertutils.LogMessageProcessor) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
 	if isGzip {
@@ -158,7 +160,7 @@ func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
 	n := 0
 	nCheckpoint := 0
 	for {
-		ok, err := readBulkLine(sc, timeField, msgField, processLogMessage)
+		ok, err := readBulkLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil || !ok {
 			rowsIngestedTotal.Add(n - nCheckpoint)
@@ -174,9 +176,7 @@ func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
 
 var lineBufferPool bytesutil.ByteBufferPool
 
-func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (bool, error) {
+func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 
 	// Read the command, must be "create" or "index"
@@ -210,7 +210,7 @@ func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
 	}
 	line = sc.Bytes()
 	p := logstorage.GetJSONParser()
-	if err := p.ParseLogMessage(line, ""); err != nil {
+	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
 
@@ -221,8 +221,8 @@ func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
 	if ts == 0 {
 		ts = time.Now().UnixNano()
 	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
 	logstorage.PutJSONParser(p)
 
 	return true, nil
@@ -272,9 +272,9 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
 		}
 		return t.UnixNano(), nil
 	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
+	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
+	if !ok {
+		return 0, fmt.Errorf("cannot parse timestamp %q", s)
 	}
-	return t.UnixNano(), nil
+	return nsecs, nil
 }

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -4,23 +4,18 @@ import (
 	"bytes"
 	"compress/gzip"
 	"fmt"
-	"reflect"
-	"strings"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestReadBulkRequestFailure(t *testing.T) {
+func TestReadBulkRequest_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
 
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
-		}
-
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, "_time", "_msg", processLogMessage)
+		rows, err := readBulkRequest(r, false, "_time", "_msg", tlp)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -37,58 +32,38 @@ func TestReadBulkRequestFailure(t *testing.T) {
 foobar`)
 }
 
-func TestReadBulkRequestSuccess(t *testing.T) {
+func TestReadBulkRequest_Success(t *testing.T) {
 	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
+		tlp := &insertutils.TestLogMessageProcessor{}
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, timeField, msgField, processLogMessage)
+		rows, err := readBulkRequest(r, false, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if rows != rowsExpected {
 			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 
 		// Read the request with compression
-		timestamps = nil
-		result = ""
+		tlp = &insertutils.TestLogMessageProcessor{}
 		compressedData := compressData(data)
 		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest(r, true, timeField, msgField, processLogMessage)
+		rows, err = readBulkRequest(r, true, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if rows != rowsExpected {
 			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatalf("verification failure after compression: %s", err)
 		}
 	}
 
@@ -111,8 +86,7 @@ func TestReadBulkRequestSuccess(t *testing.T) {
 	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
 	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
+{"_msg":"xyz","@timestamp":"","x":"y"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
 func BenchmarkReadBulkRequest(b *testing.B) {
@@ -33,7 +33,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 
 	timeField := "@timestamp"
 	msgField := "message"
-	processLogMessage := func(_ int64, _ []logstorage.Field) {}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))
@@ -41,7 +41,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
+			_, err := readBulkRequest(r, isGzip, timeField, msgField, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutils/common_params.go

@@ -2,6 +2,8 @@ package insertutils
 
 import (
 	"net/http"
+	"sync"
+	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
@@ -10,11 +12,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 // CommonParams contains common HTTP parameters used by log ingestion APIs.
 //
-// See https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/#http-parameters
+// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
 type CommonParams struct {
 	TenantID     logstorage.TenantID
 	TimeField    string
@@ -71,29 +74,126 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 	return cp, nil
 }
 
-// GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
-func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
-	return func(timestamp int64, fields []logstorage.Field) {
-		if len(fields) > *MaxFieldsPerLine {
-			rf := logstorage.RowFormatter(fields)
-			logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
-			rowsDroppedTotalTooManyFields.Inc()
-			return
-		}
-
-		lr.MustAdd(cp.TenantID, timestamp, fields)
-		if cp.Debug {
-			s := lr.GetRowString(0)
-			lr.ResetKeepSettings()
-			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
-			rowsDroppedTotalDebug.Inc()
-			return
-		}
-		if lr.NeedFlush() {
-			vlstorage.MustAddRows(lr)
-			lr.ResetKeepSettings()
-		}
+// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
+	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
+	cp := &CommonParams{
+		TenantID:  tenantID,
+		TimeField: "timestamp",
+		MsgField:  "message",
+		StreamFields: []string{
+			"hostname",
+			"app_name",
+			"proc_id",
+		},
 	}
+
+	return cp
+}
+
+// LogMessageProcessor is an interface for log message processors.
+type LogMessageProcessor interface {
+	// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
+	//
+	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
+	AddRow(timestamp int64, fields []logstorage.Field)
+
+	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
+	MustClose()
+}
+
+type logMessageProcessor struct {
+	mu            sync.Mutex
+	wg            sync.WaitGroup
+	stopCh        chan struct{}
+	lastFlushTime time.Time
+
+	cp *CommonParams
+	lr *logstorage.LogRows
+}
+
+func (lmp *logMessageProcessor) initPeriodicFlush() {
+	lmp.lastFlushTime = time.Now()
+
+	lmp.wg.Add(1)
+	go func() {
+		defer lmp.wg.Done()
+
+		d := timeutil.AddJitterToDuration(time.Second)
+		ticker := time.NewTicker(d)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-lmp.stopCh:
+				return
+			case <-ticker.C:
+				lmp.mu.Lock()
+				if time.Since(lmp.lastFlushTime) >= d {
+					lmp.flushLocked()
+				}
+				lmp.mu.Unlock()
+			}
+		}
+	}()
+}
+
+// AddRow adds new log message to lmp with the given timestamp and fields.
+func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	lmp.mu.Lock()
+	defer lmp.mu.Unlock()
+
+	if len(fields) > *MaxFieldsPerLine {
+		rf := logstorage.RowFormatter(fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		rowsDroppedTotalTooManyFields.Inc()
+		return
+	}
+
+	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
+	if lmp.cp.Debug {
+		s := lmp.lr.GetRowString(0)
+		lmp.lr.ResetKeepSettings()
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		rowsDroppedTotalDebug.Inc()
+		return
+	}
+	if lmp.lr.NeedFlush() {
+		lmp.flushLocked()
+	}
+}
+
+// flushLocked must be called under locked lmp.mu.
+func (lmp *logMessageProcessor) flushLocked() {
+	lmp.lastFlushTime = time.Now()
+	vlstorage.MustAddRows(lmp.lr)
+	lmp.lr.ResetKeepSettings()
+}
+
+// MustClose flushes the remaining data to the underlying storage and closes lmp.
+func (lmp *logMessageProcessor) MustClose() {
+	close(lmp.stopCh)
+	lmp.wg.Wait()
+
+	lmp.flushLocked()
+	logstorage.PutLogRows(lmp.lr)
+	lmp.lr = nil
+}
+
+// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
+//
+// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
+func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	lmp := &logMessageProcessor{
+		cp: cp,
+		lr: lr,
+
+		stopCh: make(chan struct{}),
+	}
+	lmp.initPeriodicFlush()
+
+	return lmp
 }
 
 var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)

app/vlinsert/insertutils/testutils.go

@@ -0,0 +1,53 @@
+package insertutils
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// TestLogMessageProcessor implements LogMessageProcessor for testing.
+type TestLogMessageProcessor struct {
+	timestamps []int64
+	rows       []string
+}
+
+// AddRow adds row with the given timestamp and fields to tlp
+func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	tlp.timestamps = append(tlp.timestamps, timestamp)
+	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
+}
+
+// MustClose closes tlp.
+func (tlp *TestLogMessageProcessor) MustClose() {
+}
+
+// Verify verifies the number of rows, timestamps and results after AddRow calls.
+func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error {
+	result := strings.Join(tlp.rows, "\n")
+	if len(tlp.rows) != rowsExpected {
+		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), rowsExpected, result, resultExpected)
+	}
+
+	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
+		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
+	}
+	if result != resultExpected {
+		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+	}
+
+	return nil
+}
+
+// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
+type BenchmarkLogMessageProcessor struct{}
+
+// AddRow implements LogMessageProcessor interface.
+func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field) {
+}
+
+// MustClose implements LogMessageProcessor interface.
+func (blp *BenchmarkLogMessageProcessor) MustClose() {
+}

app/vlinsert/insertutils/timestamp.go

@@ -0,0 +1,33 @@
+package insertutils
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+//
+// The value for the timeField is set to empty string after returning from the function,
+// so it could be ignored during data ingestion.
+//
+// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
+func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		if f.Value == "" || f.Value == "0" {
+			return time.Now().UnixNano(), nil
+		}
+		nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
+		if !ok {
+			return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
+		}
+		f.Value = ""
+		return nsecs, nil
+	}
+	return time.Now().UnixNano(), nil
+}

app/vlinsert/insertutils/timestamp_test.go

@@ -0,0 +1,75 @@
+package insertutils
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
+	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
+		t.Helper()
+
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if nsecs != nsecsExpected {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
+		}
+
+		for _, f := range fields {
+			if f.Name == timeField {
+				if f.Value != "" {
+					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
+				}
+			}
+		}
+	}
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20Z"},
+	}, 1718753840000000000)
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
+	}, 1718725040000000000)
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
+	}, 1718773640123000000)
+
+	f("time", []logstorage.Field{
+		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
+		{Name: "foo", Value: "bar"},
+	}, 1718773640123456789)
+}
+
+func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+
+		fields := []logstorage.Field{
+			{Name: "time", Value: s},
+		}
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if nsecs != 0 {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
+		}
+	}
+
+	f("foobar")
+
+	// no Z at the end
+	f("2024-06-18T23:37:20")
+
+	// incomplete time
+	f("2024-06-18")
+	f("2024-06-18T23:37")
+}

app/vlinsert/jsonline/jsonline.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 
@@ -19,13 +20,13 @@ import (
 )
 
 // RequestHandler processes jsonline insert requests
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+func RequestHandler(w http.ResponseWriter, r *http.Request) {
 	startTime := time.Now()
 	w.Header().Add("Content-Type", "application/json")
 
 	if r.Method != "POST" {
 		w.WriteHeader(http.StatusMethodNotAllowed)
-		return true
+		return
 	}
 
 	requestsTotal.Inc()
@@ -33,27 +34,40 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
 	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
 
 	reader := r.Body
 	if r.Header.Get("Content-Encoding") == "gzip" {
 		zr, err := common.GetGzipReader(reader)
 		if err != nil {
-			logger.Errorf("cannot read gzipped _bulk request: %s", err)
-			return true
+			logger.Errorf("cannot read gzipped jsonline request: %s", err)
+			return
 		}
 		defer common.PutGzipReader(zr)
 		reader = zr
 	}
 
-	wcr := writeconcurrencylimiter.GetReader(reader)
+	lmp := cp.NewLogMessageProcessor()
+	err = processStreamInternal(reader, cp.TimeField, cp.MsgField, lmp)
+	lmp.MustClose()
+
+	if err != nil {
+		logger.Errorf("jsonline: %s", err)
+	} else {
+		// update requestDuration only for successfully parsed requests.
+		// There is no need in updating requestDuration for request errors,
+		// since their timings are usually much smaller than the timing for successful request parsing.
+		requestDuration.UpdateDuration(startTime)
+	}
+}
+
+func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertutils.LogMessageProcessor) error {
+	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
 	lb := lineBufferPool.Get()
@@ -65,31 +79,21 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	n := 0
 	for {
-		ok, err := readLine(sc, cp.TimeField, cp.MsgField, processLogMessage)
+		ok, err := readLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil {
-			logger.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
-			break
+			errorsTotal.Inc()
+			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
 		}
 		if !ok {
-			break
+			return nil
 		}
 		n++
 		rowsIngestedTotal.Inc()
 	}
-
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-
-	// update jsonlineRequestDuration only for successfully parsed requests.
-	// There is no need in updating jsonlineRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	jsonlineRequestDuration.UpdateDuration(startTime)
-
-	return true
 }
 
-func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
+func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 	for len(line) == 0 {
 		if !sc.Scan() {
@@ -105,56 +109,27 @@ func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage f
 	}
 
 	p := logstorage.GetJSONParser()
-	if err := p.ParseLogMessage(line, ""); err != nil {
+	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
+	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
 	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
+		return false, fmt.Errorf("cannot get timestamp: %w", err)
 	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
 	logstorage.PutJSONParser(p)
 
 	return true, nil
 }
 
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		timestamp, err := parseISO8601Timestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
-	}
-	return 0, nil
-}
-
-func parseISO8601Timestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case for returning the current timestamp.
-		// It must be automatically converted to the current timestamp by the caller.
-		return 0, nil
-	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
-	}
-	return t.UnixNano(), nil
-}
-
 var lineBufferPool bytesutil.ByteBufferPool
 
 var (
-	requestsTotal           = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	rowsIngestedTotal       = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
-	jsonlineRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+
+	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
+	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
+
+	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
 )

app/vlinsert/jsonline/jsonline_test.go

@@ -1,59 +1,27 @@
 package jsonline
 
 import (
-	"bufio"
 	"bytes"
-	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"reflect"
-	"strings"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestReadBulkRequestSuccess(t *testing.T) {
+func TestProcessStreamInternal_Success(t *testing.T) {
 	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
-
-		// Read the request without compression
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		sc := bufio.NewScanner(r)
-		rows := 0
-		for {
-			ok, err := readLine(sc, timeField, msgField, processLogMessage)
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-			if !ok {
-				break
-			}
-			rows++
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		if err := processStreamInternal(r, timeField, msgField, tlp); err != nil {
+			t.Fatalf("unexpected error: %s", err)
 		}
 
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 	}
 
-	// Verify non-empty data
 	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
 {"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
@@ -64,7 +32,24 @@ func TestReadBulkRequestSuccess(t *testing.T) {
 	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
 	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
+{"_msg":"xyz","@timestamp":"","x":"y"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
+
+func TestProcessStreamInternal_Failure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal(r, "time", "", tlp); err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// invalid json
+	f("foobar")
+
+	// invalid timestamp field
+	f(`{"time":"foobar"}`)
+}

app/vlinsert/loki/loki.go

@@ -11,7 +11,8 @@ import (
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
 	case "/api/v1/push":
-		return handleInsert(r, w)
+		handleInsert(r, w)
+		return true
 	case "/ready":
 		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
 		w.WriteHeader(http.StatusOK)
@@ -23,14 +24,14 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 }
 
 // See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-func handleInsert(r *http.Request, w http.ResponseWriter) bool {
+func handleInsert(r *http.Request, w http.ResponseWriter) {
 	contentType := r.Header.Get("Content-Type")
 	switch contentType {
 	case "application/json":
-		return handleJSON(r, w)
+		handleJSON(r, w)
 	default:
 		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		return handleProtobuf(r, w)
+		handleProtobuf(r, w)
 	}
 }
 
@@ -45,7 +46,7 @@ func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
 	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
 		org := r.Header.Get("X-Scope-OrgID")
 		if org != "" {
-			tenantID, err := logstorage.GetTenantIDFromString(org)
+			tenantID, err := logstorage.ParseTenantID(org)
 			if err != nil {
 				return nil, err
 			}

app/vlinsert/loki/loki_json.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -20,15 +21,15 @@ import (
 
 var parserPool fastjson.ParserPool
 
-func handleJSON(r *http.Request, w http.ResponseWriter) bool {
+func handleJSON(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
-	lokiRequestsJSONTotal.Inc()
+	requestsJSONTotal.Inc()
 	reader := r.Body
 	if r.Header.Get("Content-Encoding") == "gzip" {
 		zr, err := common.GetGzipReader(reader)
 		if err != nil {
 			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return true
+			return
 		}
 		defer common.PutGzipReader(zr)
 		reader = zr
@@ -39,45 +40,41 @@ func handleJSON(r *http.Request, w http.ResponseWriter) bool {
 	writeconcurrencylimiter.PutReader(wcr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
+		return
 	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
+		return
 	}
 	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseJSONRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseJSONRequest(data, lmp)
+	lmp.MustClose()
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
-		return true
+		return
 	}
 
 	rowsIngestedJSONTotal.Add(n)
 
-	// update lokiRequestJSONDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestJSONDuration for request errors,
+	// update requestJSONDuration only for successfully parsed requests
+	// There is no need in updating requestJSONDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestJSONDuration.UpdateDuration(startTime)
-
-	return true
+	requestJSONDuration.UpdateDuration(startTime)
 }
 
 var (
-	lokiRequestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	rowsIngestedJSONTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
-	lokiRequestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
+	requestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
+	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
+	requestJSONDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
-func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
 	v, err := p.ParseBytes(data)
@@ -170,7 +167,7 @@ func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, field
 				Name:  "_msg",
 				Value: bytesutil.ToUnsafeString(msg),
 			})
-			processLogMessage(ts, fields)
+			lmp.AddRow(ts, fields)
 		}
 		rowsIngested += len(lines)
 	}

app/vlinsert/loki/loki_json_test.go

@@ -1,19 +1,17 @@
 package loki
 
 import (
-	"fmt"
-	"strings"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestParseJSONRequestFailure(t *testing.T) {
+func TestParseJSONRequest_Failure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
-			t.Fatalf("unexpected call to parseJSONRequest callback!")
-		})
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		n, err := parseJSONRequest([]byte(s), tlp)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -56,39 +54,30 @@ func TestParseJSONRequestFailure(t *testing.T) {
 	f(`{"streams":[{"values":[["123",1234]]}]}`)
 }
 
-func TestParseJSONRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
+func TestParseJSONRequest_Success(t *testing.T) {
+	f := func(s string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
-		var lines []string
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+
+		n, err := parseJSONRequest([]byte(s), tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+	f(`{"streams":[]}`, nil, ``)
+	f(`{"streams":[{"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
 
 	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 
 	// Non-empty stream labels
 	f(`{"streams":[{"stream":{
@@ -98,9 +87,9 @@ func TestParseJSONRequestSuccess(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+{"label1":"value1","label2":"value2","_msg":"abc"}
+{"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
 	// Multiple streams
 	f(`{
@@ -124,7 +113,7 @@ _time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
 			]
 		}
 	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
+{"foo":"bar","a":"b","_msg":"abc"}
+{"x":"y","_msg":"yx"}`)
 }

app/vlinsert/loki/loki_json_timing_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func BenchmarkParseJSONRequest(b *testing.B) {
@@ -22,12 +22,13 @@ func BenchmarkParseJSONRequest(b *testing.B) {
 }
 
 func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseJSONRequest(data, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/loki/loki_protobuf.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -23,53 +24,49 @@ var (
 	pushReqsPool sync.Pool
 )
 
-func handleProtobuf(r *http.Request, w http.ResponseWriter) bool {
+func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
-	lokiRequestsProtobufTotal.Inc()
+	requestsProtobufTotal.Inc()
 	wcr := writeconcurrencylimiter.GetReader(r.Body)
 	data, err := io.ReadAll(wcr)
 	writeconcurrencylimiter.PutReader(wcr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
+		return
 	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
+		return
 	}
 	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
-		return true
+		return
 	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseProtobufRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseProtobufRequest(data, lmp)
+	lmp.MustClose()
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
-		return true
+		return
 	}
 
 	rowsIngestedProtobufTotal.Add(n)
 
-	// update lokiRequestProtobufDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestProtobufDuration for request errors,
+	// update requestProtobufDuration only for successfully parsed requests
+	// There is no need in updating requestProtobufDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestProtobufDuration.UpdateDuration(startTime)
-
-	return true
+	requestProtobufDuration.UpdateDuration(startTime)
 }
 
 var (
-	lokiRequestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	rowsIngestedProtobufTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
-	lokiRequestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	requestsProtobufTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
+	requestProtobufDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
-func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
 	bb := bytesBufPool.Get()
 	defer bytesBufPool.Put(bb)
 
@@ -82,12 +79,14 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err = req.Unmarshal(bb.B)
+	err = req.UnmarshalProtobuf(bb.B)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
-	var commonFields []logstorage.Field
+	fields := getFields()
+	defer putFields(fields)
+
 	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
@@ -95,30 +94,60 @@ func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, f
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
 		// Labels are same for all entries in the stream.
-		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
+		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
 		if err != nil {
 			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
-		fields := commonFields
+		commonFieldsLen := len(fields.fields)
 
 		entries := stream.Entries
 		for j := range entries {
-			entry := &entries[j]
-			fields = append(fields[:len(commonFields)], logstorage.Field{
+			e := &entries[j]
+			fields.fields = fields.fields[:commonFieldsLen]
+
+			for _, lp := range e.StructuredMetadata {
+				fields.fields = append(fields.fields, logstorage.Field{
+					Name:  lp.Name,
+					Value: lp.Value,
+				})
+			}
+
+			fields.fields = append(fields.fields, logstorage.Field{
 				Name:  "_msg",
-				Value: entry.Line,
+				Value: e.Line,
 			})
-			ts := entry.Timestamp.UnixNano()
+
+			ts := e.Timestamp.UnixNano()
 			if ts == 0 {
 				ts = currentTimestamp
 			}
-			processLogMessage(ts, fields)
+
+			lmp.AddRow(ts, fields.fields)
 		}
 		rowsIngested += len(stream.Entries)
 	}
 	return rowsIngested, nil
 }
 
+func getFields() *fields {
+	v := fieldsPool.Get()
+	if v == nil {
+		return &fields{}
+	}
+	return v.(*fields)
+}
+
+func putFields(f *fields) {
+	f.fields = f.fields[:0]
+	fieldsPool.Put(f)
+}
+
+var fieldsPool sync.Pool
+
+type fields struct {
+	fields []logstorage.Field
+}
+
 // parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
 //
 // See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
@@ -184,6 +213,6 @@ func getPushRequest() *PushRequest {
 }
 
 func putPushRequest(req *PushRequest) {
-	req.Reset()
+	req.reset()
 	pushReqsPool.Put(req)
 }

app/vlinsert/loki/loki_protobuf_test.go

@@ -6,83 +6,80 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/golang/snappy"
 )
 
-func TestParseProtobufRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
+type testLogMessageProcessor struct {
+	pr PushRequest
+}
+
+func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	msg := ""
+	for _, f := range fields {
+		if f.Name == "_msg" {
+			msg = f.Value
+		}
+	}
+	var a []string
+	for _, f := range fields {
+		if f.Name == "_msg" {
+			continue
+		}
+		item := fmt.Sprintf("%s=%q", f.Name, f.Value)
+		a = append(a, item)
+	}
+	labels := "{" + strings.Join(a, ", ") + "}"
+	tlp.pr.Streams = append(tlp.pr.Streams, Stream{
+		Labels: labels,
+		Entries: []Entry{
+			{
+				Timestamp: time.Unix(0, timestamp),
+				Line:      strings.Clone(msg),
+			},
+		},
+	})
+}
+
+func (tlp *testLogMessageProcessor) MustClose() {
+}
+
+func TestParseProtobufRequest_Success(t *testing.T) {
+	f := func(s string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
-		var pr PushRequest
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			msg := ""
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					msg = f.Value
-				}
-			}
-			var a []string
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					continue
-				}
-				item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-				a = append(a, item)
-			}
-			labels := "{" + strings.Join(a, ", ") + "}"
-			pr.Streams = append(pr.Streams, Stream{
-				Labels: labels,
-				Entries: []Entry{
-					{
-						Timestamp: time.Unix(0, timestamp),
-						Line:      msg,
-					},
-				},
-			})
-		})
+
+		tlp := &testLogMessageProcessor{}
+		n, err := parseJSONRequest([]byte(s), tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if n != len(pr.Streams) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(pr.Streams), n)
+		if n != len(tlp.pr.Streams) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
 		}
 
-		data, err := pr.Marshal()
-		if err != nil {
-			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
-		}
+		data := tlp.pr.MarshalProtobuf(nil)
 		encodedData := snappy.Encode(nil, data)
 
-		var lines []string
-		n, err = parseProtobufRequest(encodedData, func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
+		tlp2 := &insertutils.TestLogMessageProcessor{}
+		n, err = parseProtobufRequest(encodedData, tlp2)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		if err := tlp2.Verify(n, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
 		}
 	}
 
 	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+	f(`{"streams":[]}`, nil, ``)
+	f(`{"streams":[{"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
 
 	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 
 	// Non-empty stream labels
 	f(`{"streams":[{"stream":{
@@ -92,9 +89,9 @@ func TestParseProtobufRequestSuccess(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+{"label1":"value1","label2":"value2","_msg":"abc"}
+{"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
 	// Multiple streams
 	f(`{
@@ -118,12 +115,12 @@ _time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
 			]
 		}
 	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
+{"foo":"bar","a":"b","_msg":"abc"}
+{"x":"y","_msg":"yx"}`)
 }
 
-func TestParsePromLabelsSuccess(t *testing.T) {
+func TestParsePromLabels_Success(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		fields, err := parsePromLabels(nil, s)
@@ -147,7 +144,7 @@ func TestParsePromLabelsSuccess(t *testing.T) {
 	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
 }
 
-func TestParsePromLabelsFailure(t *testing.T) {
+func TestParsePromLabels_Failure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 		fields, err := parsePromLabels(nil, s)

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -8,7 +8,8 @@ import (
 
 	"github.com/golang/snappy"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkParseProtobufRequest(b *testing.B) {
@@ -24,12 +25,13 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
 }
 
 func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
+			_, err := parseProtobufRequest(body, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
@@ -37,29 +39,47 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	})
 }
 
-func getProtobufBody(streams, rows, labels int) []byte {
-	var pr PushRequest
-
-	for i := 0; i < streams; i++ {
-		var st Stream
-
-		st.Labels = `{`
-		for j := 0; j < labels; j++ {
-			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
-			if j < labels-1 {
-				st.Labels += `,`
+func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
+	var b []byte
+	var entries []Entry
+	streams := make([]Stream, streamsCount)
+	for i := range streams {
+		b = b[:0]
+		b = append(b, '{')
+		for j := 0; j < labelsCount; j++ {
+			b = append(b, "label_"...)
+			b = strconv.AppendInt(b, int64(j), 10)
+			b = append(b, `="value_`...)
+			b = strconv.AppendInt(b, int64(j), 10)
+			b = append(b, '"')
+			if j < labelsCount-1 {
+				b = append(b, ',')
 			}
 		}
-		st.Labels += `}`
+		b = append(b, '}')
+		labels := string(b)
 
-		for j := 0; j < rows; j++ {
-			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
+		var rowsBuf []byte
+		entriesLen := len(entries)
+		for j := 0; j < rowsCount; j++ {
+			rowsBufLen := len(rowsBuf)
+			rowsBuf = append(rowsBuf, "value_"...)
+			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
+			entries = append(entries, Entry{
+				Timestamp: time.Now(),
+				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
+			})
 		}
 
-		pr.Streams = append(pr.Streams, st)
+		st := &streams[i]
+		st.Labels = labels
+		st.Entries = entries[entriesLen:]
+	}
+	pr := PushRequest{
+		Streams: streams,
 	}
 
-	body, _ := pr.Marshal()
+	body := pr.MarshalProtobuf(nil)
 	encodedBody := snappy.Encode(nil, body)
 
 	return encodedBody

app/vlinsert/loki/pb.go

@@ -0,0 +1,302 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: push_request.proto
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package loki
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/easyproto"
+)
+
+var mp easyproto.MarshalerPool
+
+// PushRequest represents Loki PushRequest
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
+type PushRequest struct {
+	Streams []Stream
+
+	entriesBuf   []Entry
+	labelPairBuf []LabelPair
+}
+
+func (pr *PushRequest) reset() {
+	pr.Streams = pr.Streams[:0]
+
+	pr.entriesBuf = pr.entriesBuf[:0]
+	pr.labelPairBuf = pr.labelPairBuf[:0]
+}
+
+// UnmarshalProtobuf unmarshals pr from protobuf message at src.
+//
+// pr remains valid until src is modified.
+func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
+	pr.reset()
+	var err error
+	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
+	return err
+}
+
+// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
+func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
+	m := mp.Get()
+	pr.marshalProtobuf(m.MessageMarshaler())
+	dst = m.Marshal(dst)
+	mp.Put(m)
+	return dst
+}
+
+func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	for _, s := range pr.Streams {
+		s.marshalProtobuf(mm.AppendMessage(1))
+	}
+}
+
+func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
+	// message PushRequest {
+	//   repeated Stream streams = 1;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			data, ok := fc.MessageData()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
+			}
+			pr.Streams = append(pr.Streams, Stream{})
+			s := &pr.Streams[len(pr.Streams)-1]
+			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
+			if err != nil {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
+			}
+		}
+	}
+	return entriesBuf, labelPairBuf, nil
+}
+
+// Stream represents Loki stream.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
+type Stream struct {
+	Labels  string
+	Entries []Entry
+}
+
+func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendString(1, s.Labels)
+	for _, e := range s.Entries {
+		e.marshalProtobuf(mm.AppendMessage(2))
+	}
+}
+
+func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
+	// message Stream {
+	//   string labels = 1;
+	//   repeated Entry entries = 2;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	entriesBufLen := len(entriesBuf)
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			labels, ok := fc.String()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
+			}
+			s.Labels = labels
+		case 2:
+			data, ok := fc.MessageData()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
+			}
+			entriesBuf = append(entriesBuf, Entry{})
+			e := &entriesBuf[len(entriesBuf)-1]
+			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
+			if err != nil {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
+			}
+		}
+	}
+	s.Entries = entriesBuf[entriesBufLen:]
+	return entriesBuf, labelPairBuf, nil
+}
+
+// Entry represents Loki entry.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
+type Entry struct {
+	Timestamp          time.Time
+	Line               string
+	StructuredMetadata []LabelPair
+}
+
+func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	marshalTime(mm, 1, e.Timestamp)
+	mm.AppendString(2, e.Line)
+	for _, lp := range e.StructuredMetadata {
+		lp.marshalProtobuf(mm.AppendMessage(3))
+	}
+}
+
+func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
+	// message Entry {
+	//   Timestamp timestamp = 1;
+	//   string line = 2;
+	//   repeated LabelPair structuredMetadata = 3;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	labelPairBufLen := len(labelPairBuf)
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			data, ok := fc.MessageData()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
+			}
+			timestamp, err := unmarshalTime(data)
+			if err != nil {
+				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
+			}
+			e.Timestamp = timestamp
+		case 2:
+			line, ok := fc.String()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read Line")
+			}
+			e.Line = line
+		case 3:
+			data, ok := fc.MessageData()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
+			}
+			labelPairBuf = append(labelPairBuf, LabelPair{})
+			lp := &labelPairBuf[len(labelPairBuf)-1]
+			if err := lp.unmarshalProtobuf(data); err != nil {
+				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
+			}
+		}
+	}
+	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
+	return labelPairBuf, nil
+}
+
+// LabelPair represents Loki label pair.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
+type LabelPair struct {
+	Name  string
+	Value string
+}
+
+func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendString(1, lp.Name)
+	mm.AppendString(2, lp.Value)
+}
+
+func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
+	// message LabelPair {
+	//   string name = 1;
+	//   string value = 2;
+	// }
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			name, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot read name")
+			}
+			lp.Name = name
+		case 2:
+			value, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot unmarshal value")
+			}
+			lp.Value = value
+		}
+	}
+	return nil
+}
+
+func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
+	nsecs := timestamp.UnixNano()
+	ts := Timestamp{
+		Seconds: nsecs / 1e9,
+		Nanos:   int32(nsecs % 1e9),
+	}
+	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
+}
+
+func unmarshalTime(src []byte) (time.Time, error) {
+	var ts Timestamp
+	if err := ts.unmarshalProtobuf(src); err != nil {
+		return time.Time{}, err
+	}
+	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	return timestamp, nil
+}
+
+// Timestamp is protobuf well-known timestamp type.
+type Timestamp struct {
+	Seconds int64
+	Nanos   int32
+}
+
+func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendInt64(1, ts.Seconds)
+	mm.AppendInt32(2, ts.Nanos)
+}
+
+func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
+	// message Timestamp {
+	//   int64 seconds = 1;
+	//   int32 nanos = 2;
+	// }
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			seconds, ok := fc.Int64()
+			if !ok {
+				return fmt.Errorf("cannot read Seconds")
+			}
+			ts.Seconds = seconds
+		case 2:
+			nanos, ok := fc.Int32()
+			if !ok {
+				return fmt.Errorf("cannot read Nanos")
+			}
+			ts.Nanos = nanos
+		}
+	}
+	return nil
+}

app/vlinsert/loki/push_request.proto

@@ -1,38 +0,0 @@
-syntax = "proto3";
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package logproto;
-
-import "gogoproto/gogo.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
-
-message PushRequest {
-  repeated StreamAdapter streams = 1 [
-    (gogoproto.jsontag) = "streams",
-    (gogoproto.customtype) = "Stream"
-  ];
-}
-
-message StreamAdapter {
-  string labels = 1 [(gogoproto.jsontag) = "labels"];
-  repeated EntryAdapter entries = 2 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "entries"
-  ];
-  // hash contains the original hash of the stream.
-  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
-}
-
-message EntryAdapter {
-  google.protobuf.Timestamp timestamp = 1 [
-    (gogoproto.stdtime) = true,
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "ts"
-  ];
-  string line = 2 [(gogoproto.jsontag) = "line"];
-}

app/vlinsert/loki/timestamp.go

@@ -1,110 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"errors"
-	"strconv"
-	"time"
-
-	"github.com/gogo/protobuf/types"
-)
-
-const (
-	// Seconds field of the earliest valid Timestamp.
-	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	minValidSeconds = -62135596800
-	// Seconds field just after the latest valid Timestamp.
-	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	maxValidSeconds = 253402300800
-)
-
-// validateTimestamp determines whether a Timestamp is valid.
-// A valid timestamp represents a time in the range
-// [0001-01-01, 10000-01-01) and has a Nanos field
-// in the range [0, 1e9).
-//
-// If the Timestamp is valid, validateTimestamp returns nil.
-// Otherwise, it returns an error that describes
-// the problem.
-//
-// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
-func validateTimestamp(ts *types.Timestamp) error {
-	if ts == nil {
-		return errors.New("timestamp: nil Timestamp")
-	}
-	if ts.Seconds < minValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
-	}
-	if ts.Seconds >= maxValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
-	}
-	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
-		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
-	}
-	return nil
-}
-
-// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
-// but avoids the escape incurred by using fmt.Sprintf, eliminating
-// unnecessary heap allocations.
-func formatTimestamp(ts *types.Timestamp) string {
-	if ts == nil {
-		return "nil"
-	}
-
-	seconds := strconv.FormatInt(ts.Seconds, 10)
-	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
-	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
-}
-
-func sizeOfStdTime(t time.Time) int {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0
-	}
-	return ts.Size()
-}
-
-func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0, err
-	}
-	return ts.MarshalTo(data)
-}
-
-func stdTimeUnmarshal(t *time.Time, data []byte) error {
-	ts := &types.Timestamp{}
-	if err := ts.Unmarshal(data); err != nil {
-		return err
-	}
-	tt, err := timestampFromProto(ts)
-	if err != nil {
-		return err
-	}
-	*t = tt
-	return nil
-}
-
-func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
-	// Don't return the zero value on error, because corresponds to a valid
-	// timestamp. Instead return whatever time.Unix gives us.
-	var t time.Time
-	if ts == nil {
-		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
-	} else {
-		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	}
-	return t, validateTimestamp(ts)
-}
-
-func timestampProto(t time.Time) (types.Timestamp, error) {
-	ts := types.Timestamp{
-		Seconds: t.Unix(),
-		Nanos:   int32(t.Nanosecond()),
-	}
-	return ts, validateTimestamp(&ts)
-}

app/vlinsert/loki/types.go

@@ -1,481 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"fmt"
-	"io"
-	"time"
-)
-
-// Stream contains a unique labels set as a string and a set of entries for it.
-// We are not using the proto generated version but this custom one so that we
-// can improve serialization see benchmark.
-type Stream struct {
-	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
-	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
-	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
-}
-
-// Entry is a log entry with a timestamp.
-type Entry struct {
-	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
-	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Stream) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Hash != 0 {
-		i = encodeVarintPush(dAtA, i, m.Hash)
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Entries) > 0 {
-		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintPush(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x12
-		}
-	}
-	if len(m.Labels) > 0 {
-		i -= len(m.Labels)
-		copy(dAtA[i:], m.Labels)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Entry) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if len(m.Line) > 0 {
-		i -= len(m.Line)
-		copy(dAtA[i:], m.Line)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
-		i--
-		dAtA[i] = 0x12
-	}
-	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
-	if err7 != nil {
-		return 0, err7
-	}
-	i -= n7
-	i = encodeVarintPush(dAtA, i, uint64(n7))
-	i--
-	dAtA[i] = 0xa
-	return len(dAtA) - i, nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Stream) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Labels = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Entries = append(m.Entries, Entry{})
-			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
-			}
-			m.Hash = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Hash |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Entry) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Line = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Size returns the size of the serialized Stream.
-func (m *Stream) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Labels)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	if len(m.Entries) > 0 {
-		for _, e := range m.Entries {
-			l = e.Size()
-			n += 1 + l + sovPush(uint64(l))
-		}
-	}
-	if m.Hash != 0 {
-		n += 1 + sovPush(m.Hash)
-	}
-	return n
-}
-
-// Size returns the size of the serialized Entry
-func (m *Entry) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = sizeOfStdTime(m.Timestamp)
-	n += 1 + l + sovPush(uint64(l))
-	l = len(m.Line)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	return n
-}
-
-// Equal returns true if the two Streams are equal.
-func (m *Stream) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Stream)
-	if !ok {
-		that2, ok := that.(Stream)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if m.Labels != that1.Labels {
-		return false
-	}
-	if len(m.Entries) != len(that1.Entries) {
-		return false
-	}
-	for i := range m.Entries {
-		if !m.Entries[i].Equal(that1.Entries[i]) {
-			return false
-		}
-	}
-	return m.Hash == that1.Hash
-}
-
-// Equal returns true if the two Entries are equal.
-func (m *Entry) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Entry)
-	if !ok {
-		that2, ok := that.(Entry)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if !m.Timestamp.Equal(that1.Timestamp) {
-		return false
-	}
-	if m.Line != that1.Line {
-		return false
-	}
-	return true
-}

app/vlinsert/main.go

@@ -7,14 +7,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
 )
 
 // Init initializes vlinsert
 func Init() {
+	syslog.MustInit()
 }
 
 // Stop stops vlinsert
 func Stop() {
+	syslog.MustStop()
 }
 
 // RequestHandler handles insert requests for VictoriaLogs
@@ -28,7 +31,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path = strings.ReplaceAll(path, "//", "/")
 
 	if path == "/jsonline" {
-		return jsonline.RequestHandler(w, r)
+		jsonline.RequestHandler(w, r)
+		return true
 	}
 	switch {
 	case strings.HasPrefix(path, "/elasticsearch/"):

app/vlinsert/syslog/syslog.go

@@ -0,0 +1,531 @@
+package syslog
+
+import (
+	"bufio"
+	"crypto/tls"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/klauspost/compress/gzip"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
+		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+
+		"The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
+		"Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
+		"The provided key file is automatically re-read every second, so it can be dynamically updated. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+
+		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+	tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"Supported values: TLS10, TLS11, TLS12, TLS13. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
+
+	compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+
+		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
+	compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+
+		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
+
+	useLocalTimestampTCP = flagutil.NewArrayBool("syslog.useLocalTimestamp.tcp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
+		"at the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
+	useLocalTimestampUDP = flagutil.NewArrayBool("syslog.useLocalTimestamp.udp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
+		"at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
+)
+
+// MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports
+//
+// This function must be called after flag.Parse().
+//
+// MustStop() must be called in order to free up resources occupied by the initialized syslog parser.
+func MustInit() {
+	if workersStopCh != nil {
+		logger.Panicf("BUG: MustInit() called twice without MustStop() call")
+	}
+	workersStopCh = make(chan struct{})
+
+	for argIdx, addr := range *listenAddrTCP {
+		workersWG.Add(1)
+		go func(addr string, argIdx int) {
+			runTCPListener(addr, argIdx)
+			workersWG.Done()
+		}(addr, argIdx)
+	}
+
+	for argIdx, addr := range *listenAddrUDP {
+		workersWG.Add(1)
+		go func(addr string, argIdx int) {
+			runUDPListener(addr, argIdx)
+			workersWG.Done()
+		}(addr, argIdx)
+	}
+
+	currentYear := time.Now().Year()
+	globalCurrentYear.Store(int64(currentYear))
+	workersWG.Add(1)
+	go func() {
+		ticker := time.NewTicker(time.Minute)
+		for {
+			select {
+			case <-workersStopCh:
+				ticker.Stop()
+				workersWG.Done()
+				return
+			case <-ticker.C:
+				currentYear := time.Now().Year()
+				globalCurrentYear.Store(int64(currentYear))
+			}
+		}
+	}()
+
+	if *syslogTimezone != "" {
+		tz, err := time.LoadLocation(*syslogTimezone)
+		if err != nil {
+			logger.Fatalf("cannot parse -syslog.timezone=%q: %s", *syslogTimezone, err)
+		}
+		globalTimezone = tz
+	} else {
+		globalTimezone = time.Local
+	}
+}
+
+var (
+	globalCurrentYear atomic.Int64
+	globalTimezone    *time.Location
+)
+
+var (
+	workersWG     sync.WaitGroup
+	workersStopCh chan struct{}
+)
+
+// MustStop stops syslog parser initialized via MustInit()
+func MustStop() {
+	close(workersStopCh)
+	workersWG.Wait()
+	workersStopCh = nil
+}
+
+func runUDPListener(addr string, argIdx int) {
+	ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr)
+	if err != nil {
+		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
+	}
+
+	tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
+	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
+	}
+
+	compressMethod := compressMethodUDP.GetOptionalArg(argIdx)
+	checkCompressMethod(compressMethod, addr, "udp")
+
+	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
+
+	doneCh := make(chan struct{})
+	go func() {
+		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
+		close(doneCh)
+	}()
+
+	logger.Infof("started accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
+	<-workersStopCh
+	if err := ln.Close(); err != nil {
+		logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err)
+	}
+	<-doneCh
+	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
+}
+
+func runTCPListener(addr string, argIdx int) {
+	var tlsConfig *tls.Config
+	if tlsEnable.GetOptionalArg(argIdx) {
+		certFile := tlsCertFile.GetOptionalArg(argIdx)
+		keyFile := tlsKeyFile.GetOptionalArg(argIdx)
+		tc, err := netutil.GetServerTLSConfig(certFile, keyFile, *tlsMinVersion, *tlsCipherSuites)
+		if err != nil {
+			logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s",
+				certFile, keyFile, *tlsMinVersion, *tlsCipherSuites, err)
+		}
+		tlsConfig = tc
+	}
+	ln, err := netutil.NewTCPListener("syslog", addr, false, tlsConfig)
+	if err != nil {
+		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
+	}
+
+	tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
+	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
+	}
+
+	compressMethod := compressMethodTCP.GetOptionalArg(argIdx)
+	checkCompressMethod(compressMethod, addr, "tcp")
+
+	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
+
+	doneCh := make(chan struct{})
+	go func() {
+		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
+		close(doneCh)
+	}()
+
+	logger.Infof("started accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
+	<-workersStopCh
+	if err := ln.Close(); err != nil {
+		logger.Fatalf("syslog: cannot close TCP listener at %s: %s", addr, err)
+	}
+	<-doneCh
+	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
+}
+
+func checkCompressMethod(compressMethod, addr, protocol string) {
+	switch compressMethod {
+	case "", "none", "gzip", "deflate":
+		return
+	default:
+		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
+	}
+}
+
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+	gomaxprocs := cgroup.AvailableCPUs()
+	var wg sync.WaitGroup
+	localAddr := ln.LocalAddr()
+	for i := 0; i < gomaxprocs; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			var bb bytesutil.ByteBuffer
+			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
+			for {
+				bb.Reset()
+				bb.B = bb.B[:cap(bb.B)]
+				n, remoteAddr, err := ln.ReadFrom(bb.B)
+				if err != nil {
+					udpErrorsTotal.Inc()
+					var ne net.Error
+					if errors.As(err, &ne) {
+						if ne.Temporary() {
+							logger.Errorf("syslog: temporary error when listening for UDP at %q: %s", localAddr, err)
+							time.Sleep(time.Second)
+							continue
+						}
+						if strings.Contains(err.Error(), "use of closed network connection") {
+							break
+						}
+					}
+					logger.Errorf("syslog: cannot read UDP data from %s at %s: %s", remoteAddr, localAddr, err)
+					continue
+				}
+				bb.B = bb.B[:n]
+				udpRequestsTotal.Inc()
+				if err := processStream(bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
+					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+	var cm ingestserver.ConnsMap
+	cm.Init("syslog")
+
+	var wg sync.WaitGroup
+	addr := ln.Addr()
+	for {
+		c, err := ln.Accept()
+		if err != nil {
+			var ne net.Error
+			if errors.As(err, &ne) {
+				if ne.Temporary() {
+					logger.Errorf("syslog: temporary error when listening for TCP addr %q: %s", addr, err)
+					time.Sleep(time.Second)
+					continue
+				}
+				if strings.Contains(err.Error(), "use of closed network connection") {
+					break
+				}
+				logger.Fatalf("syslog: unrecoverable error when accepting TCP connections at %q: %s", addr, err)
+			}
+			logger.Fatalf("syslog: unexpected error when accepting TCP connections at %q: %s", addr, err)
+		}
+		if !cm.Add(c) {
+			_ = c.Close()
+			break
+		}
+
+		wg.Add(1)
+		go func() {
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
+				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
+			}
+
+			cm.Delete(c)
+			_ = c.Close()
+			wg.Done()
+		}()
+	}
+
+	cm.CloseAll(0)
+	wg.Wait()
+}
+
+// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
+func processStream(r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
+	if err := vlstorage.CanWriteData(); err != nil {
+		return err
+	}
+
+	lmp := cp.NewLogMessageProcessor()
+	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
+	lmp.MustClose()
+
+	return err
+}
+
+func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	switch compressMethod {
+	case "", "none":
+	case "gzip":
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read gzipped data: %w", err)
+		}
+		r = zr
+	case "deflate":
+		zr, err := common.GetZlibReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read deflated data: %w", err)
+		}
+		r = zr
+	default:
+		logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
+	}
+
+	err := processUncompressedStream(r, useLocalTimestamp, lmp)
+
+	switch compressMethod {
+	case "gzip":
+		zr := r.(*gzip.Reader)
+		common.PutGzipReader(zr)
+	case "deflate":
+		zr := r.(io.ReadCloser)
+		common.PutZlibReader(zr)
+	}
+
+	return err
+}
+
+func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	wcr := writeconcurrencylimiter.GetReader(r)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	slr := getSyslogLineReader(wcr)
+	defer putSyslogLineReader(slr)
+
+	n := 0
+	for {
+		ok := slr.nextLine()
+		wcr.DecConcurrency()
+		if !ok {
+			break
+		}
+
+		currentYear := int(globalCurrentYear.Load())
+		err := processLine(slr.line, currentYear, globalTimezone, useLocalTimestamp, lmp)
+		if err != nil {
+			errorsTotal.Inc()
+			return fmt.Errorf("cannot read line #%d: %s", n, err)
+		}
+		n++
+		rowsIngestedTotal.Inc()
+	}
+	return slr.Error()
+}
+
+type syslogLineReader struct {
+	line []byte
+
+	br  *bufio.Reader
+	err error
+}
+
+func (slr *syslogLineReader) reset(r io.Reader) {
+	slr.line = slr.line[:0]
+	slr.br.Reset(r)
+	slr.err = nil
+}
+
+// Error returns the last error occurred in slr.
+func (slr *syslogLineReader) Error() error {
+	if slr.err == nil || slr.err == io.EOF {
+		return nil
+	}
+	return slr.err
+}
+
+// nextLine reads the next syslog line from slr and stores it at slr.line.
+//
+// false is returned if the next line cannot be read. Error() must be called in this case
+// in order to verify whether there is an error or just slr stream has been finished.
+func (slr *syslogLineReader) nextLine() bool {
+	if slr.err != nil {
+		return false
+	}
+
+again:
+	prefix, err := slr.br.ReadSlice(' ')
+	if err != nil {
+		if err != io.EOF {
+			slr.err = fmt.Errorf("cannot read message frame prefix: %w", err)
+			return false
+		}
+		if len(prefix) == 0 {
+			slr.err = err
+			return false
+		}
+	}
+	// skip empty lines
+	for len(prefix) > 0 && prefix[0] == '\n' {
+		prefix = prefix[1:]
+	}
+	if len(prefix) == 0 {
+		// An empty prefix or a prefix with empty lines - try reading yet another prefix.
+		goto again
+	}
+
+	if prefix[0] >= '0' && prefix[0] <= '9' {
+		// This is octet-counting method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#msgxfer
+		msgLenStr := bytesutil.ToUnsafeString(prefix[:len(prefix)-1])
+		msgLen, err := strconv.ParseUint(msgLenStr, 10, 64)
+		if err != nil {
+			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
+			return false
+		}
+		if maxMsgLen := insertutils.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
+			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
+			return false
+		}
+		slr.line = slicesutil.SetLength(slr.line, int(msgLen))
+		if _, err := io.ReadFull(slr.br, slr.line); err != nil {
+			slr.err = fmt.Errorf("cannot read message with size %d bytes: %w", msgLen, err)
+			return false
+		}
+		return true
+	}
+
+	// This is octet-stuffing method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#octet-stuffing-legacy
+	slr.line = append(slr.line[:0], prefix...)
+	for {
+		line, err := slr.br.ReadSlice('\n')
+		if err == nil {
+			slr.line = append(slr.line, line[:len(line)-1]...)
+			return true
+		}
+		if err == io.EOF {
+			slr.line = append(slr.line, line...)
+			return true
+		}
+		if err == bufio.ErrBufferFull {
+			slr.line = append(slr.line, line...)
+			continue
+		}
+		slr.err = fmt.Errorf("cannot read message in octet-stuffing method: %w", err)
+		return false
+	}
+}
+
+func getSyslogLineReader(r io.Reader) *syslogLineReader {
+	v := syslogLineReaderPool.Get()
+	if v == nil {
+		br := bufio.NewReaderSize(r, 64*1024)
+		return &syslogLineReader{
+			br: br,
+		}
+	}
+	slr := v.(*syslogLineReader)
+	slr.reset(r)
+	return slr
+}
+
+func putSyslogLineReader(slr *syslogLineReader) {
+	syslogLineReaderPool.Put(slr)
+}
+
+var syslogLineReaderPool sync.Pool
+
+func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	p := logstorage.GetSyslogParser(currentYear, timezone)
+	lineStr := bytesutil.ToUnsafeString(line)
+	p.Parse(lineStr)
+
+	var ts int64
+	if useLocalTimestamp {
+		ts = time.Now().UnixNano()
+	} else {
+		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
+		if err != nil {
+			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
+		}
+		ts = nsecs
+	}
+	logstorage.RenameField(p.Fields, "message", "_msg")
+	lmp.AddRow(ts, p.Fields)
+	logstorage.PutSyslogParser(p)
+
+	return nil
+}
+
+var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
+
+	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
+
+	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
+	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
+)

app/vlinsert/syslog/syslog_test.go

@@ -0,0 +1,130 @@
+package syslog
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+)
+
+func TestSyslogLineReader_Success(t *testing.T) {
+	f := func(data string, linesExpected []string) {
+		t.Helper()
+
+		r := bytes.NewBufferString(data)
+		slr := getSyslogLineReader(r)
+		defer putSyslogLineReader(slr)
+
+		var lines []string
+		for slr.nextLine() {
+			lines = append(lines, string(slr.line))
+		}
+		if err := slr.Error(); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if !reflect.DeepEqual(lines, linesExpected) {
+			t.Fatalf("unexpected lines read;\ngot\n%q\nwant\n%q", lines, linesExpected)
+		}
+	}
+
+	f("", nil)
+	f("\n", nil)
+	f("\n\n\n", nil)
+
+	f("foobar", []string{"foobar"})
+	f("foobar\n", []string{"foobar\n"})
+	f("\n\nfoo\n\nbar\n\n", []string{"foo\n\nbar\n\n"})
+
+	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...`, []string{"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches..."})
+
+	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
+
+48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
+
+`, []string{
+		"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...",
+		"<165>Jun  4 12:08:33 abcd systemd[345]: abc defg",
+		`<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`,
+	})
+}
+
+func TestSyslogLineReader_Failure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		r := bytes.NewBufferString(data)
+		slr := getSyslogLineReader(r)
+		defer putSyslogLineReader(slr)
+
+		if slr.nextLine() {
+			t.Fatalf("expecting failure to read the first line")
+		}
+		if err := slr.Error(); err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// invalid format for message size
+	f("12foo bar")
+
+	// too big message size
+	f("123 aa")
+	f("1233423432 abc")
+}
+
+func TestProcessStreamInternal_Success(t *testing.T) {
+	f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		MustInit()
+		defer MustStop()
+
+		globalTimezone = time.UTC
+		globalCurrentYear.Store(int64(currentYear))
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal(r, "", false, tlp); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	data := `Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
+
+48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
+`
+	currentYear := 2023
+	rowsExpected := 3
+	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
+	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
+}
+
+func TestProcessStreamInternal_Failure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		MustInit()
+		defer MustStop()
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal(r, "", false, tlp); err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// invalid format for message size
+	f("12foo bar")
+
+	// too big message size
+	f("123 foo")
+	f("123456789 bar")
+}

app/vlogsgenerator/README.md

@@ -94,13 +94,13 @@ These flags can be inspected by running `vlogsgenerator -help`. Below are the mo
 
 * `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
 * `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to generate.
+* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to generate.
 * `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
 
 The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
 
 For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
-[log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields), where every logs stream contains `10_000` log entries,
+[log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields), where every logs stream contains `10_000` log entries,
 and writes them to `http://localhost:9428/insert/jsonline`:
 
 ```
@@ -113,7 +113,7 @@ bin/vlogsgenerator \
 
 ### Churn rate
 
-It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
+It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
 by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
 logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
 but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:

app/vlogsgenerator/main.go

@@ -27,32 +27,32 @@ var (
 
 	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
 	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
-	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields")
+	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
 	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
 		"during data generation")
 	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
 	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 
 	statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
 )
@@ -230,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 	for i := 0; i < activeStreams; i++ {
 		ip := toIPv4(rand.Uint32())
 		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
+		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
 			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
 		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
 		for j := 0; j < *constFieldsPerLog; j++ {

app/vlselect/logsql/field_names_response.qtpl

@@ -1,17 +0,0 @@
-{% stripspace %}
-
-// FieldNamesResponse formats /select/logsql/field_names response
-{% func FieldNamesResponse(names []string) %}
-{
-	"names":[
-		{% if len(names) > 0 %}
-			{%q= names[0] %}
-			{% for _, v := range names[1:] %}
-				,{%q= v %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/field_names_response.qtpl.go

@@ -1,69 +0,0 @@
-// Code generated by qtc from "field_names_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// FieldNamesResponse formats /select/logsql/field_names response
-
-//line app/vlselect/logsql/field_names_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/field_names_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/field_names_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/field_names_response.qtpl:4
-func StreamFieldNamesResponse(qw422016 *qt422016.Writer, names []string) {
-//line app/vlselect/logsql/field_names_response.qtpl:4
-	qw422016.N().S(`{"names":[`)
-//line app/vlselect/logsql/field_names_response.qtpl:7
-	if len(names) > 0 {
-//line app/vlselect/logsql/field_names_response.qtpl:8
-		qw422016.N().Q(names[0])
-//line app/vlselect/logsql/field_names_response.qtpl:9
-		for _, v := range names[1:] {
-//line app/vlselect/logsql/field_names_response.qtpl:9
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/field_names_response.qtpl:10
-			qw422016.N().Q(v)
-//line app/vlselect/logsql/field_names_response.qtpl:11
-		}
-//line app/vlselect/logsql/field_names_response.qtpl:12
-	}
-//line app/vlselect/logsql/field_names_response.qtpl:12
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/field_names_response.qtpl:15
-}
-
-//line app/vlselect/logsql/field_names_response.qtpl:15
-func WriteFieldNamesResponse(qq422016 qtio422016.Writer, names []string) {
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	StreamFieldNamesResponse(qw422016, names)
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/field_names_response.qtpl:15
-}
-
-//line app/vlselect/logsql/field_names_response.qtpl:15
-func FieldNamesResponse(names []string) string {
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	WriteFieldNamesResponse(qb422016, names)
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/field_names_response.qtpl:15
-	return qs422016
-//line app/vlselect/logsql/field_names_response.qtpl:15
-}

app/vlselect/logsql/field_values_response.qtpl

@@ -1,17 +0,0 @@
-{% stripspace %}
-
-// FieldValuesResponse formats /select/logsql/field_values response
-{% func FieldValuesResponse(values []string) %}
-{
-	"values":[
-		{% if len(values) > 0 %}
-			{%q= values[0] %}
-			{% for _, v := range values[1:] %}
-				,{%q= v %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/field_values_response.qtpl.go

@@ -1,69 +0,0 @@
-// Code generated by qtc from "field_values_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// FieldValuesResponse formats /select/logsql/field_values response
-
-//line app/vlselect/logsql/field_values_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/field_values_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/field_values_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/field_values_response.qtpl:4
-func StreamFieldValuesResponse(qw422016 *qt422016.Writer, values []string) {
-//line app/vlselect/logsql/field_values_response.qtpl:4
-	qw422016.N().S(`{"values":[`)
-//line app/vlselect/logsql/field_values_response.qtpl:7
-	if len(values) > 0 {
-//line app/vlselect/logsql/field_values_response.qtpl:8
-		qw422016.N().Q(values[0])
-//line app/vlselect/logsql/field_values_response.qtpl:9
-		for _, v := range values[1:] {
-//line app/vlselect/logsql/field_values_response.qtpl:9
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/field_values_response.qtpl:10
-			qw422016.N().Q(v)
-//line app/vlselect/logsql/field_values_response.qtpl:11
-		}
-//line app/vlselect/logsql/field_values_response.qtpl:12
-	}
-//line app/vlselect/logsql/field_values_response.qtpl:12
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/field_values_response.qtpl:15
-}
-
-//line app/vlselect/logsql/field_values_response.qtpl:15
-func WriteFieldValuesResponse(qq422016 qtio422016.Writer, values []string) {
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	StreamFieldValuesResponse(qw422016, values)
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/field_values_response.qtpl:15
-}
-
-//line app/vlselect/logsql/field_values_response.qtpl:15
-func FieldValuesResponse(values []string) string {
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	WriteFieldValuesResponse(qb422016, values)
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/field_values_response.qtpl:15
-	return qs422016
-//line app/vlselect/logsql/field_values_response.qtpl:15
-}

app/vlselect/logsql/hits_response.qtpl

@@ -6,8 +6,8 @@
 
 {% stripspace %}
 
-// LabelsForHits formats labels for /select/logsql/hits response
-{% func LabelsForHits(columns []logstorage.BlockColumn, rowIdx int) %}
+// FieldsForHits formats labels for /select/logsql/hits response
+{% func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) %}
 {
 	{% if len(columns) > 0 %}
 		{%q= columns[0].Name %}:{%q= columns[0].Values[rowIdx] %}
@@ -44,7 +44,7 @@
 		hs := m[k]
 		hs.sort()
 		timestamps := hs.timestamps
-		values := hs.values
+		hits := hs.hits
 	%}
 	"fields":{%s= k %},
 	"timestamps":[
@@ -56,13 +56,14 @@
 		{% endif %}
 	],
 	"values":[
-		{% if len(values) > 0 %}
-			{%s= values[0] %}
-			{% for _, v := range values[1:] %}
-				,{%s= v %}
+		{% if len(hits) > 0 %}
+			{%dul= hits[0] %}
+			{% for _, v := range hits[1:] %}
+				,{%dul= v %}
 			{% endfor %}
 		{% endif %}
-	]
+	],
+	"total":{%dul= hs.hitsTotal %}
 }
 {% endfunc %}
 

app/vlselect/logsql/hits_response.qtpl.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-// LabelsForHits formats labels for /select/logsql/hits response
+// FieldsForHits formats labels for /select/logsql/hits response
 
 //line app/vlselect/logsql/hits_response.qtpl:10
 import (
@@ -27,7 +27,7 @@ var (
 )
 
 //line app/vlselect/logsql/hits_response.qtpl:10
-func StreamLabelsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+func StreamFieldsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
 //line app/vlselect/logsql/hits_response.qtpl:10
 	qw422016.N().S(`{`)
 //line app/vlselect/logsql/hits_response.qtpl:12
@@ -58,22 +58,22 @@ func StreamLabelsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockCo
 }
 
 //line app/vlselect/logsql/hits_response.qtpl:19
-func WriteLabelsForHits(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+func WriteFieldsForHits(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
 //line app/vlselect/logsql/hits_response.qtpl:19
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line app/vlselect/logsql/hits_response.qtpl:19
-	StreamLabelsForHits(qw422016, columns, rowIdx)
+	StreamFieldsForHits(qw422016, columns, rowIdx)
 //line app/vlselect/logsql/hits_response.qtpl:19
 	qt422016.ReleaseWriter(qw422016)
 //line app/vlselect/logsql/hits_response.qtpl:19
 }
 
 //line app/vlselect/logsql/hits_response.qtpl:19
-func LabelsForHits(columns []logstorage.BlockColumn, rowIdx int) string {
+func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) string {
 //line app/vlselect/logsql/hits_response.qtpl:19
 	qb422016 := qt422016.AcquireByteBuffer()
 //line app/vlselect/logsql/hits_response.qtpl:19
-	WriteLabelsForHits(qb422016, columns, rowIdx)
+	WriteFieldsForHits(qb422016, columns, rowIdx)
 //line app/vlselect/logsql/hits_response.qtpl:19
 	qs422016 := string(qb422016.B)
 //line app/vlselect/logsql/hits_response.qtpl:19
@@ -149,7 +149,7 @@ func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k
 	hs := m[k]
 	hs.sort()
 	timestamps := hs.timestamps
-	values := hs.values
+	hits := hs.hits
 
 //line app/vlselect/logsql/hits_response.qtpl:48
 	qw422016.N().S(`"fields":`)
@@ -174,46 +174,50 @@ func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k
 //line app/vlselect/logsql/hits_response.qtpl:56
 	qw422016.N().S(`],"values":[`)
 //line app/vlselect/logsql/hits_response.qtpl:59
-	if len(values) > 0 {
+	if len(hits) > 0 {
 //line app/vlselect/logsql/hits_response.qtpl:60
-		qw422016.N().S(values[0])
+		qw422016.N().DUL(hits[0])
 //line app/vlselect/logsql/hits_response.qtpl:61
-		for _, v := range values[1:] {
+		for _, v := range hits[1:] {
 //line app/vlselect/logsql/hits_response.qtpl:61
 			qw422016.N().S(`,`)
 //line app/vlselect/logsql/hits_response.qtpl:62
-			qw422016.N().S(v)
+			qw422016.N().DUL(v)
 //line app/vlselect/logsql/hits_response.qtpl:63
 		}
 //line app/vlselect/logsql/hits_response.qtpl:64
 	}
 //line app/vlselect/logsql/hits_response.qtpl:64
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/hits_response.qtpl:67
+	qw422016.N().S(`],"total":`)
+//line app/vlselect/logsql/hits_response.qtpl:66
+	qw422016.N().DUL(hs.hitsTotal)
+//line app/vlselect/logsql/hits_response.qtpl:66
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/hits_response.qtpl:68
 }
 
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 func writehitsSeriesLine(qq422016 qtio422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 }
 
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 func hitsSeriesLine(m map[string]*hitsSeries, k string) string {
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	writehitsSeriesLine(qb422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:67
+//line app/vlselect/logsql/hits_response.qtpl:68
 }

app/vlselect/logsql/logsql.go

@@ -5,16 +5,19 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"slices"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -57,9 +60,20 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	// Obtain field entries
 	fields := r.Form["field"]
 
-	// Prepare the query
-	q.AddCountByTimePipe(int64(step), int64(offset), fields)
+	// Obtain limit on the number of top fields entries.
+	fieldsLimit, err := httputils.GetInt(r, "fields_limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if fieldsLimit < 0 {
+		fieldsLimit = 0
+	}
+
+	// Prepare the query for hits count.
 	q.Optimize()
+	q.DropAllPipes()
+	q.AddCountByTimePipe(int64(step), int64(offset), fields)
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
@@ -76,9 +90,13 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		for i := range timestamps {
 			timestampStr := strings.Clone(timestampValues[i])
 			hitsStr := strings.Clone(hitsValues[i])
+			hits, err := strconv.ParseUint(hitsStr, 10, 64)
+			if err != nil {
+				logger.Panicf("BUG: cannot parse hitsStr=%q: %s", hitsStr, err)
+			}
 
 			bb.Reset()
-			WriteLabelsForHits(bb, columns, i)
+			WriteFieldsForHits(bb, columns, i)
 
 			mLock.Lock()
 			hs, ok := m[string(bb.B)]
@@ -88,7 +106,8 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 				m[k] = hs
 			}
 			hs.timestamps = append(hs.timestamps, timestampStr)
-			hs.values = append(hs.values, hitsStr)
+			hs.hits = append(hs.hits, hits)
+			hs.hitsTotal += hits
 			mLock.Unlock()
 		}
 		blockResultPool.Put(bb)
@@ -100,14 +119,59 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	m = getTopHitsSeries(m, fieldsLimit)
+
 	// Write response
 	w.Header().Set("Content-Type", "application/json")
 	WriteHitsSeries(w, m)
 }
 
+func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
+	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
+		return m
+	}
+
+	type fieldsHits struct {
+		fieldsStr string
+		hs        *hitsSeries
+	}
+	a := make([]fieldsHits, 0, len(m))
+	for fieldsStr, hs := range m {
+		a = append(a, fieldsHits{
+			fieldsStr: fieldsStr,
+			hs:        hs,
+		})
+	}
+	sort.Slice(a, func(i, j int) bool {
+		return a[i].hs.hitsTotal > a[j].hs.hitsTotal
+	})
+
+	hitsOther := make(map[string]uint64)
+	for _, x := range a[fieldsLimit:] {
+		for i, timestampStr := range x.hs.timestamps {
+			hitsOther[timestampStr] += x.hs.hits[i]
+		}
+	}
+	var hsOther hitsSeries
+	for timestampStr, hits := range hitsOther {
+		hsOther.timestamps = append(hsOther.timestamps, timestampStr)
+		hsOther.hits = append(hsOther.hits, hits)
+		hsOther.hitsTotal += hits
+	}
+
+	mNew := make(map[string]*hitsSeries, fieldsLimit+1)
+	for _, x := range a[:fieldsLimit] {
+		mNew[x.fieldsStr] = x.hs
+	}
+	mNew["{}"] = &hsOther
+
+	return mNew
+}
+
 type hitsSeries struct {
+	hitsTotal  uint64
 	timestamps []string
-	values     []string
+	hits       []uint64
 }
 
 func (hs *hitsSeries) sort() {
@@ -120,7 +184,7 @@ func (hs *hitsSeries) Len() int {
 
 func (hs *hitsSeries) Swap(i, j int) {
 	hs.timestamps[i], hs.timestamps[j] = hs.timestamps[j], hs.timestamps[i]
-	hs.values[i], hs.values[j] = hs.values[j], hs.values[i]
+	hs.hits[i], hs.hits[j] = hs.hits[j], hs.hits[i]
 }
 
 func (hs *hitsSeries) Less(i, j int) bool {
@@ -145,11 +209,9 @@ func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *htt
 		return
 	}
 
-	slices.Sort(fieldNames)
-
 	// Write results
 	w.Header().Set("Content-Type", "application/json")
-	WriteFieldNamesResponse(w, fieldNames)
+	WriteValuesWithHitsJSON(w, fieldNames)
 }
 
 // ProcessFieldValuesRequest handles /select/logsql/field_values request.
@@ -163,9 +225,9 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
 	}
 
 	// Parse fieldName query arg
-	fieldName := r.FormValue("field_name")
+	fieldName := r.FormValue("field")
 	if fieldName == "" {
-		httpserver.Errorf(w, r, "missing 'field_name' query arg")
+		httpserver.Errorf(w, r, "missing 'field' query arg")
 		return
 	}
 
@@ -187,16 +249,315 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
 		return
 	}
 
-	if limit == 0 || len(values) < limit {
-		// Sort values only if their number is below the limit.
-		// Otherwise there is little sense in sorting, since the query may return
-		// different subset of values on every execution.
-		slices.Sort(values)
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, values)
+}
+
+// ProcessStreamFieldNamesRequest processes /select/logsql/stream_field_names request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-names
+func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Obtain stream field names for the given query
+	q.Optimize()
+	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream field names: %s", err)
 	}
 
 	// Write results
 	w.Header().Set("Content-Type", "application/json")
-	WriteFieldValuesResponse(w, values)
+	WriteValuesWithHitsJSON(w, names)
+}
+
+// ProcessStreamFieldValuesRequest processes /select/logsql/stream_field_values request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-values
+func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse fieldName query arg
+	fieldName := r.FormValue("field")
+	if fieldName == "" {
+		httpserver.Errorf(w, r, "missing 'field' query arg")
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain stream field values for the given query and the given fieldName
+	q.Optimize()
+	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream field values: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, values)
+}
+
+// ProcessStreamIDsRequest processes /select/logsql/stream_ids request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream_ids
+func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain streamIDs for the given query
+	q.Optimize()
+	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream_ids: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, streamIDs)
+}
+
+// ProcessStreamsRequest processes /select/logsql/streams request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-streams
+func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain streams for the given query
+	q.Optimize()
+	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain streams: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, streams)
+}
+
+// ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
+func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	liveTailRequests.Inc()
+	defer liveTailRequests.Dec()
+
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if !q.CanLiveTail() {
+		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
+	}
+	q.Optimize()
+
+	refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
+
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	tp := newTailProcessor(cancel)
+
+	ticker := time.NewTicker(refreshInterval)
+	defer ticker.Stop()
+
+	end := time.Now().UnixNano()
+	doneCh := ctxWithCancel.Done()
+	flusher, ok := w.(http.Flusher)
+	if !ok {
+		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
+	}
+	for {
+		start := end - tailOffsetNsecs
+		end = time.Now().UnixNano()
+
+		qCopy := q.Clone()
+		qCopy.AddTimeFilter(start, end)
+		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
+			httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
+			return
+		}
+		resultRows, err := tp.getTailRows()
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot get tail results for query [%q]: %s", q, err)
+			return
+		}
+		if len(resultRows) > 0 {
+			WriteJSONRows(w, resultRows)
+			flusher.Flush()
+		}
+
+		select {
+		case <-doneCh:
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+var liveTailRequests = metrics.NewCounter(`vl_live_tailing_requests`)
+
+const tailOffsetNsecs = 5e9
+
+type logRow struct {
+	timestamp int64
+	fields    []logstorage.Field
+}
+
+func sortLogRows(rows []logRow) {
+	sort.SliceStable(rows, func(i, j int) bool {
+		return rows[i].timestamp < rows[j].timestamp
+	})
+}
+
+type tailProcessor struct {
+	cancel func()
+
+	mu sync.Mutex
+
+	perStreamRows  map[string][]logRow
+	lastTimestamps map[string]int64
+
+	err error
+}
+
+func newTailProcessor(cancel func()) *tailProcessor {
+	return &tailProcessor{
+		cancel: cancel,
+
+		perStreamRows:  make(map[string][]logRow),
+		lastTimestamps: make(map[string]int64),
+	}
+}
+
+func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	if len(timestamps) == 0 {
+		return
+	}
+
+	tp.mu.Lock()
+	defer tp.mu.Unlock()
+
+	if tp.err != nil {
+		return
+	}
+
+	// Make sure columns contain _time field, since it is needed for proper tail work.
+	hasTime := false
+	for _, c := range columns {
+		if c.Name == "_time" {
+			hasTime = true
+			break
+		}
+	}
+	if !hasTime {
+		tp.err = fmt.Errorf("missing _time field")
+		tp.cancel()
+		return
+	}
+
+	// Copy block rows to tp.perStreamRows
+	for i, timestamp := range timestamps {
+		streamID := ""
+		fields := make([]logstorage.Field, len(columns))
+		for j, c := range columns {
+			name := strings.Clone(c.Name)
+			value := strings.Clone(c.Values[i])
+
+			fields[j] = logstorage.Field{
+				Name:  name,
+				Value: value,
+			}
+
+			if name == "_stream_id" {
+				streamID = value
+			}
+		}
+
+		tp.perStreamRows[streamID] = append(tp.perStreamRows[streamID], logRow{
+			timestamp: timestamp,
+			fields:    fields,
+		})
+	}
+}
+
+func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
+	if tp.err != nil {
+		return nil, tp.err
+	}
+
+	var resultRows []logRow
+	for streamID, rows := range tp.perStreamRows {
+		sortLogRows(rows)
+
+		lastTimestamp, ok := tp.lastTimestamps[streamID]
+		if ok {
+			// Skip already written rows
+			for len(rows) > 0 && rows[0].timestamp <= lastTimestamp {
+				rows = rows[1:]
+			}
+		}
+		if len(rows) > 0 {
+			resultRows = append(resultRows, rows...)
+			tp.lastTimestamps[streamID] = rows[len(rows)-1].timestamp
+		}
+	}
+	clear(tp.perStreamRows)
+
+	sortLogRows(resultRows)
+
+	tailRows := make([][]logstorage.Field, len(resultRows))
+	for i, row := range resultRows {
+		tailRows[i] = row.fields
+	}
+
+	return tailRows, nil
 }
 
 // ProcessQueryRequest handles /select/logsql/query request.
@@ -215,11 +576,36 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	if limit > 0 {
-		q.AddPipeLimit(uint64(limit))
-	}
 
 	bw := getBufferedWriter(w)
+	defer func() {
+		bw.FlushIgnoreErrors()
+		putBufferedWriter(bw)
+	}()
+	w.Header().Set("Content-Type", "application/stream+json")
+
+	if limit > 0 {
+		if q.CanReturnLastNResults() {
+			rows, err := getLastNQueryResults(ctx, tenantIDs, q, limit)
+			if err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+				return
+			}
+			bb := blockResultPool.Get()
+			b := bb.B
+			for i := range rows {
+				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
+				b = append(b, '\n')
+				bw.WriteIgnoreErrors(b)
+			}
+			bb.B = b
+			blockResultPool.Put(bb)
+			return
+		}
+
+		q.AddPipeLimit(uint64(limit))
+	}
+	q.Optimize()
 
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		if len(columns) == 0 || len(columns[0].Values) == 0 {
@@ -234,20 +620,106 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		blockResultPool.Put(bb)
 	}
 
-	w.Header().Set("Content-Type", "application/stream+json")
-	q.Optimize()
-	err = vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock)
-
-	bw.FlushIgnoreErrors()
-	putBufferedWriter(bw)
-
-	if err != nil {
+	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
 		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
+		return
 	}
 }
 
 var blockResultPool bytesutil.ByteBufferPool
 
+type row struct {
+	timestamp int64
+	fields    []logstorage.Field
+}
+
+func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+	limitUpper := 2 * limit
+	q.AddPipeLimit(uint64(limitUpper))
+	q.Optimize()
+	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
+	if err != nil {
+		return nil, err
+	}
+	if len(rows) < limitUpper {
+		// Fast path - the requested time range contains up to limitUpper rows.
+		rows = getLastNRows(rows, limit)
+		return rows, nil
+	}
+
+	// Slow path - search for the time range containing up to limitUpper rows.
+	start, end := q.GetFilterTimeRange()
+	d := end/2 - start/2
+	start += d
+
+	qOrig := q
+	for {
+		q = qOrig.Clone()
+		q.AddTimeFilter(start, end)
+		rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
+			rows = getLastNRows(rows, limit)
+			return rows, nil
+		}
+
+		lastBit := d & 1
+		d /= 2
+		if len(rows) > limit {
+			start += d
+		} else {
+			start -= d + lastBit
+		}
+	}
+}
+
+func getLastNRows(rows []row, limit int) []row {
+	sort.Slice(rows, func(i, j int) bool {
+		return rows[i].timestamp < rows[j].timestamp
+	})
+	if len(rows) > limit {
+		rows = rows[len(rows)-limit:]
+	}
+	return rows
+}
+
+func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var rows []row
+	var rowsLock sync.Mutex
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		rowsLock.Lock()
+		defer rowsLock.Unlock()
+
+		for i, timestamp := range timestamps {
+			fields := make([]logstorage.Field, len(columns))
+			for j := range columns {
+				f := &fields[j]
+				f.Name = strings.Clone(columns[j].Name)
+				f.Value = strings.Clone(columns[j].Values[i])
+			}
+			rows = append(rows, row{
+				timestamp: timestamp,
+				fields:    fields,
+			})
+		}
+
+		if len(rows) >= limit {
+			cancel()
+		}
+	}
+	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
+		return nil, err
+	}
+
+	return rows, nil
+}
+
 func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {
 	// Extract tenantID
 	tenantID, err := logstorage.GetTenantIDFromRequest(r)
@@ -290,10 +762,10 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
 	if s == "" {
 		return 0, false, nil
 	}
-	currentTimestamp := float64(time.Now().UnixNano()) / 1e9
-	secs, err := promutils.ParseTimeAt(s, currentTimestamp)
+	currentTimestamp := time.Now().UnixNano()
+	nsecs, err := promutils.ParseTimeAt(s, currentTimestamp)
 	if err != nil {
 		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
 	}
-	return int64(secs * 1e9), true, nil
+	return nsecs, true, nil
 }

app/vlselect/logsql/logsql.qtpl

@@ -0,0 +1,32 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+) %}
+
+{% stripspace %}
+
+// ValuesWithHitsJSON generates JSON from the given values.
+{% func ValuesWithHitsJSON(values []logstorage.ValueWithHits) %}
+{
+	"values":{%= valuesWithHitsJSONArray(values) %}
+}
+{% endfunc %}
+
+{% func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) %}
+[
+	{% if len(values) > 0 %}
+		{%= valueWithHitsJSON(values[0]) %}
+		{% for _, v := range values[1:] %}
+			,{%= valueWithHitsJSON(v) %}
+		{% endfor %}
+	{% endif %}
+]
+{% endfunc %}
+
+{% func valueWithHitsJSON(v logstorage.ValueWithHits) %}
+{
+	"value":{%q= v.Value %},
+	"hits":{%dul= v.Hits %}
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/logsql.qtpl.go

@@ -0,0 +1,152 @@
+// Code generated by qtc from "logsql.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlselect/logsql/logsql.qtpl:1
+package logsql
+
+//line app/vlselect/logsql/logsql.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ValuesWithHitsJSON generates JSON from the given values.
+
+//line app/vlselect/logsql/logsql.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/logsql.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/logsql.qtpl:8
+func StreamValuesWithHitsJSON(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:8
+	qw422016.N().S(`{"values":`)
+//line app/vlselect/logsql/logsql.qtpl:10
+	streamvaluesWithHitsJSONArray(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:10
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:12
+func WriteValuesWithHitsJSON(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:12
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+	StreamValuesWithHitsJSON(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:12
+func ValuesWithHitsJSON(values []logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:12
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:12
+	WriteValuesWithHitsJSON(qb422016, values)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:14
+func streamvaluesWithHitsJSONArray(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:14
+	qw422016.N().S(`[`)
+//line app/vlselect/logsql/logsql.qtpl:16
+	if len(values) > 0 {
+//line app/vlselect/logsql/logsql.qtpl:17
+		streamvalueWithHitsJSON(qw422016, values[0])
+//line app/vlselect/logsql/logsql.qtpl:18
+		for _, v := range values[1:] {
+//line app/vlselect/logsql/logsql.qtpl:18
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/logsql.qtpl:19
+			streamvalueWithHitsJSON(qw422016, v)
+//line app/vlselect/logsql/logsql.qtpl:20
+		}
+//line app/vlselect/logsql/logsql.qtpl:21
+	}
+//line app/vlselect/logsql/logsql.qtpl:21
+	qw422016.N().S(`]`)
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:23
+func writevaluesWithHitsJSONArray(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+	streamvaluesWithHitsJSONArray(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:23
+func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:23
+	writevaluesWithHitsJSONArray(qb422016, values)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:25
+func streamvalueWithHitsJSON(qw422016 *qt422016.Writer, v logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:25
+	qw422016.N().S(`{"value":`)
+//line app/vlselect/logsql/logsql.qtpl:27
+	qw422016.N().Q(v.Value)
+//line app/vlselect/logsql/logsql.qtpl:27
+	qw422016.N().S(`,"hits":`)
+//line app/vlselect/logsql/logsql.qtpl:28
+	qw422016.N().DUL(v.Hits)
+//line app/vlselect/logsql/logsql.qtpl:28
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/logsql.qtpl:30
+}
+
+//line app/vlselect/logsql/logsql.qtpl:30
+func writevalueWithHitsJSON(qq422016 qtio422016.Writer, v logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:30
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+	streamvalueWithHitsJSON(qw422016, v)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+}
+
+//line app/vlselect/logsql/logsql.qtpl:30
+func valueWithHitsJSON(v logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:30
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:30
+	writevalueWithHitsJSON(qb422016, v)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:30
+}

app/vlselect/main.go

@@ -1,6 +1,7 @@
 package vlselect
 
 import (
+	"context"
 	"embed"
 	"flag"
 	"fmt"
@@ -13,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -23,7 +23,7 @@ var (
 		"See also -search.maxQueueDuration")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
+	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden on a per-query basis via 'timeout' query arg")
 )
 
 func getDefaultMaxConcurrentRequests() int {
@@ -75,10 +75,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		// Skip requests, which do not start with /select/, since these aren't our requests.
 		return false
 	}
-	path = strings.TrimPrefix(path, "/select")
 	path = strings.ReplaceAll(path, "//", "/")
 
-	if path == "/vmui" {
+	if path == "/select/vmui" {
 		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
 		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
 		// is hidden behind vmauth or similar proxy.
@@ -87,80 +86,129 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.Redirect(w, newURL)
 		return true
 	}
-	if strings.HasPrefix(path, "/vmui/") {
-		if strings.HasPrefix(path, "/vmui/static/") {
+	if strings.HasPrefix(path, "/select/vmui/") {
+		if strings.HasPrefix(path, "/select/vmui/static/") {
 			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
 			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
 			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
 			w.Header().Set("Cache-Control", "max-age=31536000")
 		}
-		r.URL.Path = path
+		r.URL.Path = strings.TrimPrefix(path, "/select")
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
 	}
 
-	// Limit the number of concurrent queries, which can consume big amounts of CPU.
+	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
 	startTime := time.Now()
 	ctx := r.Context()
-	stopCh := ctx.Done()
+	d := getMaxQueryDuration(r)
+	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
+	defer cancel()
+
+	stopCh := ctxWithTimeout.Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
 		defer func() { <-concurrencyLimitCh }()
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
-		d := getMaxQueryDuration(r)
-		if d > *maxQueueDuration {
-			d = *maxQueueDuration
-		}
-		t := timerpool.Get(d)
 		select {
 		case concurrencyLimitCh <- struct{}{}:
-			timerpool.Put(t)
 			defer func() { <-concurrencyLimitCh }()
 		case <-stopCh:
-			timerpool.Put(t)
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-				time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			return true
-		case <-t.C:
-			timerpool.Put(t)
-			concurrencyLimitTimeout.Inc()
-			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
-					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusServiceUnavailable,
+			switch ctxWithTimeout.Err() {
+			case context.Canceled:
+				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+				requestURI := httpserver.GetRequestURI(r)
+				logger.Infof("client has canceled the pending request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+					time.Since(startTime).Seconds(), remoteAddr, requestURI)
+			case context.DeadlineExceeded:
+				concurrencyLimitTimeout.Inc()
+				err := &httpserver.ErrorWithStatusCode{
+					Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
+						"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
+						"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
+						"to pass bigger value to 'timeout' query arg",
+						d.Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
+					StatusCode: http.StatusServiceUnavailable,
+				}
+				httpserver.Errorf(w, r, "%s", err)
 			}
-			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 	}
 
+	if path == "/select/logsql/tail" {
+		logsqlTailRequests.Inc()
+		// Process live tailing request without timeout (e.g. use ctx instead of ctxWithTimeout),
+		// since it is OK to run live tailing requests for very long time.
+		logsql.ProcessLiveTailRequest(ctx, w, r)
+		return true
+	}
+
+	ok := processSelectRequest(ctxWithTimeout, w, r, path)
+	if !ok {
+		return false
+	}
+
+	err := ctxWithTimeout.Err()
+	switch err {
+	case nil:
+		// nothing to do
+	case context.Canceled:
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Infof("client has canceled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+			time.Since(startTime).Seconds(), remoteAddr, requestURI)
+	case context.DeadlineExceeded:
+		err = &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
+				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", d.Seconds(), maxQueryDuration),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+		httpserver.Errorf(w, r, "%s", err)
+	default:
+		httpserver.Errorf(w, r, "unexpected error: %s", err)
+	}
+
+	return true
+}
+
+func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
+	httpserver.EnableCORS(w, r)
 	switch path {
-	case "/logsql/query":
-		logsqlQueryRequests.Inc()
-		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(ctx, w, r)
-		return true
-	case "/logsql/field_values":
-		logsqlFieldValuesRequests.Inc()
-		httpserver.EnableCORS(w, r)
-		logsql.ProcessFieldValuesRequest(ctx, w, r)
-		return true
-	case "/logsql/field_names":
+	case "/select/logsql/field_names":
 		logsqlFieldNamesRequests.Inc()
-		httpserver.EnableCORS(w, r)
 		logsql.ProcessFieldNamesRequest(ctx, w, r)
 		return true
-	case "/logsql/hits":
+	case "/select/logsql/field_values":
+		logsqlFieldValuesRequests.Inc()
+		logsql.ProcessFieldValuesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/hits":
 		logsqlHitsRequests.Inc()
-		httpserver.EnableCORS(w, r)
 		logsql.ProcessHitsRequest(ctx, w, r)
 		return true
+	case "/select/logsql/query":
+		logsqlQueryRequests.Inc()
+		logsql.ProcessQueryRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stream_field_names":
+		logsqlStreamFieldNamesRequests.Inc()
+		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stream_field_values":
+		logsqlStreamFieldValuesRequests.Inc()
+		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stream_ids":
+		logsqlStreamIDsRequests.Inc()
+		logsql.ProcessStreamIDsRequest(ctx, w, r)
+		return true
+	case "/select/logsql/streams":
+		logsqlStreamsRequests.Inc()
+		logsql.ProcessStreamsRequest(ctx, w, r)
+		return true
 	default:
 		return false
 	}
@@ -180,8 +228,13 @@ func getMaxQueryDuration(r *http.Request) time.Duration {
 }
 
 var (
-	logsqlQueryRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
-	logsqlFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
-	logsqlHitsRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
+	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
+	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
+	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
+	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
+	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
+	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
+	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
+	logsqlTailRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
 )

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.bc07cc78.css",
-    "main.js": "./static/js/main.8e7757ef.js",
+    "main.css": "./static/css/main.c9cc37dd.css",
+    "main.js": "./static/js/main.7ae2e2c4.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.da86c2db4f0b05e286b0.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.8c2e588d62b87f90dbf0.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.bc07cc78.css",
-    "static/js/main.8e7757ef.js"
+    "static/css/main.c9cc37dd.css",
+    "static/js/main.7ae2e2c4.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8e7757ef.js"></script><link href="./static/css/main.bc07cc78.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.7ae2e2c4.js"></script><link href="./static/css/main.c9cc37dd.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.7ae2e2c4.js.LICENSE.txt

@@ -5,7 +5,7 @@
 */
 
 /**
- * @remix-run/router v1.15.1
+ * @remix-run/router v1.17.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.22.1
+ * React Router DOM v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +27,7 @@
  */
 
 /**
- * React Router v6.22.1
+ * React Router v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlselect/vmui/static/media/MetricsQL.8c2e588d62b87f90dbf0.md

@@ -1,5 +1,4 @@
 ---
-sort: 23
 weight: 23
 title: MetricsQL
 menu:
@@ -10,9 +9,6 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
-
-# MetricsQL
-
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -107,7 +103,7 @@ The list of MetricsQL features on top of PromQL:
 * Trailing commas on all the lists are allowed - label filters, function args and with expressions.
   For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
   This simplifies maintenance of multi-line queries.
-* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a value MetricsQL expression.
+* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a valid MetricsQL expression.
 * Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
   It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
   Additionally, the following escape sequences are supported:
@@ -220,9 +216,11 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time).
+See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
 
 #### changes
 
@@ -262,6 +260,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -272,6 +272,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -282,6 +284,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -292,7 +296,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### count_over_time
 
@@ -313,6 +319,8 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -328,6 +336,7 @@ See also [increases_over_time](#increases_over_time).
 
 `default_rollup(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 value on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+Compared to [last_over_time](#last_over_time) it accounts for [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) to detect stale series.
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -431,6 +440,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -442,11 +453,15 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -454,6 +469,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -461,8 +478,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
-returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`.
+
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -494,13 +512,14 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -509,12 +528,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -523,6 +543,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
+
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -539,15 +563,18 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -586,6 +613,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -593,9 +622,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time).
+See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 
 #### median_over_time
 
@@ -603,6 +634,8 @@ See also [tmax_over_time](#tmax_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -610,9 +643,11 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time).
+See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
 
 #### mode_over_time
 
@@ -621,6 +656,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -631,6 +668,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -658,6 +697,8 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -669,6 +710,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -679,11 +722,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -702,6 +746,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -709,10 +755,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 #### rollup
@@ -724,6 +771,10 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [rollup_rate](#rollup_rate).
+
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -735,6 +786,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -761,6 +814,8 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -773,12 +828,17 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -786,10 +846,12 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
+
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -823,6 +885,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -837,6 +901,8 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -848,6 +914,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -865,6 +933,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -876,35 +946,43 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
 
 #### sum_eq_over_time
 
-`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
 
-`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
 
-`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -914,6 +992,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -923,6 +1003,8 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1000,6 +1082,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2229,7 +2313,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2262,5 +2347,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
-  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
+  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).

app/vlselect/vmui/victorialogs.html

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta
+    name="description"
+    content="UI for VictoriaMetrics"
+  />
+  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
+  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <!--
+    Notice the use of %PUBLIC_URL% in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>VM UI</title>
+
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:title" content="UI for VictoriaMetrics">
+  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta name="twitter:site" content="@VictoriaMetrics">
+
+  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
+  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta property="og:type" content="website">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlselect/vmui/victoriametrics.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta
+    name="description"
+    content="UI for VictoriaMetrics"
+  />
+  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
+  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <!--
+    Notice the use of %PUBLIC_URL% in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>VM UI</title>
+  <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
+
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:title" content="UI for VictoriaMetrics">
+  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta name="twitter:site" content="@VictoriaMetrics">
+
+  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
+  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta property="og:type" content="website">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlselect/vmui/vmanomaly.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta
+    name="description"
+    content="UI for VictoriaMetrics"
+  />
+  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
+  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <!--
+    Notice the use of %PUBLIC_URL% in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>VM UI</title>
+  <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
+
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:title" content="UI for VictoriaMetrics">
+  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta name="twitter:site" content="@VictoriaMetrics">
+
+  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
+  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta property="og:type" content="website">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlstorage/main.go

@@ -20,19 +20,21 @@ import (
 var (
 	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
 		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
+		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
+	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
+		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
 	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory with the VictoriaLogs data; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#storage")
+		"see https://docs.victoriametrics.com/victorialogs/#retention")
+	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
+		"see https://docs.victoriametrics.com/victorialogs/#storage")
 	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
 		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
 		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
 		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
 	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows")
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows")
 	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/ ; see also -logNewStreams")
+		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
 		"the storage stops accepting new data")
 )
@@ -49,12 +51,13 @@ func Init() {
 		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
 	}
 	cfg := &logstorage.StorageConfig{
-		Retention:             retentionPeriod.Duration(),
-		FlushInterval:         *inmemoryDataFlushInterval,
-		FutureRetention:       futureRetention.Duration(),
-		LogNewStreams:         *logNewStreams,
-		LogIngestedRows:       *logIngestedRows,
-		MinFreeDiskSpaceBytes: minFreeDiskSpaceBytes.N,
+		Retention:              retentionPeriod.Duration(),
+		MaxDiskSpaceUsageBytes: maxDiskSpaceUsageBytes.N,
+		FlushInterval:          *inmemoryDataFlushInterval,
+		FutureRetention:        futureRetention.Duration(),
+		LogNewStreams:          *logNewStreams,
+		LogIngestedRows:        *logIngestedRows,
+		MinFreeDiskSpaceBytes:  minFreeDiskSpaceBytes.N,
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
@@ -77,7 +80,7 @@ func Init() {
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics)
+	metrics.UnregisterSet(storageMetrics, true)
 	storageMetrics = nil
 
 	strg.MustClose()
@@ -112,17 +115,43 @@ func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorag
 }
 
 // GetFieldNames executes q and returns field names seen in results.
-func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]string, error) {
+func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
 	return strg.GetFieldNames(ctx, tenantIDs, q)
 }
 
 // GetFieldValues executes q and returns unique values for the fieldName seen in results.
 //
 // If limit > 0, then up to limit unique values are returned.
-func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]string, error) {
+func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
 	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
+// GetStreamFieldNames executes q and returns stream field names seen in results.
+func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
+}
+
+// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
+//
+// If limit > 0, then up to limit unique stream field values are returned.
+func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+}
+
+// GetStreams executes q and returns streams seen in query results.
+//
+// If limit > 0, then up to limit unique streams are returned.
+func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreams(ctx, tenantIDs, q, limit)
+}
+
+// GetStreamIDs executes q and returns streamIDs seen in query results.
+//
+// If limit > 0, then up to limit unique streamIDs are returned.
+func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
+}
+
 func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	var ss logstorage.StorageStats
 	strg.UpdateStats(&ss)

app/vmagent/main.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
@@ -24,7 +26,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -37,14 +38,13 @@ import (
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
-	statsdserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 )
 
 var (
@@ -63,12 +63,6 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	statsdListenAddr = flag.String("statsdListenAddr", "", "TCP and UDP address to listen for Statsd plaintext data. Usually :8125 must be set. Doesn't work if empty. "+
-		"See also -statsdListenAddr.useProxyProtocol")
-	statsdUseProxyProtocol = flag.Bool("statsdListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -statsdListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	statsdDisableAggregationEnforcemenet = flag.Bool(`statsd.disableAggregationEnforcement`, false, "Whether to disable streaming aggregation requirement check. "+
-		"It's recommended to run statsdServer with pre-configured streaming aggregation to decrease load at database.")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
@@ -78,8 +72,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -88,7 +82,6 @@ var (
 var (
 	influxServer       *influxserver.Server
 	graphiteServer     *graphiteserver.Server
-	statsdServer       *statsdserver.Server
 	opentsdbServer     *opentsdbserver.Server
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
@@ -123,7 +116,7 @@ func main() {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
+			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
@@ -146,12 +139,6 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
 	}
-	if len(*statsdListenAddr) > 0 {
-		if !remotewrite.HasAnyStreamAggrConfigured() && !*statsdDisableAggregationEnforcemenet {
-			logger.Fatalf("streaming aggregation must be configured with enabled statsd server. It's recommended  to aggregate metrics received at statsd listener. This check could be disabled with flag -statsd.disableAggregationEnforcement")
-		}
-		statsdServer = statsdserver.MustStart(*statsdListenAddr, *statsdUseProxyProtocol, statsd.InsertHandler)
-	}
 	if len(*opentsdbListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
 		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
@@ -187,9 +174,6 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer.MustStop()
 	}
-	if len(*statsdListenAddr) > 0 {
-		statsdServer.MustStop()
-	}
 	if len(*opentsdbListenAddr) > 0 {
 		opentsdbServer.MustStop()
 	}
@@ -334,6 +318,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -452,7 +440,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -461,17 +449,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		var bb bytesutil.ByteBuffer
 		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
 		}
 		promscrapeConfigReloadRequests.Inc()
@@ -580,6 +568,10 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -690,7 +682,8 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmagent/prometheusimport/request_handler_test.go

@@ -26,11 +26,11 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Errorf("unxepected error %s", err)
+		t.Fatalf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 

app/vmagent/remotewrite/client.go

@@ -16,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -33,11 +34,13 @@ var (
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+	sendTimeout      = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
+	retryMaxTime     = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
+	proxyURL         = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for estabilishing tls connections to the corresponding -remoteWrite.url")
+	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
@@ -89,6 +92,9 @@ type client struct {
 	fq *persistentqueue.FastQueue
 	hc *http.Client
 
+	retryMinInterval time.Duration
+	retryMaxTime     time.Duration
+
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
@@ -119,7 +125,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
-		DialContext:         statDial,
+		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
 		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
@@ -142,13 +148,15 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
-		sanitizedURL:   sanitizedURL,
-		remoteWriteURL: remoteWriteURL,
-		authCfg:        authCfg,
-		awsCfg:         awsCfg,
-		fq:             fq,
-		hc:             hc,
-		stopCh:         make(chan struct{}),
+		sanitizedURL:     sanitizedURL,
+		remoteWriteURL:   remoteWriteURL,
+		authCfg:          authCfg,
+		awsCfg:           awsCfg,
+		fq:               fq,
+		hc:               hc,
+		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
+		retryMaxTime:     retryMaxTime.GetOptionalArg(argIdx),
+		stopCh:           make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
 
@@ -395,11 +403,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
 //
 // The function returns false only if c.stopCh is closed.
-// Otherwise it tries sending the block to remote storage indefinitely.
+// Otherwise, it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
-	retryDuration := timeutil.AddJitterToDuration(time.Second)
+	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
+	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
 	retriesCount := 0
 
 again:

app/vmagent/remotewrite/pendingseries.go

@@ -109,10 +109,9 @@ type writeRequest struct {
 
 	wr prompbmarshal.WriteRequest
 
-	tss       []prompbmarshal.TimeSeries
-	labels    []prompbmarshal.Label
-	samples   []prompbmarshal.Sample
-	exemplars []prompbmarshal.Exemplar
+	tss     []prompbmarshal.TimeSeries
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
 
 	// buf holds labels data
 	buf []byte
@@ -130,7 +129,6 @@ func (wr *writeRequest) reset() {
 	wr.labels = wr.labels[:0]
 
 	wr.samples = wr.samples[:0]
-	wr.exemplars = wr.exemplars[:0]
 	wr.buf = wr.buf[:0]
 }
 
@@ -202,7 +200,6 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	labelsDst := wr.labels
 	labelsLen := len(wr.labels)
 	samplesDst := wr.samples
-	exemplarsDst := wr.exemplars
 	buf := wr.buf
 	for i := range src.Labels {
 		labelsDst = append(labelsDst, prompbmarshal.Label{})
@@ -219,12 +216,8 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	samplesDst = append(samplesDst, src.Samples...)
 	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
 
-	exemplarsDst = append(exemplarsDst, src.Exemplars...)
-	dst.Exemplars = exemplarsDst[len(exemplarsDst)-len(src.Exemplars):]
-
 	wr.samples = samplesDst
 	wr.labels = labelsDst
-	wr.exemplars = exemplarsDst
 	wr.buf = buf
 }
 
@@ -236,6 +229,7 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 		// Nothing to push
 		return true
 	}
+
 	marshalConcurrencyCh <- struct{}{}
 
 	bb := writeRequestBufPool.Get()
@@ -272,8 +266,6 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 	if len(wr.Timeseries) == 1 {
 		// A single time series left. Recursively split its samples into smaller parts if possible.
 		samples := wr.Timeseries[0].Samples
-		exemplars := wr.Timeseries[0].Exemplars
-
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
 			return true
@@ -285,16 +277,11 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 			return false
 		}
 		wr.Timeseries[0].Samples = samples[n:]
-		// We do not want to send exemplars twice
-		wr.Timeseries[0].Exemplars = nil
-
 		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
 			wr.Timeseries[0].Samples = samples
-			wr.Timeseries[0].Exemplars = exemplars
 			return false
 		}
 		wr.Timeseries[0].Samples = samples
-		wr.Timeseries[0].Exemplars = exemplars
 		return true
 	}
 	timeseries := wr.Timeseries

app/vmagent/remotewrite/pendingseries_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestPushWriteRequest(t *testing.T) {
 	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
-	expectedBlockLensProm := []int{248, 1952, 17433, 180381, 1861994}
-	expectedBlockLensVM := []int{170, 575, 4748, 44936, 367096}
+	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
+	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
 	for i, rowsCount := range rowsCounts {
 		expectedBlockLenProm := expectedBlockLensProm[i]
 		expectedBlockLenVM := expectedBlockLensVM[i]
@@ -34,7 +34,7 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 			return true
 		}
 		if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
-			t.Fatalf("cannot push data to to remote storage")
+			t.Fatalf("cannot push data to remote storage")
 		}
 		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
 			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
@@ -59,20 +59,6 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 				Value: fmt.Sprintf("value_%d_%d", i, j),
 			})
 		}
-		exemplar := prompbmarshal.Exemplar{
-			Labels: []prompbmarshal.Label{
-				{
-					Name:  "trace_id",
-					Value: "123456",
-				},
-				{
-					Name:  "log_id",
-					Value: "987654",
-				},
-			},
-			Value:     float64(i),
-			Timestamp: 1000 * int64(i),
-		}
 		wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
 			Labels: labels,
 			Samples: []prompbmarshal.Sample{
@@ -81,10 +67,6 @@ func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteReque
 					Timestamp: 1000 * int64(i),
 				},
 			},
-
-			Exemplars: []prompbmarshal.Exemplar{
-				exemplar,
-			},
 		})
 	}
 	return &wr

app/vmagent/remotewrite/relabel.go

@@ -181,7 +181,7 @@ func (rctx *relabelCtx) reset() {
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return &relabelCtx{}
 	},
 }

app/vmagent/remotewrite/remotewrite.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -88,15 +89,15 @@ var (
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence ."+
-		"See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flagutil.NewArrayBool("remoteWrite.dropSamplesOnOverload", "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
+		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
+	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
 )
 
 var (
-	// rwctxs contains statically populated entries when -remoteWrite.url is specified.
-	rwctxs []*remoteWriteCtx
+	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
+	rwctxsGlobal []*remoteWriteCtx
 
 	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
 	defaultAuthToken = &auth.Token{}
@@ -109,8 +110,11 @@ var (
 		StatusCode: http.StatusTooManyRequests,
 	}
 
-	// disableOnDiskQueueAll is set to true if all remoteWrite.urls were configured to disable persistent queue via disableOnDiskQueue
-	disableOnDiskQueueAll bool
+	// disableOnDiskQueueAny is set to true if at least a single -remoteWrite.url is configured with -remoteWrite.disableOnDiskQueue
+	disableOnDiskQueueAny bool
+
+	// dropSamplesOnFailureGlobal is set to true if -remoteWrite.dropSamplesOnOverload is set or if multiple -remoteWrite.disableOnDiskQueue options are set.
+	dropSamplesOnFailureGlobal bool
 )
 
 // MultitenancyEnabled returns true if -enableMultitenantHandlers is specified.
@@ -203,28 +207,18 @@ func Init() {
 	relabelConfigSuccess.Set(1)
 	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
-	sasFile, sasOpts := getStreamAggrOpts(-1)
-	if sasFile != "" {
-		sas, err := newStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
-		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -streamAggr.config=%q: %s", sasFile, err)
-		}
-		sasGlobal.Store(sas)
-	} else if sasOpts.DedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesDropFailed, sasOpts.DedupInterval, sasOpts.DropInputLabels)
-	}
+	initStreamAggrConfigGlobal()
 
-	if len(*remoteWriteURLs) > 0 {
-		rwctxs = newRemoteWriteCtxs(nil, *remoteWriteURLs)
-	}
+	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 
-	disableOnDiskQueueAll = true
-	for _, v := range *disableOnDiskQueue {
-		if !v {
-			disableOnDiskQueueAll = false
-			break
-		}
-	}
+	disableOnDiskQueues := []bool(*disableOnDiskQueue)
+	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
+
+	// Samples must be dropped if multiple -remoteWrite.disableOnDiskQueue options are configured and at least a single is set to true.
+	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
+	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
+	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
+	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
 
 	dropDanglingQueues()
 
@@ -234,9 +228,9 @@ func Init() {
 		defer configReloaderWG.Done()
 		for {
 			select {
-			case <-sighupCh:
 			case <-configReloaderStopCh:
 				return
+			case <-sighupCh:
 			}
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
@@ -255,8 +249,8 @@ func dropDanglingQueues() {
 	// In case if there were many persistent queues with identical *remoteWriteURLs
 	// the queue with the last index will be dropped.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxs))
-	for _, rwctx := range rwctxs {
+	existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
 		existingQueues[rwctx.fq.Dirname()] = struct{}{}
 	}
 
@@ -273,7 +267,7 @@ func dropDanglingQueues() {
 		}
 	}
 	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
 	}
 }
 
@@ -377,13 +371,15 @@ func Stop() {
 	configReloaderWG.Wait()
 
 	sasGlobal.Load().MustStop()
-	deduplicatorGlobal.MustStop()
-	deduplicatorGlobal = nil
+	if deduplicatorGlobal != nil {
+		deduplicatorGlobal.MustStop()
+		deduplicatorGlobal = nil
+	}
 
-	for _, rwctx := range rwctxs {
+	for _, rwctx := range rwctxsGlobal {
 		rwctx.MustStop()
 	}
-	rwctxs = nil
+	rwctxsGlobal = nil
 
 	if sl := hourlySeriesLimiter; sl != nil {
 		sl.MustStop()
@@ -395,6 +391,8 @@ func Stop() {
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
 //
+// PushDropSamplesOnFailure drops wr samples if they cannot be sent to -remoteWrite.url by any reason.
+//
 // PushDropSamplesOnFailure can modify wr contents.
 func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
@@ -407,7 +405,7 @@ func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
 func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, false)
+	return tryPush(at, wr, dropSamplesOnFailureGlobal)
 }
 
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
@@ -424,33 +422,30 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		tenantRctx = getRelabelCtx()
 		defer putRelabelCtx(tenantRctx)
 	}
-	rowsCount := getRowsCount(tss)
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
 	// if some of remote storage systems cannot keep up with the data ingestion rate.
-	// this shortcut is only applicable if all remote writes have disableOnDiskQueue = true
-	if disableOnDiskQueueAll {
-		for _, rwctx := range rwctxs {
-			if rwctx.fq.IsWriteBlocked() {
-				rwctx.pushFailures.Inc()
-				if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
-					// Just drop samples
-					rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-					continue
-				}
-				return false
-			}
-		}
+	rwctxs, ok := getEligibleRemoteWriteCtxs(tss, forceDropSamplesOnFailure)
+	if !ok {
+		// At least a single remote write queue is blocked and dropSamplesOnFailure isn't set.
+		// Return false to the caller, so it could re-send samples again.
+		return false
+	}
+	if len(rwctxs) == 0 {
+		// All the remote write queues are skipped because they are blocked and dropSamplesOnFailure is set to true.
+		// Return true to the caller, so it doesn't re-send the samples again.
+		return true
 	}
 
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 {
+	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}
+	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
@@ -492,7 +487,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
-		if sas != nil {
+		if sas.IsEnabled() {
 			matchIdxs := matchIdxsPool.Get()
 			matchIdxs.B = sas.Push(tssBlock, matchIdxs.B)
 			if !*streamAggrGlobalKeepInput {
@@ -503,20 +498,46 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
-		if !tryPushBlockToRemoteStorages(tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
 	return true
 }
 
-func pushToRemoteStoragesDropFailed(tss []prompbmarshal.TimeSeries) {
-	if tryPushBlockToRemoteStorages(tss, true) {
+func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
+	if !disableOnDiskQueueAny {
+		return rwctxsGlobal, true
+	}
+
+	// This code is applicable if at least a single remote storage has -disableOnDiskQueue
+	rwctxs := make([]*remoteWriteCtx, 0, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
+		if !rwctx.fq.IsWriteBlocked() {
+			rwctxs = append(rwctxs, rwctx)
+		} else {
+			rwctx.pushFailures.Inc()
+			if !forceDropSamplesOnFailure {
+				return nil, false
+			}
+			rowsCount := getRowsCount(tss)
+			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+		}
+	}
+	return rwctxs, true
+}
+
+func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
+	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
+	if len(rwctxs) == 0 {
 		return
 	}
+	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
+		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
+	}
 }
 
-func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -535,7 +556,7 @@ func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDrop
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingBlockAmongRemoteStorages(tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
@@ -556,7 +577,7 @@ func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDrop
 	return !anyPushFailed.Load()
 }
 
-func tryShardingBlockAmongRemoteStorages(tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -743,10 +764,8 @@ type remoteWriteCtx struct {
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
 
-	streamAggrKeepInput   bool
-	streamAggrDropInput   bool
-	disableOnDiskQueue    bool
-	dropSamplesOnOverload bool
+	streamAggrKeepInput bool
+	streamAggrDropInput bool
 
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
@@ -771,6 +790,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
+
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -815,38 +835,20 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		c:   c,
 		pss: pss,
 
-		dropSamplesOnOverload: dropSamplesOnOverload.GetOptionalArg(argIdx),
-		disableOnDiskQueue:    isPQDisabled,
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 
-		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-	}
-
-	// Initialize sas
-	sasFile, sasOpts := getStreamAggrOpts(argIdx)
-	if sasFile != "" {
-		sas, err := newStreamAggrConfig(argIdx, rwctx.pushInternalTrackDropped)
-		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
-		}
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
-	} else if sasOpts.DedupInterval > 0 {
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, sasOpts.DedupInterval, sasOpts.DropInputLabels)
+		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
+	rwctx.initStreamAggrConfig()
 
 	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
 	// sas and deduplicator must be stopped before rwctx is closed
-	// because sas can write pending series to rwctx.pss if there are any
+	// because they can write pending series to rwctx.pss if there are any
 	sas := rwctx.sas.Swap(nil)
 	sas.MustStop()
 
@@ -873,12 +875,20 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 // TryPush sends tss series to the configured remote write endpoint
 //
-// TryPush can be called concurrently for multiple remoteWriteCtx,
-// so it shouldn't modify tss entries.
+// TryPush doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
 func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
-	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
+	defer func() {
+		if rctx == nil {
+			return
+		}
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
+	}()
+
+	// Apply relabeling
 	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
 	if pcs.Len() > 0 {
@@ -899,7 +909,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 
 	// Apply stream aggregation or deduplication if they are configured
 	sas := rwctx.sas.Load()
-	if sas != nil {
+	if sas.IsEnabled() {
 		matchIdxs := matchIdxsPool.Get()
 		matchIdxs.B = sas.Push(tss, matchIdxs.B)
 		if !rwctx.streamAggrKeepInput {
@@ -914,28 +924,22 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 		matchIdxsPool.Put(matchIdxs)
 	} else if rwctx.deduplicator != nil {
 		rwctx.deduplicator.Push(tss)
-		tss = tss[:0]
+		return true
 	}
 
-	// Try pushing the data to remote storage
-	ok := rwctx.tryPushInternal(tss)
-
-	// Return back relabeling contexts to the pool
-	if rctx != nil {
-		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
+	// Try pushing tss to remote storage
+	if rwctx.tryPushInternal(tss) {
+		return true
 	}
 
-	if !ok {
-		rwctx.pushFailures.Inc()
-		if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
-			rwctx.rowsDroppedOnPushFailure.Add(len(tss))
-			return true
-		}
+	// Couldn't push tss to remote storage
+	rwctx.pushFailures.Inc()
+	if forceDropSamplesOnFailure {
+		rowsCount := getRowsCount(tss)
+		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+		return true
 	}
-
-	return ok
+	return false
 }
 
 var matchIdxsPool bytesutil.ByteBufferPool
@@ -959,19 +963,26 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSe
 	if rwctx.tryPushInternal(tss) {
 		return
 	}
-	if !rwctx.disableOnDiskQueue {
+	if !rwctx.fq.IsPersistentQueueDisabled() {
 		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
 	}
 	rwctx.pushFailures.Inc()
-	if dropSamplesOnOverload.GetOptionalArg(rwctx.idx) {
-		rowsCount := getRowsCount(tss)
-		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-	}
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 }
 
 func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
+	defer func() {
+		if rctx == nil {
+			return
+		}
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
+	}()
+
 	if len(labelsGlobal) > 0 {
 		// Make a copy of tss before adding extra labels in order to prevent
 		// from affecting time series for other remoteWrite.url configs.
@@ -984,19 +995,11 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 	pss := rwctx.pss
 	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
 
-	ok := pss[idx].TryPush(tss)
-
-	if rctx != nil {
-		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssPool.Put(v)
-		putRelabelCtx(rctx)
-	}
-
-	return ok
+	return pss[idx].TryPush(tss)
 }
 
 var tssPool = &sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		a := []prompbmarshal.TimeSeries{}
 		return &a
 	},

app/vmagent/remotewrite/remotewrite_test.go

@@ -3,14 +3,12 @@ package remotewrite
 import (
 	"fmt"
 	"math"
-	"os"
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -79,24 +77,28 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil)
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
 		}
 
-		if len(streamAggrConfig) > 0 {
-			f := createFile(t, []byte(streamAggrConfig))
-			sas, err := streamaggr.LoadFromFile(f.Name(), nil, nil)
+		if streamAggrConfig != "" {
+			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
+			defer sas.MustStop()
 			rwctx.sas.Store(sas)
 		}
 
-		inputTss := mustParsePromMetrics(input)
+		offsetMsecs := time.Now().UnixMilli()
+		inputTss := prompbmarshal.MustParsePromMetrics(input, offsetMsecs)
 		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
 
 		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		rwctx.TryPush(inputTss, false)
+		if !rwctx.TryPush(inputTss, false) {
+			t.Fatalf("cannot push samples to rwctx")
+		}
 
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
@@ -165,51 +167,3 @@ metric{env="dev"} 15
 metric{env="bar"} 25
 `)
 }
-
-func mustParsePromMetrics(s string) []prompbmarshal.TimeSeries {
-	var rows prometheus.Rows
-	errLogger := func(s string) {
-		panic(fmt.Errorf("unexpected error when parsing Prometheus metrics: %s", s))
-	}
-	rows.UnmarshalWithErrLogger(s, errLogger)
-	var tss []prompbmarshal.TimeSeries
-	samples := make([]prompbmarshal.Sample, 0, len(rows.Rows))
-	for _, row := range rows.Rows {
-		labels := make([]prompbmarshal.Label, 0, len(row.Tags)+1)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: row.Metric,
-		})
-		for _, tag := range row.Tags {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  tag.Key,
-				Value: tag.Value,
-			})
-		}
-		samples = append(samples, prompbmarshal.Sample{
-			Value:     row.Value,
-			Timestamp: row.Timestamp,
-		})
-		ts := prompbmarshal.TimeSeries{
-			Labels:  labels,
-			Samples: samples[len(samples)-1:],
-		}
-		tss = append(tss, ts)
-	}
-	return tss
-}
-
-func createFile(t *testing.T, data []byte) *os.File {
-	t.Helper()
-	f, err := os.CreateTemp("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := os.WriteFile(f.Name(), data, 0644); err != nil {
-		t.Fatal(err)
-	}
-	if err := f.Sync(); err != nil {
-		t.Fatal(err)
-	}
-	return f
-}

app/vmagent/remotewrite/statconn.go

@@ -1,73 +0,0 @@
-package remotewrite
-
-import (
-	"context"
-	"net"
-	"sync/atomic"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-func statDial(ctx context.Context, _, addr string) (conn net.Conn, err error) {
-	network := netutil.GetTCPNetwork()
-	conn, err = netutil.DialMaybeSRV(ctx, network, addr)
-	dialsTotal.Inc()
-	if err != nil {
-		dialErrors.Inc()
-		return nil, err
-	}
-	conns.Inc()
-	sc := &statConn{
-		Conn: conn,
-	}
-	return sc, nil
-}
-
-var (
-	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
-	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
-	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
-)
-
-type statConn struct {
-	closed atomic.Int32
-	net.Conn
-}
-
-func (sc *statConn) Read(p []byte) (int, error) {
-	n, err := sc.Conn.Read(p)
-	connReadsTotal.Inc()
-	if err != nil {
-		connReadErrors.Inc()
-	}
-	connBytesRead.Add(n)
-	return n, err
-}
-
-func (sc *statConn) Write(p []byte) (int, error) {
-	n, err := sc.Conn.Write(p)
-	connWritesTotal.Inc()
-	if err != nil {
-		connWriteErrors.Inc()
-	}
-	connBytesWritten.Add(n)
-	return n, err
-}
-
-func (sc *statConn) Close() error {
-	err := sc.Conn.Close()
-	if sc.closed.Add(1) == 1 {
-		conns.Dec()
-	}
-	return err
-}
-
-var (
-	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
-	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
-	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
-	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
-	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
-	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
-)

app/vmagent/remotewrite/streamaggr.go

@@ -36,125 +36,207 @@ var (
 		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 
 	// Per URL config
-	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
+	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
 		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config . See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
-		"aggregation interval for the corresponding -remoteWrite.streamAggr.config . "+
+		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if "+
-		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from clients pushing data into the vmagent. "+
+	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
+		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
 func CheckStreamAggrConfigs() error {
-	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+	// Check global config
+	sas, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		return err
+	}
+	sas.MustStop()
 
-	if _, err := newStreamAggrConfig(-1, pushNoop); err != nil {
-		return fmt.Errorf("could not load -streamAggr.config stream aggregation config: %w", err)
-	}
 	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
-		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*streamAggrConfig), len(*remoteWriteURLs))
+		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
 	}
+
+	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
 	for idx := range *streamAggrConfig {
-		if _, err := newStreamAggrConfig(idx, pushNoop); err != nil {
+		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
+		if err != nil {
 			return err
 		}
+		sas.MustStop()
 	}
 	return nil
 }
 
-// HasAnyStreamAggrConfigured checks if any streaming aggregation config provided
-func HasAnyStreamAggrConfigured() bool {
-	return len(*streamAggrConfig) > 0 || *streamAggrGlobalConfig != ""
-}
-
 func reloadStreamAggrConfigs() {
-	reloadStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
-	for idx, rwctx := range rwctxs {
-		reloadStreamAggrConfig(idx, rwctx.pushInternalTrackDropped)
+	reloadStreamAggrConfigGlobal()
+	for _, rwctx := range rwctxsGlobal {
+		rwctx.reloadStreamAggrConfig()
 	}
 }
 
-func reloadStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) {
-	path, opts := getStreamAggrOpts(idx)
-	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := newStreamAggrConfigWithOpts(pushFunc, path, opts)
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload stream aggregation config at %q; continue using the previously loaded config; error: %s", path, err)
+func reloadStreamAggrConfigGlobal() {
+	path := *streamAggrGlobalConfig
+	if path == "" {
 		return
 	}
 
-	var sas *streamaggr.Aggregators
-	if idx < 0 {
-		sas = sasGlobal.Load()
-	} else {
-		sas = rwctxs[idx].sas.Load()
+	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
+
+	sasNew, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
+		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		return
 	}
 
+	sas := sasGlobal.Load()
 	if !sasNew.Equal(sas) {
-		var sasOld *streamaggr.Aggregators
-		if idx < 0 {
-			sasOld = sasGlobal.Swap(sasNew)
-		} else {
-			sasOld = rwctxs[idx].sas.Swap(sasNew)
-		}
+		sasOld := sasGlobal.Swap(sasNew)
 		sasOld.MustStop()
-		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
+		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
 	} else {
 		sasNew.MustStop()
-		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
+		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
 }
 
-func getStreamAggrOpts(idx int) (string, *streamaggr.Options) {
-	if idx < 0 {
-		return *streamAggrGlobalConfig, &streamaggr.Options{
-			DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
-			DropInputLabels:      *streamAggrGlobalDropInputLabels,
-			IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
-			IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+func initStreamAggrConfigGlobal() {
+	sas, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
+	}
+	if sas != nil {
+		filePath := sas.FilePath()
+		sasGlobal.Store(sas)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
+	} else {
+		dedupInterval := streamAggrGlobalDedupInterval.Duration()
+		if dedupInterval > 0 {
+			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 		}
 	}
-	opts := streamaggr.Options{
+}
+
+func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
+	idx := rwctx.idx
+
+	sas, err := rwctx.newStreamAggrConfig()
+	if err != nil {
+		logger.Fatalf("cannot initialize stream aggregators: %s", err)
+	}
+	if sas != nil {
+		filePath := sas.FilePath()
+		rwctx.sas.Store(sas)
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
+		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
+	} else {
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+		if dedupInterval > 0 {
+			alias := fmt.Sprintf("dedup-%d", idx+1)
+			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
+		}
+	}
+}
+
+func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
+	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
+	if path == "" {
+		return
+	}
+
+	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
+
+	sasNew, err := rwctx.newStreamAggrConfig()
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
+		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		return
+	}
+
+	sas := rwctx.sas.Load()
+	if !sasNew.Equal(sas) {
+		sasOld := rwctx.sas.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
+	} else {
+		sasNew.MustStop()
+		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
+	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
+}
+
+func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
+	path := *streamAggrGlobalConfig
+	if path == "" {
+		return nil, nil
+	}
+
+	opts := &streamaggr.Options{
+		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
+		DropInputLabels:      *streamAggrGlobalDropInputLabels,
+		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
+		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+		KeepInput:            *streamAggrGlobalKeepInput,
+	}
+
+	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
+	if err != nil {
+		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
+	}
+	return sas, nil
+}
+
+func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
+	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
+}
+
+func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
+	path := streamAggrConfig.GetOptionalArg(idx)
+	if path == "" {
+		return nil, nil
+	}
+
+	alias := fmt.Sprintf("%d:secret-url", idx+1)
+	if *showRemoteWriteURL {
+		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
+	}
+	opts := &streamaggr.Options{
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
+		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
 	}
-	if len(*streamAggrConfig) == 0 {
-		return "", &opts
-	}
-	return (*streamAggrConfig)[idx], &opts
-}
 
-func newStreamAggrConfigWithOpts(pushFunc streamaggr.PushFunc, path string, opts *streamaggr.Options) (*streamaggr.Aggregators, error) {
-	if len(path) == 0 {
-		// Skip empty stream aggregation config.
-		return nil, nil
+	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
+	if err != nil {
+		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
 	}
-	return streamaggr.LoadFromFile(path, pushFunc, opts)
-}
-
-func newStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path, opts := getStreamAggrOpts(idx)
-	return newStreamAggrConfigWithOpts(pushFunc, path, opts)
+	return sas, nil
 }

app/vmagent/statsd/request_handler.go

@@ -1,72 +0,0 @@
-package statsd
-
-import (
-	"io"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/statsd/stream"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="statsd"}`)
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="statsd"}`)
-)
-
-// InsertHandler processes remote write for statsd plaintext protocol.
-//
-// See https://github.com/statsd/statsd/blob/master/docs/metric_types.md
-func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
-		return insertRows(nil, rows)
-	})
-}
-
-func insertRows(at *auth.Token, rows []parser.Row) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		labelsLen := len(labels)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: r.Metric,
-		})
-		for j := range r.Tags {
-			tag := &r.Tags[j]
-			labels = append(labels, prompbmarshal.Label{
-				Name:  tag.Key,
-				Value: tag.Value,
-			})
-		}
-		samplesLen := len(samples)
-		for _, v := range r.Values {
-			samples = append(samples, prompbmarshal.Sample{
-				Value:     v,
-				Timestamp: r.Timestamp,
-			})
-		}
-
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	rowsPerInsert.Update(float64(len(rows)))
-	return nil
-}

app/vmalert-tool/Makefile

@@ -81,6 +81,9 @@ vmalert-tool-linux-ppc64le:
 vmalert-tool-linux-s390x:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
+vmalert-tool-linux-loong64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+
 vmalert-tool-linux-386:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert-tool/main.go

@@ -26,8 +26,14 @@ func main() {
 				UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html#Unit-testing-for-rules",
 				Flags: []cli.Flag{
 					&cli.StringSliceFlag{
-						Name:     "files",
-						Usage:    "files to run unittest with. Supports an array of values separated by comma or specified via multiple flags.",
+						Name: "files",
+						Usage: `File path or http url with test files. Supports an array of values separated by comma or specified via multiple flags.
+						Supports hierarchical patterns and regexpes.
+Examples:
+ -files="/path/to/file". Path to a single test file.
+ -files="http://<some-server-addr>/path/to/test.yaml". HTTP URL to a test file.
+ -files="dir/**/*.yaml". Includes all the .yaml files in "dir" subfolders recursively.
+ `,
 						Required: true,
 					},
 					&cli.BoolFlag{
@@ -35,9 +41,19 @@ func main() {
 						Usage:    "disable adding group's Name as label to generated alerts and time series.",
 						Required: false,
 					},
+					&cli.StringSliceFlag{
+						Name:     "external.label",
+						Usage:    `Optional label in the form 'name=value' to add to all generated recording rules and alerts. Supports an array of values separated by comma or specified via multiple flags.`,
+						Required: false,
+					},
+					&cli.StringFlag{
+						Name:     "external.url",
+						Usage:    `Optional external URL to template in rule's labels or annotations.`,
+						Required: false,
+					},
 				},
 				Action: func(c *cli.Context) error {
-					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
+					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url")); failed {
 						return fmt.Errorf("unittest failed")
 					}
 					return nil

app/vmalert-tool/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmalert-tool/unittest/input.go

@@ -18,7 +18,7 @@ import (
 	"github.com/VictoriaMetrics/metricsql"
 )
 
-var numReg = regexp.MustCompile(`\D?\d*\D?`)
+var numReg = regexp.MustCompile(`\D?\d*\.?\d*\D?`)
 
 // series holds input_series defined in the test file
 type series struct {
@@ -74,10 +74,7 @@ func writeInputSeries(input []series, interval *promutils.Duration, startStamp t
 		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
 	}
 
-	data, err := testutil.Compress(r)
-	if err != nil {
-		return fmt.Errorf("failed to compress data: %v", err)
-	}
+	data := testutil.Compress(r)
 	// write input series to vm
 	httpWrite(dst, bytes.NewBuffer(data))
 	vmstorage.Storage.DebugFlush()

app/vmalert-tool/unittest/input_test.go

@@ -6,88 +6,61 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )
 
-func TestParseInputValue(t *testing.T) {
-	testCases := []struct {
-		input  string
-		exp    []sequenceValue
-		failed bool
-	}{
-		{
-			"",
-			nil,
-			true,
-		},
-		{
-			"testfailed",
-			nil,
-			true,
-		},
-		// stale doesn't support operations
-		{
-			"stalex3",
-			nil,
-			true,
-		},
-		{
-			"-4",
-			[]sequenceValue{{Value: -4}},
-			false,
-		},
-		{
-			"_",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"stale",
-			[]sequenceValue{{Value: decimal.StaleNaN}},
-			false,
-		},
-		{
-			"-4x1",
-			[]sequenceValue{{Value: -4}, {Value: -4}},
-			false,
-		},
-		{
-			"_x1",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"1+1x4",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 4}, {Value: 5}},
-			false,
-		},
-		{
-			"2-1x4",
-			[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
-			false,
-		},
-		{
-			"1+1x1 _ -4 stale 3+20x1",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
-			false,
-		},
+func TestParseInputValue_Failure(t *testing.T) {
+	f := func(input string) {
+		t.Helper()
+
+		_, err := parseInputValue(input, true)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
 	}
 
-	for _, tc := range testCases {
-		output, err := parseInputValue(tc.input, true)
-		if err != nil != tc.failed {
-			t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
+	f("")
+	f("testfailed")
+
+	// stale doesn't support operations
+	f("stalex3")
+}
+
+func TestParseInputValue_Success(t *testing.T) {
+	f := func(input string, outputExpected []sequenceValue) {
+		t.Helper()
+
+		output, err := parseInputValue(input, true)
+		if err != nil {
+			t.Fatalf("unexpected error in parseInputValue: %s", err)
 		}
-		if len(tc.exp) != len(output) {
-			t.Fatalf("expect %v, got %v", tc.exp, output)
+
+		if len(outputExpected) != len(output) {
+			t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
 		}
-		for i := 0; i < len(tc.exp); i++ {
-			if tc.exp[i].Omitted != output[i].Omitted {
-				t.Fatalf("expect %v, got %v", tc.exp, output)
+		for i := 0; i < len(outputExpected); i++ {
+			if outputExpected[i].Omitted != output[i].Omitted {
+				t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}
-			if tc.exp[i].Value != output[i].Value {
-				if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
+			if outputExpected[i].Value != output[i].Value {
+				if decimal.IsStaleNaN(outputExpected[i].Value) && decimal.IsStaleNaN(output[i].Value) {
 					continue
 				}
-				t.Fatalf("expect %v, got %v", tc.exp, output)
+				t.Fatalf("unexpeccted Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}
 		}
 	}
+
+	f("-4", []sequenceValue{{Value: -4}})
+
+	f("_", []sequenceValue{{Omitted: true}})
+
+	f("stale", []sequenceValue{{Value: decimal.StaleNaN}})
+
+	f("-4x1", []sequenceValue{{Value: -4}, {Value: -4}})
+
+	f("_x1", []sequenceValue{{Omitted: true}})
+
+	f("1+1x2 0.1 0.1+0.3x2 3.14", []sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}})
+
+	f("2-1x4", []sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}})
+
+	f("1+1x1 _ -4 stale 3+20x1", []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}})
 }

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -24,20 +24,16 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
-              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
+              description: "localhost:9090 of job vmagent2 in cluster  has been down for more than 5 minutes."
+              dashboard: "/d/dashboard?orgId=1"
 
       - eval_time: 0
         alertname: AlwaysFiring
         exp_alerts:
-          - exp_labels:
-              datacenter: dc-123
+          - {}
 
       - eval_time: 0
         alertname: InstanceDown
         exp_alerts: []
-
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -8,7 +8,8 @@ groups:
           severity: page
         annotations:
           summary: "Instance {{ $labels.instance }} down"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+          description: "{{ $labels.instance }} of job {{ $labels.job }} in cluster {{ $externalLabels.cluster }} has been down for more than 5 minutes."
+          dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
       - alert: AlwaysFiring
         expr: 1
       - alert: SameAlertNameWithDifferentGroup
@@ -37,3 +38,6 @@ groups:
     rules:
       - record: t3
         expr: t1
+
+  - name: emptyGroup
+    rules: []

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -16,7 +16,8 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - {}
+          - exp_labels:
+              cluster: prod
       - eval_time: 1m
         groupname: group2
         alertname: SameAlertNameWithDifferentGroup
@@ -25,7 +26,8 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - {}
+          - exp_labels:
+              cluster: prod
       - eval_time: 6m
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
@@ -61,18 +63,18 @@ tests:
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t1", datacenter="dc-123"}'
+            labels: '{__name__="t1", cluster="prod"}'
       - expr: t2
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t2", datacenter="dc-123"}'
+            labels: '{__name__="t2", cluster="prod"}'
       - expr: t3
         eval_time: 4m
         exp_samples:
           # t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
           - value: 3
-            labels: '{__name__="t3", datacenter="dc-123"}'
+            labels: '{__name__="t3", cluster="prod"}'
 
     alert_rule_test:
       - eval_time: 10m
@@ -83,22 +85,21 @@ tests:
               job: vmagent1
               severity: page
               instance: localhost:9090
-              datacenter: dc-123
+              cluster: prod
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
+              description: "localhost:9090 of job vmagent1 in cluster prod has been down for more than 5 minutes."
+              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              datacenter: dc-123
+              cluster: prod
 
       - eval_time: 0
         groupname: alerts
         alertname: InstanceDown
         exp_alerts: []
 
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test", cluster="prod", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -25,22 +25,21 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
-              datacenter: dc-123
+              cluster: prod
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
+              description: "localhost:9090 of job vmagent2 in cluster prod has been down for more than 5 minutes."
+              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              datacenter: dc-123
+              cluster: prod
 
       - eval_time: 0
         groupname: group1
         alertname: InstanceDown
         exp_alerts: []
 
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/unittest.go

@@ -9,10 +9,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"sort"
+	"strings"
 	"time"
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -53,7 +55,7 @@ const (
 )
 
 // UnitTest runs unittest for files
-func UnitTest(files []string, disableGroupLabel bool) bool {
+func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL string) bool {
 	if err := templates.Load([]string{}, true); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
@@ -66,36 +68,58 @@ func UnitTest(files []string, disableGroupLabel bool) bool {
 	defer vminsert.Stop()
 	defer vmselect.Stop()
 	disableAlertgroupLabel = disableGroupLabel
-	return rulesUnitTest(files)
-}
 
-func rulesUnitTest(files []string) bool {
+	testfiles, err := config.ReadFromFS(files)
+	if err != nil {
+		logger.Fatalf("failed to load test files %q: %v", files, err)
+	}
+	if len(testfiles) == 0 {
+		fmt.Println("no test file found")
+		return false
+	}
+
+	labels := make(map[string]string)
+	for _, s := range externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+	_, err = notifier.Init(nil, labels, externalURL)
+	if err != nil {
+		logger.Fatalf("failed to init notifier: %v", err)
+	}
+
 	var failed bool
-	for _, f := range files {
-		if err := ruleUnitTest(f); err != nil {
+	for fileName, file := range testfiles {
+		if err := ruleUnitTest(fileName, file, labels); err != nil {
 			fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", f, err)
+			fmt.Printf("\nfailed to run unit test for file %q: \n%v", fileName, err)
 			failed = true
 		} else {
 			fmt.Println("  SUCCESS")
 		}
 	}
+
 	return failed
 }
 
-func ruleUnitTest(filename string) []error {
+func ruleUnitTest(filename string, content []byte, externalLabels map[string]string) []error {
 	fmt.Println("\nUnit Testing: ", filename)
-	b, err := os.ReadFile(filename)
-	if err != nil {
-		return []error{fmt.Errorf("failed to read file: %w", err)}
-	}
-
 	var unitTestInp unitTestFile
-	if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
+	if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
 		return []error{fmt.Errorf("failed to unmarshal file: %w", err)}
 	}
-	if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
-		return []error{fmt.Errorf("failed to resolve path for `rule_files`: %w", err)}
+
+	// add file directory for rule files if needed
+	for i, rf := range unitTestInp.RuleFiles {
+		if rf != "" && !filepath.IsAbs(rf) && !strings.HasPrefix(rf, "http") {
+			unitTestInp.RuleFiles[i] = filepath.Join(filepath.Dir(filename), rf)
+		}
 	}
 
 	if unitTestInp.EvaluationInterval.Duration() == 0 {
@@ -122,7 +146,7 @@ func ruleUnitTest(filename string) []error {
 			errs = append(errs, err)
 			continue
 		}
-		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
+		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups, externalLabels)
 		errs = append(errs, testErrs...)
 	}
 
@@ -139,26 +163,30 @@ func verifyTestGroup(group testGroup) error {
 	}
 	for _, at := range group.AlertRuleTests {
 		if at.Alertname == "" {
-			return fmt.Errorf("\n%s    missing required filed \"alertname\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"alertname\"", testGroupName)
 		}
 		if !disableAlertgroupLabel && at.GroupName == "" {
-			return fmt.Errorf("\n%s    missing required filed \"groupname\" when flag \"disableAlertGroupLabel\" is false", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"groupname\" when flag \"disableAlertgroupLabel\" is false", testGroupName)
 		}
 		if disableAlertgroupLabel && at.GroupName != "" {
-			return fmt.Errorf("\n%s    shouldn't set filed \"groupname\" when flag \"disableAlertGroupLabel\" is true", testGroupName)
+			return fmt.Errorf("\n%s    shouldn't set field \"groupname\" when flag \"disableAlertgroupLabel\" is true", testGroupName)
 		}
 		if at.EvalTime == nil {
-			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"eval_time\"", testGroupName)
 		}
 	}
 	for _, et := range group.MetricsqlExprTests {
 		if et.Expr == "" {
-			return fmt.Errorf("\n%s    missing required filed \"expr\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"expr\"", testGroupName)
 		}
 		if et.EvalTime == nil {
-			return fmt.Errorf("\n%s    missing required filed \"eval_time\"", testGroupName)
+			return fmt.Errorf("\n%s    missing required field \"eval_time\"", testGroupName)
 		}
 	}
+	if group.ExternalLabels != nil {
+		fmt.Printf("\n%s    warning: filed `external_labels` will be deprecated soon, please use `-external.label` cmd-line flag instead. "+
+			"Check https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735 for details.\n", testGroupName)
+	}
 	return nil
 }
 
@@ -175,6 +203,7 @@ func processFlags() {
 		{flag: "retentionPeriod", value: "100y"},
 		{flag: "datasource.url", value: testDataSourcePath},
 		{flag: "remoteWrite.url", value: testRemoteWritePath},
+		{flag: "notifier.blackhole", value: "true"},
 	} {
 		// panics if flag doesn't exist
 		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
@@ -235,31 +264,7 @@ func tearDown() {
 	fs.MustRemoveAll(storagePath)
 }
 
-// resolveAndGlobFilepaths joins all relative paths in a configuration
-// with a given base directory and replaces all globs with matching files.
-func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
-	for i, rf := range utf.RuleFiles {
-		if rf != "" && !filepath.IsAbs(rf) {
-			utf.RuleFiles[i] = filepath.Join(baseDir, rf)
-		}
-	}
-
-	var globbedFiles []string
-	for _, rf := range utf.RuleFiles {
-		m, err := filepath.Glob(rf)
-		if err != nil {
-			return err
-		}
-		if len(m) == 0 {
-			fmt.Fprintln(os.Stderr, "  WARNING: no file match pattern", rf)
-		}
-		globbedFiles = append(globbedFiles, m...)
-	}
-	utf.RuleFiles = globbedFiles
-	return nil
-}
-
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group, externalLabels map[string]string) (checkErrs []error) {
 	// set up vmstorage and http server for ingest and read queries
 	setUp()
 	// tear down vmstorage and clean the data dir
@@ -308,7 +313,14 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	// create groups with given rule
 	var groups []*rule.Group
 	for _, group := range testGroups {
-		ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
+		mergedExternalLabels := make(map[string]string)
+		for k, v := range tg.ExternalLabels {
+			mergedExternalLabels[k] = v
+		}
+		for k, v := range externalLabels {
+			mergedExternalLabels[k] = v
+		}
+		ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
 		groups = append(groups, ng)
 	}
 
@@ -316,11 +328,15 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	maxEvalTime := testStartTime.Add(tg.maxEvalTime())
 	for ts := testStartTime; ts.Before(maxEvalTime) || ts.Equal(maxEvalTime); ts = ts.Add(evalInterval) {
 		for _, g := range groups {
+			if len(g.Rules) == 0 {
+				continue
+			}
 			errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts)
 			for err := range errs {
 				if err != nil {
 					checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name,
 						ts, err))
+					return
 				}
 			}
 			// flush series after each group evaluation
@@ -374,7 +390,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 						if expAlert.ExpLabels == nil {
 							expAlert.ExpLabels = make(map[string]string)
 						}
-						// alertGroupNameLabel is added as additional labels when `disableAlertGroupLabel` is false
+						// alertGroupNameLabel is added as additional labels when `disableAlertgroupLabel` is false
 						if !disableAlertgroupLabel {
 							expAlert.ExpLabels["alertgroup"] = groupname
 						}

app/vmalert-tool/unittest/unittest_test.go

@@ -14,34 +14,34 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestUnitRule(t *testing.T) {
-	testCases := []struct {
-		name              string
-		disableGroupLabel bool
-		files             []string
-		failed            bool
-	}{
-		{
-			name:   "run multi files",
-			files:  []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
-			failed: false,
-		},
-		{
-			name:              "disable group label",
-			disableGroupLabel: true,
-			files:             []string{"./testdata/disable-group-label.yaml"},
-			failed:            false,
-		},
-		{
-			name:   "failing test",
-			files:  []string{"./testdata/failed-test.yaml"},
-			failed: true,
-		},
-	}
-	for _, tc := range testCases {
-		fail := UnitTest(tc.files, tc.disableGroupLabel)
-		if fail != tc.failed {
-			t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
+func TestUnitTest_Failure(t *testing.T) {
+	f := func(files []string) {
+		t.Helper()
+
+		failed := UnitTest(files, false, nil, "")
+		if !failed {
+			t.Fatalf("expecting failed test")
 		}
 	}
+
+	// failing test
+	f([]string{"./testdata/failed-test.yaml"})
+}
+
+func TestUnitTest_Success(t *testing.T) {
+	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
+		t.Helper()
+
+		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL)
+		if failed {
+			t.Fatalf("unexpected failed test")
+		}
+	}
+
+	// run multi files
+	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
+
+	// disable group label
+	// template with null external values
+	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
 }

app/vmalert/Makefile

@@ -101,8 +101,7 @@ replay-vmalert: vmalert
 		-remoteWrite.url=http://localhost:8428 \
 		-external.label=cluster=east-1 \
 		-external.label=replica=a \
-		-replay.timeFrom=2021-05-11T07:21:43Z \
-		-replay.timeTo=2021-05-29T18:40:43Z
+		-replay.timeFrom=2024-06-01T00:00:00Z
 
 vmalert-linux-amd64:
 	APP_NAME=vmalert CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch

app/vmalert/config/config.go

@@ -45,11 +45,11 @@ type Group struct {
 	// EvalAlignment will make the timestamp of group query requests be aligned with interval
 	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]interface{} `yaml:",inline"`
+	XXX map[string]any `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
 	type group Group
 	if err := unmarshal((*group)(g)); err != nil {
 		return err
@@ -142,11 +142,11 @@ type Rule struct {
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]interface{} `yaml:",inline"`
+	XXX map[string]any `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (r *Rule) UnmarshalYAML(unmarshal func(any) error) error {
 	type rule Rule
 	if err := unmarshal((*rule)(r)); err != nil {
 		return err
@@ -234,7 +234,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 	cLogger.Suppress(true)
 	defer cLogger.Suppress(false)
 
-	files, err := readFromFS(pathPatterns)
+	files, err := ReadFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
@@ -243,7 +243,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns)
+	files, err := ReadFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
@@ -301,7 +301,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.
-		XXX map[string]interface{} `yaml:",inline"`
+		XXX map[string]any `yaml:",inline"`
 	}{}
 	err = yaml.Unmarshal(data, &g)
 	if err != nil {
@@ -310,7 +310,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	return g.Groups, checkOverflow(g.XXX, "config")
 }
 
-func checkOverflow(m map[string]interface{}, ctx string) error {
+func checkOverflow(m map[string]any, ctx string) error {
 	if len(m) > 0 {
 		var keys []string
 		for k := range m {

app/vmalert/config/config_test.go

@@ -23,12 +23,6 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
-		t.Errorf("error parsing files %s", err)
-	}
-}
-
 func TestParseFromURL(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/bad", func(w http.ResponseWriter, _ *http.Request) {
@@ -55,438 +49,353 @@ groups:
 	defer srv.Close()
 
 	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
-		t.Errorf("error parsing URLs %s", err)
+		t.Fatalf("error parsing URLs %s", err)
 	}
 
 	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
-		t.Errorf("expected parsing error: %s", err)
+		t.Fatalf("expected parsing error: %s", err)
 	}
 }
 
-func TestParseBad(t *testing.T) {
-	testCases := []struct {
-		path   []string
-		expErr string
-	}{
-		{
-			[]string{"testdata/rules/rules_interval_bad.rules"},
-			"eval_offset should be smaller than interval",
-		},
-		{
-			[]string{"testdata/rules/rules0-bad.rules"},
-			"unexpected token",
-		},
-		{
-			[]string{"testdata/dir/rules0-bad.rules"},
-			"error parsing annotation",
-		},
-		{
-			[]string{"testdata/dir/rules1-bad.rules"},
-			"duplicate in file",
-		},
-		{
-			[]string{"testdata/dir/rules2-bad.rules"},
-			"function \"unknown\" not defined",
-		},
-		{
-			[]string{"testdata/dir/rules3-bad.rules"},
-			"either `record` or `alert` must be set",
-		},
-		{
-			[]string{"testdata/dir/rules4-bad.rules"},
-			"either `record` or `alert` must be set",
-		},
-		{
-			[]string{"testdata/rules/rules1-bad.rules"},
-			"bad graphite expr",
-		},
-		{
-			[]string{"testdata/dir/rules6-bad.rules"},
-			"missing ':' in header",
-		},
-		{
-			[]string{"http://unreachable-url"},
-			"failed to",
-		},
+func TestParse_Success(t *testing.T) {
+	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
+	if err != nil {
+		t.Fatalf("error parsing files %s", err)
 	}
-	for _, tc := range testCases {
-		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
+}
+
+func TestParse_Failure(t *testing.T) {
+	f := func(paths []string, errStrExpected string) {
+		t.Helper()
+
+		_, err := Parse(paths, notifier.ValidateTemplates, true)
 		if err == nil {
-			t.Errorf("expected to get error")
-			return
+			t.Fatalf("expected to get error")
 		}
-		if !strings.Contains(err.Error(), tc.expErr) {
-			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		if !strings.Contains(err.Error(), errStrExpected) {
+			t.Fatalf("expected err to contain %q; got %q instead", errStrExpected, err)
 		}
 	}
+
+	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
+	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
+	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
+	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
+	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
+	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
+	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
+	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
+	f([]string{"http://unreachable-url"}, "failed to")
 }
 
-func TestRule_Validate(t *testing.T) {
+func TestRuleValidate(t *testing.T) {
 	if err := (&Rule{}).Validate(); err == nil {
-		t.Errorf("expected empty name error")
+		t.Fatalf("expected empty name error")
 	}
 	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
-		t.Errorf("expected empty expr error")
+		t.Fatalf("expected empty expr error")
 	}
 	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
-		t.Errorf("expected valid rule; got %s", err)
+		t.Fatalf("expected valid rule; got %s", err)
 	}
 }
 
-func TestGroup_Validate(t *testing.T) {
-	testCases := []struct {
-		group               *Group
-		rules               []Rule
-		validateAnnotations bool
-		validateExpressions bool
-		expErr              string
-	}{
-		{
-			group:  &Group{},
-			expErr: "group name must be set",
-		},
-		{
-			group: &Group{
-				Name:     "negative interval",
-				Interval: promutils.NewDuration(-1),
+func TestGroupValidate_Failure(t *testing.T) {
+	f := func(group *Group, validateExpressions bool, errStrExpected string) {
+		t.Helper()
+
+		err := group.Validate(nil, validateExpressions)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		errStr := err.Error()
+		if !strings.Contains(errStr, errStrExpected) {
+			t.Fatalf("missing %q in the returned error %q", errStrExpected, errStr)
+		}
+	}
+
+	f(&Group{}, false, "group name must be set")
+
+	f(&Group{
+		Name:     "negative interval",
+		Interval: promutils.NewDuration(-1),
+	}, false, "interval shouldn't be lower than 0")
+
+	f(&Group{
+		Name:       "wrong eval_offset",
+		Interval:   promutils.NewDuration(time.Minute),
+		EvalOffset: promutils.NewDuration(2 * time.Minute),
+	}, false, "eval_offset should be smaller than interval")
+
+	f(&Group{
+		Name:  "wrong limit",
+		Limit: -1,
+	}, false, "invalid limit")
+
+	f(&Group{
+		Name:        "wrong concurrency",
+		Concurrency: -1,
+	}, false, "invalid concurrency")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
 			},
-			expErr: "interval shouldn't be lower than 0",
-		},
-		{
-			group: &Group{
-				Name:       "wrong eval_offset",
-				Interval:   promutils.NewDuration(time.Minute),
-				EvalOffset: promutils.NewDuration(2 * time.Minute),
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
 			},
-			expErr: "eval_offset should be smaller than interval",
 		},
-		{
-			group: &Group{
-				Name:  "wrong limit",
-				Limit: -1,
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Record: "record", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Record: "record", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test graphite prometheus bad expr",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Expr: "sum(up == 0 ) by (host)",
+				For:  promutils.NewDuration(10 * time.Millisecond),
 			},
-			expErr: "invalid limit",
-		},
-		{
-			group: &Group{
-				Name:        "wrong concurrency",
-				Concurrency: -1,
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
 			},
-			expErr: "invalid concurrency",
 		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Record: "record",
-						Expr:   "up | 0",
-					},
+	}, false, "invalid rule")
+
+	f(&Group{
+		Name: "test graphite inherit",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
+				For:  promutils.NewDuration(10 * time.Millisecond),
+			},
+			{
+				Expr: "sum(up == 0 ) by (host)",
+			},
+		},
+	}, false, "either `record` or `alert` must be set")
+
+	// validate expressions
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, true, "invalid expression")
+
+	f(&Group{
+		Name: "test thanos",
+		Type: NewRawType("thanos"),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, true, "unknown datasource type")
+
+	f(&Group{
+		Name: "test graphite",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "some-description",
+			}},
+		},
+	}, true, "bad graphite expr")
+}
+
+func TestGroupValidate_Success(t *testing.T) {
+	f := func(group *Group, validateAnnotations, validateExpressions bool) {
+		t.Helper()
+
+		var validateTplFn ValidateTplFn
+		if validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := group.Validate(validateTplFn, validateExpressions)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	}
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, false, false)
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
+				Labels: map[string]string{
+					"summary": "{{ value|query }}",
 				},
 			},
-			expErr: "",
 		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Record: "record",
-						Expr:   "up | 0",
-					},
-				},
-			},
-			expErr:              "invalid expression",
-			validateExpressions: true,
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-						Labels: map[string]string{
-							"summary": "{{ value|query }}",
-						},
-					},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-						Labels: map[string]string{
-							"summary": `
+	}, false, false)
+
+	// validate annotiations
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
+				Labels: map[string]string{
+					"summary": `
 {{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
   {{ . | first | value | humanize1024 }}B
 {{ end }}`,
-						},
-					},
-				},
-			},
-			validateAnnotations: true,
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-					},
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-					},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Record: "record", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Record: "record", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test thanos",
-				Type: NewRawType("thanos"),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "unknown datasource type",
-		},
-		{
-			group: &Group{
-				Name: "test graphite",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "some-description",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "",
-		},
-		{
-			group: &Group{
-				Name: "test prometheus",
-				Type: NewPrometheusType(),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "",
-		},
-		{
-			group: &Group{
-				Name: "test graphite inherit",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{
-						Expr: "sumSeries(time('foo.bar',10))",
-						For:  promutils.NewDuration(10 * time.Millisecond),
-					},
-					{
-						Expr: "sum(up == 0 ) by (host)",
-					},
 				},
 			},
 		},
-		{
-			group: &Group{
-				Name: "test graphite prometheus bad expr",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{
-						Expr: "sum(up == 0 ) by (host)",
-						For:  promutils.NewDuration(10 * time.Millisecond),
-					},
-					{
-						Expr: "sumSeries(time('foo.bar',10))",
-					},
-				},
-			},
-			expErr: "invalid rule",
-		},
-	}
+	}, true, false)
 
-	for _, tc := range testCases {
-		var validateTplFn ValidateTplFn
-		if tc.validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
-		}
-		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
-		if err == nil {
-			if tc.expErr != "" {
-				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
-			}
-			continue
-		}
-		if !strings.Contains(err.Error(), tc.expErr) {
-			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
-		}
-	}
+	// validate expressions
+	f(&Group{
+		Name: "test prometheus",
+		Type: NewPrometheusType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, false, true)
 }
 
-func TestHashRule(t *testing.T) {
-	testCases := []struct {
-		a, b  Rule
-		equal bool
-	}{
-		{
-			Rule{Record: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1"},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-				"foo": "bar",
-			}},
-			true,
-		},
-		{
-			Rule{Alert: "record", Expr: "up == 1"},
-			Rule{Alert: "record", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 1"},
-			false,
-		},
-		{
-			Rule{Record: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 2"},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-				"foo": "baz",
-			}},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-			}},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			false,
-		},
-	}
-	for i, tc := range testCases {
-		aID, bID := HashRule(tc.a), HashRule(tc.b)
-		if tc.equal != (aID == bID) {
-			t.Fatalf("missmatch for rule %d", i)
+func TestHashRule_NotEqual(t *testing.T) {
+	f := func(a, b Rule) {
+		t.Helper()
+
+		aID, bID := HashRule(a), HashRule(b)
+		if aID == bID {
+			t.Fatalf("rule hashes mustn't be equal; got %d", aID)
 		}
 	}
+
+	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
+
+	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 2"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+		"foo": "baz",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1"})
+}
+
+func TestHashRule_Equal(t *testing.T) {
+	f := func(a, b Rule) {
+		t.Helper()
+
+		aID, bID := HashRule(a), HashRule(b)
+		if aID != bID {
+			t.Fatalf("rule hashes must be equal; got %d and %d", aID, bID)
+		}
+	}
+
+	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1"}, Rule{Alert: "alert", Expr: "up == 1"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+		"foo": "bar",
+	}})
+
+	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Alert: "record", Expr: "up == 1"})
+
+	f(Rule{
+		Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute),
+	}, Rule{Alert: "alert", Expr: "up == 1"})
 }
 
 func TestGroupChecksum(t *testing.T) {

app/vmalert/config/fs.go

@@ -32,14 +32,14 @@ var (
 	fsRegistry   = make(map[string]FS)
 )
 
-// readFromFS parses the given path list and inits FS for each item.
-// Once initialed, readFromFS will try to read and return files from each FS.
-// readFromFS returns an error if at least one FS failed to init.
+// ReadFromFS parses the given path list and inits FS for each item.
+// Once initialed, ReadFromFS will try to read and return files from each FS.
+// ReadFromFS returns an error if at least one FS failed to init.
 // The function can be called multiple times but each unique path
 // will be initialed only once.
 //
 // It is allowed to mix different FS types in path list.
-func readFromFS(paths []string) (map[string][]byte, error) {
+func ReadFromFS(paths []string) (map[string][]byte, error) {
 	var err error
 	result := make(map[string][]byte)
 	for _, path := range paths {

app/vmalert/config/log/logger.go

@@ -29,7 +29,7 @@ func (l *Logger) isDisabled() bool {
 }
 
 // Errorf logs error message.
-func (l *Logger) Errorf(format string, args ...interface{}) {
+func (l *Logger) Errorf(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -37,7 +37,7 @@ func (l *Logger) Errorf(format string, args ...interface{}) {
 }
 
 // Warnf logs warning message.
-func (l *Logger) Warnf(format string, args ...interface{}) {
+func (l *Logger) Warnf(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -45,7 +45,7 @@ func (l *Logger) Warnf(format string, args ...interface{}) {
 }
 
 // Infof logs info message.
-func (l *Logger) Infof(format string, args ...interface{}) {
+func (l *Logger) Infof(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -54,6 +54,6 @@ func (l *Logger) Infof(format string, args ...interface{}) {
 
 // Panicf logs panic message and panics.
 // Panicf can't be suppressed
-func (l *Logger) Panicf(format string, args ...interface{}) {
+func (l *Logger) Panicf(format string, args ...any) {
 	logger.Panicf(format, args...)
 }

app/vmalert/config/log/logger_test.go

@@ -18,14 +18,14 @@ func TestOutput(t *testing.T) {
 
 	mustMatch := func(exp string) {
 		t.Helper()
+
 		if exp == "" {
 			if testOutput.String() != "" {
-				t.Errorf("expected output to be empty; got %q", testOutput.String())
-				return
+				t.Fatalf("expected output to be empty; got %q", testOutput.String())
 			}
 		}
 		if !strings.Contains(testOutput.String(), exp) {
-			t.Errorf("output %q should contain %q", testOutput.String(), exp)
+			t.Fatalf("output %q should contain %q", testOutput.String(), exp)
 		}
 		fmt.Println(testOutput.String())
 		testOutput.Reset()

app/vmalert/config/types.go

@@ -69,7 +69,7 @@ func (t *Type) ValidateExpr(expr string) error {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err
@@ -87,7 +87,7 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 // MarshalYAML implements the yaml.Unmarshaler interface.
-func (t Type) MarshalYAML() (interface{}, error) {
+func (t Type) MarshalYAML() (any, error) {
 	return t.Name, nil
 }
 
@@ -98,7 +98,7 @@ type Header struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (h *Header) UnmarshalYAML(unmarshal func(any) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err

app/vmalert/datasource/init.go

@@ -12,11 +12,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
-	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
-		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend and -datasource.showURL")
+	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
@@ -56,9 +58,10 @@ var (
 		`Whether to align "time" parameter with evaluation interval. `+
 		"Alignment supposed to produce deterministic results despite number of vmalert replicas or time they were started. "+
 		"See more details at https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
-	maxIdleConnections = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
-	disableKeepAlive   = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
-		`If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.`)
+	maxIdleConnections    = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
+	idleConnectionTimeout = flag.Duration("datasource.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
+	disableKeepAlive      = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
+		`If true, disables HTTP keep-alive and will only use the connection to the server for a single HTTP request.`)
 	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
@@ -96,13 +99,15 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
 	}
+	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
+	tr.IdleConnTimeout = *idleConnectionTimeout
 
 	if extraParams == nil {
 		extraParams = url.Values{}

app/vmalert/datasource/vm.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -140,7 +141,7 @@ func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Resu
 	}
 	resp, err := s.do(req)
 	if err != nil {
-		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
 			// Return unexpected error to the caller.
 			return Result{}, nil, err
 		}
@@ -185,7 +186,7 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	}
 	resp, err := s.do(req)
 	if err != nil {
-		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
 			// Return unexpected error to the caller.
 			return res, err
 		}

app/vmalert/datasource/vm_prom_api.go

@@ -57,20 +57,23 @@ var jsonParserPool fastjson.ParserPool
 //	[{"metric":{"__name__":"up","job":"prometheus"},value": [ 1435781451.781,"1"]},
 //	{"metric":{"__name__":"up","job":"node"},value": [ 1435781451.781,"0"]}]
 func (pi *promInstant) Unmarshal(b []byte) error {
+	var metrics []json.RawMessage
+	// metrics slice could be large, so parsing it with fastjson could consume a lot of memory.
+	// We parse the slice with standard lib to keep mem usage low.
+	// And each metric object will be parsed with fastjson to reduce allocations.
+	if err := json.Unmarshal(b, &metrics); err != nil {
+		return fmt.Errorf("cannot unmarshal metrics: %w", err)
+	}
+
 	p := jsonParserPool.Get()
 	defer jsonParserPool.Put(p)
 
-	v, err := p.ParseBytes(b)
-	if err != nil {
-		return err
-	}
-
-	rows, err := v.Array()
-	if err != nil {
-		return fmt.Errorf("cannot find the top-level array of result objects: %w", err)
-	}
-	pi.ms = make([]Metric, len(rows))
-	for i, row := range rows {
+	pi.ms = make([]Metric, len(metrics))
+	for i, data := range metrics {
+		row, err := p.ParseBytes(data)
+		if err != nil {
+			return fmt.Errorf("cannot parse metric object: %w", err)
+		}
 		metric := row.Get("metric")
 		if metric == nil {
 			return fmt.Errorf("can't find `metric` object in %q", row)
@@ -119,7 +122,7 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 type promRange struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TVs    [][2]interface{}  `json:"values"`
+		TVs    [][2]any          `json:"values"`
 	} `json:"result"`
 }
 
@@ -147,7 +150,7 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
-type promScalar [2]interface{}
+type promScalar [2]any
 
 func (r promScalar) metrics() ([]Metric, error) {
 	var m Metric