Fix version mismatch

Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>

.github/workflows/benchmark.yml

@@ -0,0 +1,56 @@
+name: benchmark
+on:
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-streamaggr-benchmark-image:
+    name: build
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'streamaggr-benchmark')
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Package VMAgent Docker image for benchmark
+        run: |
+          SKIP_SCRATCH_BUILD=true \
+          DOCKER_BUILD_OPTS='--cache-to type=gha,mode=max --cache-from type=gha' \
+          PKG_TAG=${{ github.event.pull_request.head.sha }} \
+          DOCKER_REGISTRY=ghcr.io \
+          TARGET_PLATFORM=linux/amd64 make publish-vmagent

.github/workflows/main.yml

@@ -6,12 +6,14 @@ on:
       - cluster
       - master
     paths:
+      - '.github/workflows/main.yml'
       - '**.go'
   pull_request:
     branches:
       - cluster
       - master
     paths:
+      - '.github/workflows/main.yml'
       - '**.go'
 
 permissions:
@@ -46,10 +48,25 @@ jobs:
           key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-check-all-
 
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
+      - name: Run fmt
+        run: make fmt
+
+      - name: Run vet
+        run: make vet
+      
+        # Use action instead of `make golangci-lint` to speed up the process
+        # as it caches data between builds.
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          # The version must match "install-golangci-lint" Makefile target version.
+          version: v1.59.1
+
+      - name: Run govulncheck
+        run: make govulncheck
+
+      - name: Check diff
+        run: git diff --exit-code
 
   test:
     name: test

.gitignore

@@ -7,7 +7,6 @@
 .vscode
 *.test
 *.swp
-/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
@@ -24,6 +23,4 @@ Gemfile.lock
 _site
 *.tmp
 /docs/.jekyll-metadata
-coverage.txt
-cspell.json
-*~
+coverage.txt

.wwhrd.yml

@@ -4,4 +4,3 @@ allowlist:
   - BSD-3-Clause
   - BSD-2-Clause
   - ISC
-  - MPL-2.0

CODE_OF_CONDUCT_RU.md

@@ -0,0 +1,120 @@
+
+# Кодекс Поведения участника
+
+## Наши обязательства
+
+Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
+свободным от притеснений для всех, независимо от возраста, телосложения,
+видимых или невидимых ограничений способности, этнической принадлежности,
+половых признаков, гендерной идентичности и выражения, уровня опыта,
+образования, социо-экономического статуса, национальности, внешности,
+расы, религии, или сексуальной идентичности и ориентации.
+
+Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
+дружелюбное, многообразное, инклюзивное и здоровое сообщество.
+
+## Наши стандарты
+
+Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
+
+* Проявление доброты и эмпатии к другим участникам проекта
+* Уважение к чужой точке зрения и опыту
+* Конструктивная критика и принятие конструктивной критики
+* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
+  и извлечение уроков из опыта
+* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
+
+Примеры неприемлемого поведения участников включают в себя:
+
+* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
+* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
+* Публичное или приватное домогательство
+* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
+* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
+
+## Обязанности
+
+Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
+поведения и будут предпринимать соответствующие и честные меры по исправлению положения
+в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
+
+Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
+комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
+с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
+
+## Область применения
+
+Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
+а также когда человек официально представляет сообщество в публичных местах.
+Примеры представления проекта или сообщества включают использование официальной электронной почты,
+публикации в официальном аккаунте в социальных сетях,
+или упоминания как представителя в онлайн или оффлайн мероприятии.
+
+## Приведение в исполнение
+
+О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
+поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
+Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
+
+Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
+неприкосновенность автора сообщения.
+
+## Руководство по исполнению
+
+Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
+чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
+
+### 1. Исправление
+
+**Общественное влияние**: Использование недопустимой лексики или другое поведение,
+считающиеся непрофессиональным или нежелательным в сообществе.
+
+**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
+объясняющее суть нарушения и почему такое поведение
+было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
+
+### 2. Предупреждение
+
+**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
+
+**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
+На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
+включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
+в публичных пространствах, а так же во внешних каналах,
+таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
+
+### 3. Временный бан
+
+**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
+включая продолжительное неуместное поведение.
+
+**Последствия**: Временный запрет (бан) на любое взаимодействие
+или публичное общение с сообществом на определенный период времени.
+На этот период не допускается публичное или личное взаимодействие с людьми,
+вовлеченными в инцидент, включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса.
+Нарушение этих правил влечет за собой вечный бан.
+
+### 4. Вечный бан
+
+**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
+включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
+или проявление агрессии либо пренебрежительного отношения к категориям лиц.
+
+**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
+
+## Атрибуция
+
+Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
+версии 2.0, доступной по адресу
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
+
+Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

Makefile

@@ -13,12 +13,10 @@ PKG_TAG := $(BUILDINFO_TAG)
 endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
-TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
@@ -27,7 +25,6 @@ include package/release/Makefile
 all: \
 	victoria-metrics-prod \
 	victoria-logs-prod \
-	vlogscli-prod \
 	vmagent-prod \
 	vmalert-prod \
 	vmalert-tool-prod \
@@ -52,7 +49,6 @@ publish: \
 package: \
 	package-victoria-metrics \
 	package-victoria-logs \
-	package-vlogscli \
 	package-vmagent \
 	package-vmalert \
 	package-vmalert-tool \
@@ -248,7 +244,7 @@ release-victoria-metrics-windows-amd64:
 
 release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
@@ -265,14 +261,6 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-metrics-windows-$(GOARCH)-prod.exe
 
-release-victoria-logs-bundle: \
-	release-victoria-logs \
-	release-vlogscli
-
-publish-victoria-logs-bundle: \
-	publish-victoria-logs \
-	publish-vlogscli
-
 release-victoria-logs:
 	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
 		release-victoria-logs-linux-amd64 \
@@ -313,7 +301,7 @@ release-victoria-logs-windows-amd64:
 
 release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
@@ -330,63 +318,6 @@ release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-logs-windows-$(GOARCH)-prod.exe
 
-release-vlogscli:
-	$(MAKE_PARALLEL) release-vlogscli-linux-386 \
-		release-vlogscli-linux-amd64 \
-		release-vlogscli-linux-arm \
-		release-vlogscli-linux-arm64 \
-		release-vlogscli-darwin-amd64 \
-		release-vlogscli-darwin-arm64 \
-		release-vlogscli-freebsd-amd64 \
-		release-vlogscli-openbsd-amd64 \
-		release-vlogscli-windows-amd64
-
-release-vlogscli-linux-386:
-	GOOS=linux GOARCH=386 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-amd64:
-	GOOS=linux GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-arm:
-	GOOS=linux GOARCH=arm $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-arm64:
-	GOOS=linux GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-darwin-arm64:
-	GOOS=darwin GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-freebsd-amd64:
-	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-openbsd-amd64:
-	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vlogscli-windows-goarch
-
-release-vlogscli-goos-goarch: vlogscli-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vlogscli-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vlogscli-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf vlogscli-$(GOOS)-$(GOARCH)-prod
-
-release-vlogscli-windows-goarch: vlogscli-windows-$(GOARCH)-prod
-	cd bin && \
-		zip vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vlogscli-windows-$(GOARCH)-prod.exe \
-		&& sha256sum vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vlogscli-windows-$(GOARCH)-prod.exe \
-			> vlogscli-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vlogscli-windows-$(GOARCH)-prod.exe
-
 release-vmutils: \
 	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
@@ -400,7 +331,7 @@ release-vmutils: \
 
 release-vmutils-linux-386:
 	GOOS=linux GOARCH=386 $(MAKE) release-vmutils-goos-goarch
-
+	
 release-vmutils-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
@@ -434,7 +365,7 @@ release-vmutils-goos-goarch: \
 	vmrestore-$(GOOS)-$(GOARCH)-prod \
 	vmctl-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
 			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
@@ -511,19 +442,19 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
 	go test -bench=. ./lib/...
@@ -534,9 +465,9 @@ benchmark-pure:
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	go get -u ./lib/...
-	go get -u ./app/...
-	go mod tidy -compat=1.23
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.22
 	go mod vendor
 
 app-local:
@@ -562,7 +493,8 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
+	# The version must match GitHub main.yml lint job "Run golangci-lint" step version.
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`
@@ -581,3 +513,33 @@ install-wwhrd:
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
+
+copy-docs:
+# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
+# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
+	echo "---" > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "sort: ${ORDER}" >> ${DST}; \
+		echo "weight: ${ORDER}" >> ${DST}; \
+		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
+	fi
+
+	echo "title: ${TITLE}" >> ${DST}
+	@if [ ${OLD_URL} ]; then \
+		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
+	fi
+	echo "---" >> ${DST}
+	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	rm -rf docs/*.tmp
+
+# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
+# For ORDER=0 it adds no order tag/weight tag.
+# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
+# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
+# Images starting with <img src="docs/ are replaced with <img src="
+# Cluster docs are supposed to be ordered as 2nd.
+# The rest of docs is ordered manually.
+docs-sync:
+	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
+	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to <security@victoriametrics.com>
+Please report any security issues to security@victoriametrics.com

app/victoria-logs/Makefile

@@ -81,9 +81,6 @@ victoria-logs-linux-ppc64le:
 victoria-logs-linux-s390x:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-victoria-logs-linux-loong64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 victoria-logs-linux-386:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 9428
 
 ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -92,9 +92,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if vlselect.RequestHandler(w, r) {
 		return true
 	}
-	if vlstorage.RequestHandler(w, r) {
-		return true
-	}
 	return false
 }
 

app/victoria-logs/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/victoria-metrics/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 8428
 
 ENTRYPOINT ["/victoria-metrics-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./victoria-metrics-prod

app/victoria-metrics/main_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -118,7 +117,7 @@ func (q *Query) metrics() ([]Metric, error) {
 type QueryInstant struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TV     [2]any            `json:"value"`
+		TV     [2]interface{}    `json:"value"`
 	} `json:"result"`
 }
 
@@ -141,7 +140,7 @@ func (q QueryInstant) metrics() ([]Metric, error) {
 type QueryRange struct {
 	Result []struct {
 		Metric map[string]string `json:"metric"`
-		Values [][]any           `json:"values"`
+		Values [][]interface{}   `json:"values"`
 	} `json:"result"`
 }
 
@@ -250,24 +249,24 @@ func TestWriteRead(t *testing.T) {
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
-		for _, test := range readIn("prometheus", insertionTime) {
+		for _, test := range readIn("prometheus", t, insertionTime) {
 			if test.Data == nil {
 				continue
 			}
+			s := newSuite(t)
 			r := testutil.WriteRequest{}
-			testData := strings.Join(test.Data, "\n")
-			if err := json.Unmarshal([]byte(testData), &r.Timeseries); err != nil {
-				panic(fmt.Errorf("BUG: cannot unmarshal TimeSeries: %s\ntest data\n%s", err, testData))
+			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
+			data, err := testutil.Compress(r)
+			s.greaterThan(len(r.Timeseries), 0)
+			if err != nil {
+				t.Errorf("error compressing %v %s", r, err)
+				t.Fail()
 			}
-			if n := len(r.Timeseries); n <= 0 {
-				panic(fmt.Errorf("BUG: expecting non-empty Timeseries in test data:\n%s", testData))
-			}
-			data := testutil.Compress(r)
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
 	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", insertionTime) {
+		for _, test := range readIn("csv", t, insertionTime) {
 			if test.Data == nil {
 				continue
 			}
@@ -276,7 +275,7 @@ func testWrite(t *testing.T) {
 	})
 
 	t.Run("influxdb", func(t *testing.T) {
-		for _, x := range readIn("influxdb", insertionTime) {
+		for _, x := range readIn("influxdb", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -285,7 +284,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("graphite", func(t *testing.T) {
-		for _, x := range readIn("graphite", insertionTime) {
+		for _, x := range readIn("graphite", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -294,7 +293,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdb", func(t *testing.T) {
-		for _, x := range readIn("opentsdb", insertionTime) {
+		for _, x := range readIn("opentsdb", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -303,7 +302,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdbhttp", func(t *testing.T) {
-		for _, x := range readIn("opentsdbhttp", insertionTime) {
+		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -317,7 +316,7 @@ func testWrite(t *testing.T) {
 func testRead(t *testing.T) {
 	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
-			for _, x := range readIn(engine, insertionTime) {
+			for _, x := range readIn(engine, t, insertionTime) {
 				test := x
 				t.Run(test.Name, func(t *testing.T) {
 					t.Parallel()
@@ -366,10 +365,11 @@ func testRead(t *testing.T) {
 	}
 }
 
-func readIn(readFor string, insertTime time.Time) []test {
-	testDir := filepath.Join(testFixturesDir, readFor)
+func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
+	t.Helper()
+	s := newSuite(t)
 	var tt []test
-	err := filepath.Walk(testDir, func(path string, _ os.FileInfo, err error) error {
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -377,129 +377,84 @@ func readIn(readFor string, insertTime time.Time) []test {
 			return nil
 		}
 		b, err := os.ReadFile(path)
-		if err != nil {
-			panic(fmt.Errorf("BUG: cannot read %s: %s", path, err))
-		}
+		s.noError(err)
 		item := test{}
-		if err := json.Unmarshal(b, &item); err != nil {
-			panic(fmt.Errorf("cannot parse %T from %s: %s; data:\n%s", &item, path, err, b))
-		}
+		s.noError(json.Unmarshal(b, &item))
 		for i := range item.Data {
 			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
 		}
 		tt = append(tt, item)
 		return nil
-	})
-	if err != nil {
-		panic(fmt.Errorf("BUG: cannot read test data at %s: %w", testDir, err))
-	}
+	}))
 	if len(tt) == 0 {
-		panic(fmt.Errorf("BUG: no tests found in %s", testDir))
+		t.Fatalf("no test found in %s", filepath.Join(testFixturesDir, readFor))
 	}
 	return tt
 }
 
 func httpWrite(t *testing.T, address, query string, r io.Reader) {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Post(requestURL, "", r)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
-	_ = resp.Body.Close()
-	if resp.StatusCode != 204 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 204", requestURL, resp.StatusCode)
-	}
+	s := newSuite(t)
+	resp, err := http.Post(address+query, "", r)
+	s.noError(err)
+	s.noError(resp.Body.Close())
+	s.equalInt(resp.StatusCode, 204)
 }
 
-func tcpWrite(t *testing.T, address, data string) {
+func tcpWrite(t *testing.T, address string, data string) {
 	t.Helper()
-
+	s := newSuite(t)
 	conn, err := net.Dial("tcp", address)
-	if err != nil {
-		t.Fatalf("cannot dial %s: %s", address, err)
-	}
+	s.noError(err)
 	defer func() {
 		_ = conn.Close()
 	}()
 	n, err := conn.Write([]byte(data))
-	if err != nil {
-		t.Fatalf("cannot write %d bytes to %s: %s", len(data), address, err)
-	}
-	if n != len(data) {
-		panic(fmt.Errorf("BUG: conn.Write() returned unexpected number of written bytes to %s; got %d; want %d", address, n, len(data)))
-	}
+	s.noError(err)
+	s.equalInt(n, len(data))
 }
 
 func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Get(requestURL)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	if resp.StatusCode != 200 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
-	}
-
+	s.equalInt(resp.StatusCode, 200)
 	var rows []Metric
-	dec := json.NewDecoder(resp.Body)
-	for {
+	for dec := json.NewDecoder(resp.Body); dec.More(); {
 		var row Metric
-		err := dec.Decode(&row)
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				return rows
-			}
-			t.Fatalf("cannot decode %T from response received from %s: %s", &row, requestURL, err)
-		}
+		s.noError(dec.Decode(&row))
 		rows = append(rows, row)
 	}
+	return rows
 }
 
-func httpReadStruct(t *testing.T, address, query string, dst any) {
+func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Get(requestURL)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	if resp.StatusCode != 200 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
-	}
-	err = json.NewDecoder(resp.Body).Decode(dst)
-	if err != nil {
-		t.Fatalf("cannot decode %T from response received from %s: %s", dst, requestURL, err)
-	}
+	s.equalInt(resp.StatusCode, 200)
+	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
 func httpReadData(t *testing.T, address, query string) []byte {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Get(requestURL)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	if resp.StatusCode != 200 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
-	}
+	s.equalInt(resp.StatusCode, 200)
 	data, err := io.ReadAll(resp.Body)
-	if err != nil {
-		t.Fatalf("cannot read response from %s: %s", requestURL, err)
-	}
+	s.noError(err)
 	return data
 }
 
@@ -548,6 +503,34 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
+type suite struct{ t *testing.T }
+
+func newSuite(t *testing.T) *suite { return &suite{t: t} }
+
+func (s *suite) noError(err error) {
+	s.t.Helper()
+	if err != nil {
+		s.t.Errorf("unexpected error %v", err)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) equalInt(a, b int) {
+	s.t.Helper()
+	if a != b {
+		s.t.Errorf("%d not equal %d", a, b)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) greaterThan(a, b int) {
+	s.t.Helper()
+	if a <= b {
+		s.t.Errorf("%d less or equal then %d", a, b)
+		s.t.FailNow()
+	}
+}
+
 func TestImportJSONLines(t *testing.T) {
 	f := func(labelsCount, labelLen int) {
 		t.Helper()

app/victoria-metrics/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/victoria-metrics/test/prom_writer.go

@@ -1,16 +1,12 @@
 package test
 
-import (
-	"fmt"
-
-	"github.com/golang/snappy"
-)
+import "github.com/golang/snappy"
 
 // Compress marshals and compresses wr.
-func Compress(wr WriteRequest) []byte {
+func Compress(wr WriteRequest) ([]byte, error) {
 	data, err := wr.Marshal()
 	if err != nil {
-		panic(fmt.Errorf("BUG: cannot compress WriteRequest: %s", err))
+		return nil, err
 	}
-	return snappy.Encode(nil, data)
+	return snappy.Encode(nil, data), nil
 }

app/vlinsert/elasticsearch/elasticsearch.go

@@ -6,7 +6,9 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -55,12 +57,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
-		// Return fake response for Logstash APIs requests.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
 	switch path {
 	case "/":
 		switch r.Method {
@@ -249,8 +245,19 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
 		return 0, nil
 	}
 	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in seconds or milliseconds
-		return insertutils.ParseUnixTimestamp(s)
+		// Try parsing timestamp in milliseconds
+		n, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
+		}
+		if n > int64(math.MaxInt64)/1e6 {
+			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+		}
+		if n < int64(math.MinInt64)/1e6 {
+			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+		}
+		n *= 1e6
+		return n, nil
 	}
 	if len(s) == len("YYYY-MM-DD") {
 		t, err := time.Parse("2006-01-02", s)

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -76,20 +76,17 @@ func TestReadBulkRequest_Success(t *testing.T) {
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
 {"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06 04:48:12.735+01:00","message":"baz"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
 {"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"1686026893735","x":"y"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty","@timestamp":"1686026893"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
-	rowsExpected := 4
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}
-{"_msg":"qwe rty"}`
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlinsert/insertutils/common_params.go

@@ -2,7 +2,6 @@ package insertutils
 
 import (
 	"net/http"
-	"strings"
 	"sync"
 	"time"
 
@@ -39,46 +38,22 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		return nil, err
 	}
 
-	// Extract time field name from _time_field query arg or header
-	timeField := "_time"
+	// Extract time field name from _time_field query arg
+	var timeField = "_time"
 	if tf := r.FormValue("_time_field"); tf != "" {
 		timeField = tf
-	} else if tf = r.Header.Get("VL-Time-Field"); tf != "" {
-		timeField = tf
 	}
 
-	// Extract message field name from _msg_field query arg or header
-	msgField := ""
+	// Extract message field name from _msg_field query arg
+	var msgField = ""
 	if msgf := r.FormValue("_msg_field"); msgf != "" {
 		msgField = msgf
-	} else if msgf = r.Header.Get("VL-Msg-Field"); msgf != "" {
-		msgField = msgf
 	}
 
 	streamFields := httputils.GetArray(r, "_stream_fields")
-	if len(streamFields) == 0 {
-		if sf := r.Header.Get("VL-Stream-Fields"); len(sf) > 0 {
-			streamFields = strings.Split(sf, ",")
-		}
-	}
 	ignoreFields := httputils.GetArray(r, "ignore_fields")
-	if len(ignoreFields) == 0 {
-		if f := r.Header.Get("VL-Ignore-Fields"); len(f) > 0 {
-			ignoreFields = strings.Split(f, ",")
-		}
-	}
 
 	debug := httputils.GetBool(r, "debug")
-	if !debug {
-		if dh := r.Header.Get("VL-Debug"); len(dh) > 0 {
-			hv := strings.ToLower(dh)
-			switch hv {
-			case "", "0", "f", "false", "no":
-			default:
-				debug = true
-			}
-		}
-	}
 	debugRequestURI := ""
 	debugRemoteAddr := ""
 	if debug {
@@ -96,7 +71,6 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		DebugRequestURI: debugRequestURI,
 		DebugRemoteAddr: debugRemoteAddr,
 	}
-
 	return cp, nil
 }
 
@@ -176,27 +150,11 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Fiel
 		return
 	}
 
-	// _msg field must be non-empty according to VictoriaLogs data model.
-	// See https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field
-	msgExist := false
-	for i := range fields {
-		if fields[i].Name == "_msg" {
-			msgExist = len(fields[i].Value) > 0
-			break
-		}
-	}
-	if !msgExist {
-		rf := logstorage.RowFormatter(fields)
-		logger.Warnf("dropping log line without _msg field; %s", rf)
-		rowsDroppedTotalMsgNotValid.Inc()
-		return
-	}
-
 	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
 	if lmp.cp.Debug {
 		s := lmp.lr.GetRowString(0)
 		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
 		rowsDroppedTotalDebug.Inc()
 		return
 	}
@@ -238,8 +196,5 @@ func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
 	return lmp
 }
 
-var (
-	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
-	rowsDroppedTotalMsgNotValid   = metrics.NewCounter(`vl_rows_dropped_total{reason="msg_not_exist"}`)
-)
+var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/timestamp.go

@@ -2,8 +2,6 @@ package insertutils
 
 import (
 	"fmt"
-	"math"
-	"strconv"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -21,49 +19,15 @@ func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage
 		if f.Name != timeField {
 			continue
 		}
-		nsecs, err := parseTimestamp(f.Value)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", timeField, err)
+		if f.Value == "" || f.Value == "0" {
+			return time.Now().UnixNano(), nil
+		}
+		nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
+		if !ok {
+			return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
 		}
 		f.Value = ""
-		if nsecs == 0 {
-			nsecs = time.Now().UnixNano()
-		}
 		return nsecs, nil
 	}
 	return time.Now().UnixNano(), nil
 }
-
-func parseTimestamp(s string) (int64, error) {
-	if s == "" || s == "0" {
-		return time.Now().UnixNano(), nil
-	}
-	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
-		return ParseUnixTimestamp(s)
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
-	}
-	return nsecs, nil
-}
-
-// ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.
-func ParseUnixTimestamp(s string) (int64, error) {
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-	}
-	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds. Convert it to milliseconds
-		n *= 1e3
-	}
-	if n > int64(math.MaxInt64)/1e6 {
-		return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
-	}
-	if n < int64(math.MinInt64)/1e6 {
-		return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
-	}
-	n *= 1e6
-	return n, nil
-}

app/vlinsert/insertutils/timestamp_test.go

@@ -27,41 +27,25 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 		}
 	}
 
-	// UTC time
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
 		{Name: "time", Value: "2024-06-18T23:37:20Z"},
 	}, 1718753840000000000)
 
-	// Time with timezone
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
 		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
 	}, 1718725040000000000)
 
-	// SQL datetime format
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
+		{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
 	}, 1718773640123000000)
 
-	// Time with nanosecond precision
 	f("time", []logstorage.Field{
 		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
 		{Name: "foo", Value: "bar"},
 	}, 1718773640123456789)
-
-	// Unix timestamp in milliseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123"},
-	}, 1718773640123000000)
-
-	// Unix timestamp in seconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640"},
-	}, 1718773640000000000)
 }
 
 func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
@@ -82,6 +66,9 @@ func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
 
 	f("foobar")
 
+	// no Z at the end
+	f("2024-06-18T23:37:20")
+
 	// incomplete time
 	f("2024-06-18")
 	f("2024-06-18T23:37")

app/vlinsert/jsonline/jsonline_test.go

@@ -23,16 +23,16 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	}
 
 	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06 04:48:13.735Z","x":"y"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
 	rowsExpected := 3
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}`
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlinsert/loki/loki_protobuf.go

@@ -79,14 +79,12 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err = req.UnmarshalProtobuf(bb.B)
+	err = req.Unmarshal(bb.B)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
-	fields := getFields()
-	defer putFields(fields)
-
+	var commonFields []logstorage.Field
 	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
@@ -94,60 +92,30 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
 		// Labels are same for all entries in the stream.
-		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
+		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
 		if err != nil {
 			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
-		commonFieldsLen := len(fields.fields)
+		fields := commonFields
 
 		entries := stream.Entries
 		for j := range entries {
-			e := &entries[j]
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			for _, lp := range e.StructuredMetadata {
-				fields.fields = append(fields.fields, logstorage.Field{
-					Name:  lp.Name,
-					Value: lp.Value,
-				})
-			}
-
-			fields.fields = append(fields.fields, logstorage.Field{
+			entry := &entries[j]
+			fields = append(fields[:len(commonFields)], logstorage.Field{
 				Name:  "_msg",
-				Value: e.Line,
+				Value: entry.Line,
 			})
-
-			ts := e.Timestamp.UnixNano()
+			ts := entry.Timestamp.UnixNano()
 			if ts == 0 {
 				ts = currentTimestamp
 			}
-
-			lmp.AddRow(ts, fields.fields)
+			lmp.AddRow(ts, fields)
 		}
 		rowsIngested += len(stream.Entries)
 	}
 	return rowsIngested, nil
 }
 
-func getFields() *fields {
-	v := fieldsPool.Get()
-	if v == nil {
-		return &fields{}
-	}
-	return v.(*fields)
-}
-
-func putFields(f *fields) {
-	f.fields = f.fields[:0]
-	fieldsPool.Put(f)
-}
-
-var fieldsPool sync.Pool
-
-type fields struct {
-	fields []logstorage.Field
-}
-
 // parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
 //
 // See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
@@ -213,6 +181,6 @@ func getPushRequest() *PushRequest {
 }
 
 func putPushRequest(req *PushRequest) {
-	req.reset()
+	req.Reset()
 	pushReqsPool.Put(req)
 }

app/vlinsert/loki/loki_protobuf_test.go

@@ -36,7 +36,7 @@ func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.
 		Entries: []Entry{
 			{
 				Timestamp: time.Unix(0, timestamp),
-				Line:      strings.Clone(msg),
+				Line:      msg,
 			},
 		},
 	})
@@ -58,7 +58,10 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
 		}
 
-		data := tlp.pr.MarshalProtobuf(nil)
+		data, err := tlp.pr.Marshal()
+		if err != nil {
+			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
+		}
 		encodedData := snappy.Encode(nil, data)
 
 		tlp2 := &insertutils.TestLogMessageProcessor{}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkParseProtobufRequest(b *testing.B) {
@@ -39,47 +38,29 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	})
 }
 
-func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
-	var b []byte
-	var entries []Entry
-	streams := make([]Stream, streamsCount)
-	for i := range streams {
-		b = b[:0]
-		b = append(b, '{')
-		for j := 0; j < labelsCount; j++ {
-			b = append(b, "label_"...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, `="value_`...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, '"')
-			if j < labelsCount-1 {
-				b = append(b, ',')
+func getProtobufBody(streams, rows, labels int) []byte {
+	var pr PushRequest
+
+	for i := 0; i < streams; i++ {
+		var st Stream
+
+		st.Labels = `{`
+		for j := 0; j < labels; j++ {
+			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
+			if j < labels-1 {
+				st.Labels += `,`
 			}
 		}
-		b = append(b, '}')
-		labels := string(b)
+		st.Labels += `}`
 
-		var rowsBuf []byte
-		entriesLen := len(entries)
-		for j := 0; j < rowsCount; j++ {
-			rowsBufLen := len(rowsBuf)
-			rowsBuf = append(rowsBuf, "value_"...)
-			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
-			entries = append(entries, Entry{
-				Timestamp: time.Now(),
-				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
-			})
+		for j := 0; j < rows; j++ {
+			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
 		}
 
-		st := &streams[i]
-		st.Labels = labels
-		st.Entries = entries[entriesLen:]
-	}
-	pr := PushRequest{
-		Streams: streams,
+		pr.Streams = append(pr.Streams, st)
 	}
 
-	body := pr.MarshalProtobuf(nil)
+	body, _ := pr.Marshal()
 	encodedBody := snappy.Encode(nil, body)
 
 	return encodedBody

app/vlinsert/loki/pb.go

@@ -1,302 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: push_request.proto
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package loki
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// PushRequest represents Loki PushRequest
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
-type PushRequest struct {
-	Streams []Stream
-
-	entriesBuf   []Entry
-	labelPairBuf []LabelPair
-}
-
-func (pr *PushRequest) reset() {
-	pr.Streams = pr.Streams[:0]
-
-	pr.entriesBuf = pr.entriesBuf[:0]
-	pr.labelPairBuf = pr.labelPairBuf[:0]
-}
-
-// UnmarshalProtobuf unmarshals pr from protobuf message at src.
-//
-// pr remains valid until src is modified.
-func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
-	pr.reset()
-	var err error
-	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
-	return err
-}
-
-// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
-func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	pr.marshalProtobuf(m.MessageMarshaler())
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, s := range pr.Streams {
-		s.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message PushRequest {
-	//   repeated Stream streams = 1;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
-			}
-			pr.Streams = append(pr.Streams, Stream{})
-			s := &pr.Streams[len(pr.Streams)-1]
-			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
-			}
-		}
-	}
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Stream represents Loki stream.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
-type Stream struct {
-	Labels  string
-	Entries []Entry
-}
-
-func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, s.Labels)
-	for _, e := range s.Entries {
-		e.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message Stream {
-	//   string labels = 1;
-	//   repeated Entry entries = 2;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	entriesBufLen := len(entriesBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			labels, ok := fc.String()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
-			}
-			s.Labels = labels
-		case 2:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
-			}
-			entriesBuf = append(entriesBuf, Entry{})
-			e := &entriesBuf[len(entriesBuf)-1]
-			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
-			}
-		}
-	}
-	s.Entries = entriesBuf[entriesBufLen:]
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Entry represents Loki entry.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
-type Entry struct {
-	Timestamp          time.Time
-	Line               string
-	StructuredMetadata []LabelPair
-}
-
-func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	marshalTime(mm, 1, e.Timestamp)
-	mm.AppendString(2, e.Line)
-	for _, lp := range e.StructuredMetadata {
-		lp.marshalProtobuf(mm.AppendMessage(3))
-	}
-}
-
-func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
-	// message Entry {
-	//   Timestamp timestamp = 1;
-	//   string line = 2;
-	//   repeated LabelPair structuredMetadata = 3;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	labelPairBufLen := len(labelPairBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
-			}
-			timestamp, err := unmarshalTime(data)
-			if err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
-			}
-			e.Timestamp = timestamp
-		case 2:
-			line, ok := fc.String()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Line")
-			}
-			e.Line = line
-		case 3:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
-			}
-			labelPairBuf = append(labelPairBuf, LabelPair{})
-			lp := &labelPairBuf[len(labelPairBuf)-1]
-			if err := lp.unmarshalProtobuf(data); err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
-			}
-		}
-	}
-	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
-	return labelPairBuf, nil
-}
-
-// LabelPair represents Loki label pair.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
-type LabelPair struct {
-	Name  string
-	Value string
-}
-
-func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, lp.Name)
-	mm.AppendString(2, lp.Value)
-}
-
-func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
-	// message LabelPair {
-	//   string name = 1;
-	//   string value = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			name, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot read name")
-			}
-			lp.Name = name
-		case 2:
-			value, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot unmarshal value")
-			}
-			lp.Value = value
-		}
-	}
-	return nil
-}
-
-func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
-	nsecs := timestamp.UnixNano()
-	ts := Timestamp{
-		Seconds: nsecs / 1e9,
-		Nanos:   int32(nsecs % 1e9),
-	}
-	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
-}
-
-func unmarshalTime(src []byte) (time.Time, error) {
-	var ts Timestamp
-	if err := ts.unmarshalProtobuf(src); err != nil {
-		return time.Time{}, err
-	}
-	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	return timestamp, nil
-}
-
-// Timestamp is protobuf well-known timestamp type.
-type Timestamp struct {
-	Seconds int64
-	Nanos   int32
-}
-
-func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendInt64(1, ts.Seconds)
-	mm.AppendInt32(2, ts.Nanos)
-}
-
-func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
-	// message Timestamp {
-	//   int64 seconds = 1;
-	//   int32 nanos = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			seconds, ok := fc.Int64()
-			if !ok {
-				return fmt.Errorf("cannot read Seconds")
-			}
-			ts.Seconds = seconds
-		case 2:
-			nanos, ok := fc.Int32()
-			if !ok {
-				return fmt.Errorf("cannot read Nanos")
-			}
-			ts.Nanos = nanos
-		}
-	}
-	return nil
-}

app/vlinsert/loki/push_request.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package logproto;
+
+import "gogoproto/gogo.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
+
+message PushRequest {
+  repeated StreamAdapter streams = 1 [
+    (gogoproto.jsontag) = "streams",
+    (gogoproto.customtype) = "Stream"
+  ];
+}
+
+message StreamAdapter {
+  string labels = 1 [(gogoproto.jsontag) = "labels"];
+  repeated EntryAdapter entries = 2 [
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "entries"
+  ];
+  // hash contains the original hash of the stream.
+  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
+}
+
+message EntryAdapter {
+  google.protobuf.Timestamp timestamp = 1 [
+    (gogoproto.stdtime) = true,
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "ts"
+  ];
+  string line = 2 [(gogoproto.jsontag) = "line"];
+}

app/vlinsert/loki/timestamp.go

@@ -0,0 +1,110 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+)
+
+const (
+	// Seconds field of the earliest valid Timestamp.
+	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	minValidSeconds = -62135596800
+	// Seconds field just after the latest valid Timestamp.
+	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	maxValidSeconds = 253402300800
+)
+
+// validateTimestamp determines whether a Timestamp is valid.
+// A valid timestamp represents a time in the range
+// [0001-01-01, 10000-01-01) and has a Nanos field
+// in the range [0, 1e9).
+//
+// If the Timestamp is valid, validateTimestamp returns nil.
+// Otherwise, it returns an error that describes
+// the problem.
+//
+// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
+func validateTimestamp(ts *types.Timestamp) error {
+	if ts == nil {
+		return errors.New("timestamp: nil Timestamp")
+	}
+	if ts.Seconds < minValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
+	}
+	if ts.Seconds >= maxValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
+	}
+	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
+		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
+	}
+	return nil
+}
+
+// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
+// but avoids the escape incurred by using fmt.Sprintf, eliminating
+// unnecessary heap allocations.
+func formatTimestamp(ts *types.Timestamp) string {
+	if ts == nil {
+		return "nil"
+	}
+
+	seconds := strconv.FormatInt(ts.Seconds, 10)
+	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
+	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
+}
+
+func sizeOfStdTime(t time.Time) int {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0
+	}
+	return ts.Size()
+}
+
+func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0, err
+	}
+	return ts.MarshalTo(data)
+}
+
+func stdTimeUnmarshal(t *time.Time, data []byte) error {
+	ts := &types.Timestamp{}
+	if err := ts.Unmarshal(data); err != nil {
+		return err
+	}
+	tt, err := timestampFromProto(ts)
+	if err != nil {
+		return err
+	}
+	*t = tt
+	return nil
+}
+
+func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
+	// Don't return the zero value on error, because corresponds to a valid
+	// timestamp. Instead return whatever time.Unix gives us.
+	var t time.Time
+	if ts == nil {
+		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
+	} else {
+		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	}
+	return t, validateTimestamp(ts)
+}
+
+func timestampProto(t time.Time) (types.Timestamp, error) {
+	ts := types.Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.Nanosecond()),
+	}
+	return ts, validateTimestamp(&ts)
+}

app/vlinsert/loki/types.go

@@ -0,0 +1,481 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"fmt"
+	"io"
+	"time"
+)
+
+// Stream contains a unique labels set as a string and a set of entries for it.
+// We are not using the proto generated version but this custom one so that we
+// can improve serialization see benchmark.
+type Stream struct {
+	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
+	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
+	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
+}
+
+// Entry is a log entry with a timestamp.
+type Entry struct {
+	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
+	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Stream) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Hash != 0 {
+		i = encodeVarintPush(dAtA, i, m.Hash)
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Entries) > 0 {
+		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintPush(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		i -= len(m.Labels)
+		copy(dAtA[i:], m.Labels)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Entry) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Line) > 0 {
+		i -= len(m.Line)
+		copy(dAtA[i:], m.Line)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
+		i--
+		dAtA[i] = 0x12
+	}
+	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
+	if err7 != nil {
+		return 0, err7
+	}
+	i -= n7
+	i = encodeVarintPush(dAtA, i, uint64(n7))
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Stream) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Labels = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Entries = append(m.Entries, Entry{})
+			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
+			}
+			m.Hash = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Hash |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Entry) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Line = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Size returns the size of the serialized Stream.
+func (m *Stream) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Labels)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	if len(m.Entries) > 0 {
+		for _, e := range m.Entries {
+			l = e.Size()
+			n += 1 + l + sovPush(uint64(l))
+		}
+	}
+	if m.Hash != 0 {
+		n += 1 + sovPush(m.Hash)
+	}
+	return n
+}
+
+// Size returns the size of the serialized Entry
+func (m *Entry) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = sizeOfStdTime(m.Timestamp)
+	n += 1 + l + sovPush(uint64(l))
+	l = len(m.Line)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	return n
+}
+
+// Equal returns true if the two Streams are equal.
+func (m *Stream) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Stream)
+	if !ok {
+		that2, ok := that.(Stream)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if m.Labels != that1.Labels {
+		return false
+	}
+	if len(m.Entries) != len(that1.Entries) {
+		return false
+	}
+	for i := range m.Entries {
+		if !m.Entries[i].Equal(that1.Entries[i]) {
+			return false
+		}
+	}
+	return m.Hash == that1.Hash
+}
+
+// Equal returns true if the two Entries are equal.
+func (m *Entry) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Entry)
+	if !ok {
+		that2, ok := that.(Entry)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if !m.Timestamp.Equal(that1.Timestamp) {
+		return false
+	}
+	if m.Line != that1.Line {
+		return false
+	}
+	return true
+}

app/vlinsert/main.go

@@ -7,7 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
 )
 
@@ -42,9 +41,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case strings.HasPrefix(path, "/loki/"):
 		path = strings.TrimPrefix(path, "/loki")
 		return loki.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/opentelemetry/"):
-		path = strings.TrimPrefix(path, "/opentelemetry")
-		return opentelemetry.RequestHandler(path, w, r)
 	default:
 		return false
 	}

app/vlinsert/opentelemetry/opentelemetry.go

@@ -1,143 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// RequestHandler processes Opentelemetry insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	// use the same path as opentelemetry collector
-	// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
-	case "/v1/logs":
-		if r.Header.Get("Content-Type") == "application/json" {
-			httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-			return true
-		}
-		handleProtobuf(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
-
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	lmp := cp.NewLogMessageProcessor()
-	n, err := pushProtobufRequest(data, lmp)
-	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
-		return
-	}
-
-	rowsIngestedProtobufTotal.Add(n)
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-}
-
-var (
-	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="opentelemetry",format="protobuf"}`)
-
-	requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-
-	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-)
-
-func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
-	var req pb.ExportLogsServiceRequest
-	if err := req.UnmarshalProtobuf(data); err != nil {
-		errorsTotal.Inc()
-		return 0, fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
-	}
-
-	var rowsIngested int
-	var commonFields []logstorage.Field
-	for _, rl := range req.ResourceLogs {
-		attributes := rl.Resource.Attributes
-		commonFields = slicesutil.SetLength(commonFields, len(attributes))
-		for i, attr := range attributes {
-			commonFields[i].Name = attr.Key
-			commonFields[i].Value = attr.Value.FormatString()
-		}
-		commonFieldsLen := len(commonFields)
-		for _, sc := range rl.ScopeLogs {
-			var scopeIngested int
-			commonFields, scopeIngested = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp)
-			rowsIngested += scopeIngested
-		}
-	}
-
-	return rowsIngested, nil
-}
-
-func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutils.LogMessageProcessor) ([]logstorage.Field, int) {
-	fields := commonFields
-	for _, lr := range sc.LogRecords {
-		fields = fields[:len(commonFields)]
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: lr.Body.FormatString(),
-		})
-		for _, attr := range lr.Attributes {
-			fields = append(fields, logstorage.Field{
-				Name:  attr.Key,
-				Value: attr.Value.FormatString(),
-			})
-		}
-		fields = append(fields, logstorage.Field{
-			Name:  "severity",
-			Value: lr.FormatSeverity(),
-		})
-
-		lmp.AddRow(lr.ExtractTimestampNano(), fields)
-	}
-	return fields, len(sc.LogRecords)
-}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -1,128 +0,0 @@
-package opentelemetry
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func TestPushProtoOk(t *testing.T) {
-	f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-		lr := pb.ExportLogsServiceRequest{
-			ResourceLogs: src,
-		}
-
-		pData := lr.MarshalProtobuf(nil)
-		tlp := &insertutils.TestLogMessageProcessor{}
-		n, err := pushProtobufRequest(pData, tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-	// single line without resource attributes
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234},
-		`{"_msg":"log-line-message","severity":"Trace"}`,
-	)
-	// multi-line with resource attributes
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
-	)
-
-	// multi-scope with resource attributes and multi-line
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
-						{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
-					},
-				},
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
-						{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 2345, 2346, 2347, 2348},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
-{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
-	)
-}
-
-func ptrTo[T any](s T) *T {
-	return &s
-}

app/vlinsert/opentelemetry/opentemetry_timing_test.go

@@ -1,79 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, scopes := range []int{1, 2} {
-		for _, rows := range []int{100, 1000} {
-			for _, attributes := range []int{5, 10} {
-				b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, scopes, rows, attributes)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			_, err := pushProtobufRequest(body, blp)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
-	msg := "12345678910"
-
-	attrValues := []*pb.AnyValue{
-		{StringValue: ptrTo("string-attribute")},
-		{IntValue: ptrTo[int64](12345)},
-		{DoubleValue: ptrTo(3.14)},
-	}
-	attrs := make([]*pb.KeyValue, attributesCount)
-	for j := 0; j < attributesCount; j++ {
-		attrs[j] = &pb.KeyValue{
-			Key:   fmt.Sprintf("key-%d", j),
-			Value: attrValues[j%3],
-		}
-	}
-	entries := make([]pb.LogRecord, rowsCount)
-	for j := 0; j < rowsCount; j++ {
-		entries[j] = pb.LogRecord{
-			TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
-		}
-	}
-	scopes := make([]pb.ScopeLogs, scopesCount)
-
-	for j := 0; j < scopesCount; j++ {
-		scopes[j] = pb.ScopeLogs{
-			LogRecords: entries,
-		}
-	}
-
-	pr := pb.ExportLogsServiceRequest{
-		ResourceLogs: []pb.ResourceLogs{
-			{
-				Resource: pb.Resource{
-					Attributes: attrs,
-				},
-				ScopeLogs: scopes,
-			},
-		},
-	}
-
-	return pr.MarshalProtobuf(nil)
-}

app/vlinsert/syslog/syslog_test.go

@@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	currentYear := 2023
 	rowsExpected := 3
 	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
-	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
 	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlogscli/Makefile

@@ -1,109 +0,0 @@
-# All these commands must run from repository root.
-
-vlogscli:
-	APP_NAME=vlogscli $(MAKE) app-local
-
-vlogscli-race:
-	APP_NAME=vlogscli RACE=-race $(MAKE) app-local
-
-vlogscli-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker
-
-vlogscli-pure-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-pure
-
-vlogscli-linux-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-amd64
-
-vlogscli-linux-arm-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm
-
-vlogscli-linux-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm64
-
-vlogscli-linux-ppc64le-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-ppc64le
-
-vlogscli-linux-386-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-386
-
-vlogscli-darwin-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-amd64
-
-vlogscli-darwin-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-arm64
-
-vlogscli-freebsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-freebsd-amd64
-
-vlogscli-openbsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-openbsd-amd64
-
-vlogscli-windows-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-windows-amd64
-
-package-vlogscli:
-	APP_NAME=vlogscli $(MAKE) package-via-docker
-
-package-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-pure
-
-package-vlogscli-amd64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-amd64
-
-package-vlogscli-arm:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm
-
-package-vlogscli-arm64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm64
-
-package-vlogscli-ppc64le:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-ppc64le
-
-package-vlogscli-386:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-386
-
-publish-vlogscli:
-	APP_NAME=vlogscli $(MAKE) publish-via-docker
-
-vlogscli-linux-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-ppc64le:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-s390x:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-loong64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-386:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-freebsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-openbsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-windows-amd64:
-	GOARCH=amd64 APP_NAME=vlogscli $(MAKE) app-local-windows-goarch
-
-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) app-local-pure
-
-run-vlogscli:
-	APP_NAME=vlogscli $(MAKE) run-via-docker

app/vlogscli/README.md

@@ -1,5 +0,0 @@
-# vlogscli
-
-Command-line utility for querying [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).

app/vlogscli/deployment/Dockerfile

@@ -1,6 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-ENTRYPOINT ["/vlogscli-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vlogscli-prod

app/vlogscli/json_prettifier.go

@@ -1,238 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io"
-	"sort"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-type outputMode int
-
-const (
-	outputModeJSONMultiline  = outputMode(0)
-	outputModeJSONSingleline = outputMode(1)
-	outputModeLogfmt         = outputMode(2)
-	outputModeCompact        = outputMode(3)
-)
-
-func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
-	switch outputMode {
-	case outputModeJSONMultiline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, true)
-		}
-	case outputModeJSONSingleline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, false)
-		}
-	case outputModeLogfmt:
-		return writeLogfmtObject
-	case outputModeCompact:
-		return writeCompactObject
-	default:
-		panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
-	}
-}
-
-type jsonPrettifier struct {
-	r         io.ReadCloser
-	formatter func(w io.Writer, fields []logstorage.Field) error
-
-	d *json.Decoder
-
-	pr *io.PipeReader
-	pw *io.PipeWriter
-	bw *bufio.Writer
-
-	wg sync.WaitGroup
-}
-
-func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
-	d := json.NewDecoder(r)
-	pr, pw := io.Pipe()
-	bw := bufio.NewWriter(pw)
-
-	formatter := getOutputFormatter(outputMode)
-
-	jp := &jsonPrettifier{
-		r:         r,
-		formatter: formatter,
-
-		d: d,
-
-		pr: pr,
-		pw: pw,
-		bw: bw,
-	}
-
-	jp.wg.Add(1)
-	go func() {
-		defer jp.wg.Done()
-		err := jp.prettifyJSONLines()
-		jp.closePipesWithError(err)
-	}()
-
-	return jp
-}
-
-func (jp *jsonPrettifier) closePipesWithError(err error) {
-	_ = jp.pr.CloseWithError(err)
-	_ = jp.pw.CloseWithError(err)
-}
-
-func (jp *jsonPrettifier) prettifyJSONLines() error {
-	for jp.d.More() {
-		fields, err := readNextJSONObject(jp.d)
-		if err != nil {
-			return err
-		}
-		sort.Slice(fields, func(i, j int) bool {
-			return fields[i].Name < fields[j].Name
-		})
-		if err := jp.formatter(jp.bw, fields); err != nil {
-			return err
-		}
-
-		// Flush bw after every output line in order to show results as soon as they appear.
-		if err := jp.bw.Flush(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (jp *jsonPrettifier) Close() error {
-	jp.closePipesWithError(io.ErrUnexpectedEOF)
-	err := jp.r.Close()
-	jp.wg.Wait()
-	return err
-}
-
-func (jp *jsonPrettifier) Read(p []byte) (int, error) {
-	return jp.pr.Read(p)
-}
-
-func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
-	t, err := d.Token()
-	if err != nil {
-		return nil, fmt.Errorf("cannot read '{': %w", err)
-	}
-	delim, ok := t.(json.Delim)
-	if !ok || delim.String() != "{" {
-		return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
-	}
-
-	var fields []logstorage.Field
-	for {
-		// Read object key
-		t, err := d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object key or closing brace: %w", err)
-		}
-		delim, ok := t.(json.Delim)
-		if ok {
-			if delim.String() == "}" {
-				return fields, nil
-			}
-			return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
-		}
-		key, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object key: %v; want string or '}'", t)
-		}
-
-		// read object value
-		t, err = d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object value: %w", err)
-		}
-		value, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
-		}
-
-		fields = append(fields, logstorage.Field{
-			Name:  key,
-			Value: value,
-		})
-	}
-}
-
-func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
-	data := logstorage.MarshalFieldsToLogfmt(nil, fields)
-	_, err := fmt.Fprintf(w, "%s\n", data)
-	return err
-}
-
-func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
-	if len(fields) == 1 {
-		// Just write field value as is without name
-		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
-		return err
-	}
-	if len(fields) == 2 && fields[0].Name == "_time" || fields[1].Name == "_time" {
-		// Write _time\tfieldValue as is
-		if fields[0].Name == "_time" {
-			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)
-			return err
-		}
-		_, err := fmt.Fprintf(w, "%s\t%s\n", fields[1].Value, fields[0].Value)
-		return err
-	}
-
-	// Fall back to logfmt
-	return writeLogfmtObject(w, fields)
-}
-
-func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
-	if len(fields) == 0 {
-		fmt.Fprintf(w, "{}\n")
-		return nil
-	}
-
-	fmt.Fprintf(w, "{")
-	writeNewlineIfNeeded(w, isMultiline)
-	if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
-		return err
-	}
-	for _, f := range fields[1:] {
-		fmt.Fprintf(w, ",")
-		writeNewlineIfNeeded(w, isMultiline)
-		if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
-			return err
-		}
-	}
-	writeNewlineIfNeeded(w, isMultiline)
-	fmt.Fprintf(w, "}\n")
-	return nil
-}
-
-func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
-	if isMultiline {
-		fmt.Fprintf(w, "\n")
-	}
-}
-
-func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
-	key := getJSONString(f.Name)
-	value := getJSONString(f.Value)
-	if isMultiline {
-		_, err := fmt.Fprintf(w, "  %s: %s", key, value)
-		return err
-	}
-	_, err := fmt.Fprintf(w, "%s:%s", key, value)
-	return err
-}
-
-func getJSONString(s string) string {
-	data, err := json.Marshal(s)
-	if err != nil {
-		panic(fmt.Errorf("unexpected error when marshaling string to JSON: %w", err))
-	}
-	return string(data)
-}

app/vlogscli/less_wrapper.go

@@ -1,116 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-
-	"github.com/mattn/go-isatty"
-)
-
-func isTerminal() bool {
-	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
-}
-
-func readWithLess(r io.Reader) error {
-	if !isTerminal() {
-		// Just write everything to stdout if no terminal is available.
-		_, err := io.Copy(os.Stdout, r)
-		if err != nil && !isErrPipe(err) {
-			return fmt.Errorf("error when forwarding data to stdout: %w", err)
-		}
-		if err := os.Stdout.Sync(); err != nil {
-			return fmt.Errorf("cannot sync data to stdout: %w", err)
-		}
-		return nil
-	}
-
-	pr, pw, err := os.Pipe()
-	if err != nil {
-		return fmt.Errorf("cannot create pipe: %w", err)
-	}
-	defer func() {
-		_ = pr.Close()
-		_ = pw.Close()
-	}()
-
-	// Ignore Ctrl+C in the current process, so 'less' could handle it properly
-	cancel := ignoreSignals(os.Interrupt)
-	defer cancel()
-
-	// Start 'less' process
-	path, err := exec.LookPath("less")
-	if err != nil {
-		return fmt.Errorf("cannot find 'less' command: %w", err)
-	}
-	p, err := os.StartProcess(path, []string{"less", "-F", "-X"}, &os.ProcAttr{
-		Env:   append(os.Environ(), "LESSCHARSET=utf-8"),
-		Files: []*os.File{pr, os.Stdout, os.Stderr},
-	})
-	if err != nil {
-		return fmt.Errorf("cannot start 'less' process: %w", err)
-	}
-
-	// Close pr after 'less' finishes in a parallel goroutine
-	// in order to unblock forwarding data to stopped 'less' below.
-	waitch := make(chan *os.ProcessState)
-	go func() {
-		// Wait for 'less' process to finish.
-		ps, err := p.Wait()
-		if err != nil {
-			fatalf("unexpected error when waiting for 'less' process: %w", err)
-		}
-		_ = pr.Close()
-		waitch <- ps
-	}()
-
-	// Forward data from r to 'less'
-	_, err = io.Copy(pw, r)
-	_ = pw.Sync()
-	_ = pw.Close()
-
-	// Wait until 'less' finished
-	ps := <-waitch
-
-	// Verify 'less' status.
-	if !ps.Success() {
-		return fmt.Errorf("'less' finished with unexpected code %d", ps.ExitCode())
-	}
-
-	if err != nil && !isErrPipe(err) {
-		return fmt.Errorf("error when forwarding data to 'less': %w", err)
-	}
-
-	return nil
-}
-
-func isErrPipe(err error) bool {
-	return errors.Is(err, syscall.EPIPE) || errors.Is(err, io.ErrClosedPipe)
-}
-
-func ignoreSignals(sigs ...os.Signal) func() {
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, sigs...)
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for {
-			_, ok := <-ch
-			if !ok {
-				return
-			}
-		}
-	}()
-	return func() {
-		signal.Stop(ch)
-		close(ch)
-		wg.Wait()
-	}
-}

app/vlogscli/main.go

@@ -1,420 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strconv"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/ergochat/readline"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
-		"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs . See also -tail.url")
-	tailURL = flag.String("tail.url", "", "URL for live tailing queries to VictoriaLogs; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing ."+
-		"The url is automatically detected from -datasource.url by replacing /query with /tail at the end if -tail.url is empty")
-	historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
-	header      = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
-	accountID   = flag.Int("accountID", 0, "Account ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-	projectID   = flag.Int("projectID", 0, "Project ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-)
-
-const (
-	firstLinePrompt = ";> "
-	nextLinePrompt  = ""
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.InitNoLogFlags()
-
-	hes, err := parseHeaders(*header)
-	if err != nil {
-		fatalf("cannot parse -header command-line flag: %s", err)
-	}
-	headers = hes
-
-	incompleteLine := ""
-	cfg := &readline.Config{
-		Prompt:                 firstLinePrompt,
-		DisableAutoSaveHistory: true,
-		Listener: func(line []rune, pos int, _ rune) ([]rune, int, bool) {
-			incompleteLine = string(line)
-			return line, pos, false
-		},
-	}
-	rl, err := readline.NewFromConfig(cfg)
-	if err != nil {
-		fatalf("cannot initialize readline: %s", err)
-	}
-
-	fmt.Fprintf(rl, "sending queries to %s\n", *datasourceURL)
-
-	runReadlineLoop(rl, &incompleteLine)
-
-	if err := rl.Close(); err != nil {
-		fatalf("cannot close readline: %s", err)
-	}
-
-}
-
-func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
-	historyLines, err := loadFromHistory(*historyFile)
-	if err != nil {
-		fatalf("cannot load query history: %s", err)
-	}
-	for _, line := range historyLines {
-		if err := rl.SaveToHistory(line); err != nil {
-			fatalf("cannot initialize query history: %s", err)
-		}
-	}
-
-	outputMode := outputModeJSONMultiline
-	s := ""
-	for {
-		line, err := rl.ReadLine()
-		if err != nil {
-			switch err {
-			case io.EOF:
-				if s != "" {
-					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode)
-				}
-				return
-			case readline.ErrInterrupt:
-				if s == "" && *incompleteLine == "" {
-					fmt.Fprintf(rl, "interrupted\n")
-					os.Exit(128 + int(syscall.SIGINT))
-				}
-				// Default value for Ctrl+C - clear the prompt and store the incompletely entered line into history
-				s += *incompleteLine
-				historyLines = pushToHistory(rl, historyLines, s)
-				s = ""
-				rl.SetPrompt(firstLinePrompt)
-				continue
-			default:
-				fatalf("unexpected error in readline: %s", err)
-			}
-		}
-
-		s += line
-		if s == "" {
-			// Skip empty lines
-			continue
-		}
-
-		if isQuitCommand(s) {
-			fmt.Fprintf(rl, "bye!\n")
-			_ = pushToHistory(rl, historyLines, s)
-			return
-		}
-		if isHelpCommand(s) {
-			printCommandsHelp(rl)
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\s` {
-			fmt.Fprintf(rl, "singleline json output mode\n")
-			outputMode = outputModeJSONSingleline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\m` {
-			fmt.Fprintf(rl, "multiline json output mode\n")
-			outputMode = outputModeJSONMultiline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\c` {
-			fmt.Fprintf(rl, "compact output mode\n")
-			outputMode = outputModeCompact
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\logfmt` {
-			fmt.Fprintf(rl, "logfmt output mode\n")
-			outputMode = outputModeLogfmt
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if line != "" && !strings.HasSuffix(line, ";") {
-			// Assume the query is incomplete and allow the user finishing the query on the next line
-			s += "\n"
-			rl.SetPrompt(nextLinePrompt)
-			continue
-		}
-
-		// Execute the query
-		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode)
-		cancel()
-
-		historyLines = pushToHistory(rl, historyLines, s)
-		s = ""
-		rl.SetPrompt(firstLinePrompt)
-	}
-}
-
-func pushToHistory(rl *readline.Instance, historyLines []string, s string) []string {
-	s = strings.TrimSpace(s)
-	if len(historyLines) == 0 || historyLines[len(historyLines)-1] != s {
-		historyLines = append(historyLines, s)
-		if len(historyLines) > 500 {
-			historyLines = historyLines[len(historyLines)-500:]
-		}
-		if err := saveToHistory(*historyFile, historyLines); err != nil {
-			fatalf("cannot save query history: %s", err)
-		}
-	}
-	if err := rl.SaveToHistory(s); err != nil {
-		fatalf("cannot update query history: %s", err)
-	}
-	return historyLines
-}
-
-func loadFromHistory(filePath string) ([]string, error) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	linesQuoted := strings.Split(string(data), "\n")
-	lines := make([]string, 0, len(linesQuoted))
-	i := 0
-	for _, lineQuoted := range linesQuoted {
-		i++
-		if lineQuoted == "" {
-			continue
-		}
-		line, err := strconv.Unquote(lineQuoted)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse line #%d at %s: %w; line: [%s]", i, filePath, err, line)
-		}
-		lines = append(lines, line)
-	}
-	return lines, nil
-}
-
-func saveToHistory(filePath string, lines []string) error {
-	linesQuoted := make([]string, len(lines))
-	for i, line := range lines {
-		lineQuoted := strconv.Quote(line)
-		linesQuoted[i] = lineQuoted
-	}
-	data := strings.Join(linesQuoted, "\n")
-	return os.WriteFile(filePath, []byte(data), 0600)
-}
-
-func isQuitCommand(s string) bool {
-	switch s {
-	case `\q`, "q", "quit", "exit":
-		return true
-	default:
-		return false
-	}
-}
-
-func isHelpCommand(s string) bool {
-	switch s {
-	case `\h`, "h", "help", "?":
-		return true
-	default:
-		return false
-	}
-}
-
-func printCommandsHelp(w io.Writer) {
-	fmt.Fprintf(w, "%s", `List of available commands:
-\q - quit
-\h - show this help
-\s - singleline json output mode
-\m - multiline json output mode
-\c - compact output
-\logfmt - logfmt output mode
-\tail <query> - live tail <query> results
-`)
-}
-
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
-	if strings.HasPrefix(qStr, `\tail `) {
-		tailQuery(ctx, output, qStr, outputMode)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, *datasourceURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if err := readWithLess(respBody); err != nil {
-		fmt.Fprintf(output, "error when reading query response: %s\n", err)
-		return
-	}
-}
-
-func tailQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
-	qStr = strings.TrimPrefix(qStr, `\tail `)
-	qURL, err := getTailURL()
-	if err != nil {
-		fmt.Fprintf(output, "%s\n", err)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, qURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if _, err := io.Copy(output, respBody); err != nil {
-		if !errors.Is(err, context.Canceled) && !isErrPipe(err) {
-			fmt.Fprintf(output, "error when live tailing query response: %s\n", err)
-		}
-		fmt.Fprintf(output, "\n")
-		return
-	}
-}
-
-func getTailURL() (string, error) {
-	if *tailURL != "" {
-		return *tailURL, nil
-	}
-
-	u, err := url.Parse(*datasourceURL)
-	if err != nil {
-		return "", fmt.Errorf("cannot parse -datasource.url=%q: %w", *datasourceURL, err)
-	}
-	if !strings.HasSuffix(u.Path, "/query") {
-		return "", fmt.Errorf("cannot find /query suffix in -datasource.url=%q", *datasourceURL)
-	}
-	u.Path = u.Path[:len(u.Path)-len("/query")] + "/tail"
-	return u.String(), nil
-}
-
-func getQueryResponse(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, qURL string) io.ReadCloser {
-	// Parse the query and convert it to canonical view.
-	qStr = strings.TrimSuffix(qStr, ";")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		fmt.Fprintf(output, "cannot parse query: %s\n", err)
-		return nil
-	}
-	qStr = q.String()
-	fmt.Fprintf(output, "executing [%s]...", qStr)
-
-	// Prepare HTTP request for qURL
-	args := make(url.Values)
-	args.Set("query", qStr)
-	data := strings.NewReader(args.Encode())
-
-	req, err := http.NewRequestWithContext(ctx, "POST", qURL, data)
-	if err != nil {
-		panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	for _, h := range headers {
-		req.Header.Set(h.Name, h.Value)
-	}
-	req.Header.Set("AccountID", strconv.Itoa(*accountID))
-	req.Header.Set("ProjectID", strconv.Itoa(*projectID))
-
-	// Execute HTTP request at qURL
-	startTime := time.Now()
-	resp, err := httpClient.Do(req)
-	queryDuration := time.Since(startTime)
-	fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
-	if err != nil {
-		if errors.Is(err, context.Canceled) {
-			fmt.Fprintf(output, "\n")
-		} else {
-			fmt.Fprintf(output, "cannot execute query: %s\n", err)
-		}
-		return nil
-	}
-
-	// Verify response code
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			body = []byte(fmt.Sprintf("cannot read response body: %s", err))
-		}
-		fmt.Fprintf(output, "unexpected status code: %d; response body:\n%s\n", resp.StatusCode, body)
-		return nil
-	}
-
-	// Prettify the response body
-	jp := newJSONPrettifier(resp.Body, outputMode)
-
-	return jp
-}
-
-var httpClient = &http.Client{}
-
-var headers []headerEntry
-
-type headerEntry struct {
-	Name  string
-	Value string
-}
-
-func parseHeaders(a []string) ([]headerEntry, error) {
-	hes := make([]headerEntry, len(a))
-	for i, s := range a {
-		a := strings.SplitN(s, ":", 2)
-		if len(a) != 2 {
-			return nil, fmt.Errorf("cannot parse header=%q; it must contain at least one ':'; for example, 'Cookie: foo'", s)
-		}
-		hes[i] = headerEntry{
-			Name:  strings.TrimSpace(a[0]),
-			Value: strings.TrimSpace(a[1]),
-		}
-	}
-	return hes, nil
-}
-
-func fatalf(format string, args ...any) {
-	fmt.Fprintf(os.Stderr, format+"\n", args...)
-	os.Exit(1)
-}
-
-func usage() {
-	const s = `
-vlogscli is a command-line tool for querying VictoriaLogs.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/querying/vlogscli/
-`
-	flagutil.Usage(s)
-}

app/vlogscli/multiarch/Dockerfile

@@ -1,11 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-ENTRYPOINT ["/vlogscli-prod"]
-ARG TARGETARCH
-COPY vlogscli-linux-${TARGETARCH}-prod ./vlogscli-prod

app/vlogsgenerator/main.go

@@ -230,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 	for i := 0; i < activeStreams; i++ {
 		ip := toIPv4(rand.Uint32())
 		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
+		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
 			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
 		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
 		for j := 0; j < *constFieldsPerLog; j++ {

app/vlselect/logsql/logsql.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -45,7 +44,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 	if step <= 0 {
 		httpserver.Errorf(w, r, "'step' must be bigger than zero")
-		return
 	}
 
 	// Obtain offset
@@ -72,10 +70,9 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		fieldsLimit = 0
 	}
 
-	// Prepare the query for hits count.
-	q.Optimize()
-	q.DropAllPipes()
+	// Prepare the query
 	q.AddCountByTimePipe(int64(step), int64(offset), fields)
+	q.Optimize()
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
@@ -382,8 +379,6 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 }
 
 // ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#live-tailing
 func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	liveTailRequests.Inc()
 	defer liveTailRequests.Dec()
@@ -394,9 +389,7 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 	if !q.CanLiveTail() {
-		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; "+
-			"see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
-		return
+		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
 	}
 	q.Optimize()
 
@@ -419,17 +412,13 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 	if !ok {
 		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
 	}
-	qOrig := q
 	for {
 		start := end - tailOffsetNsecs
 		end = time.Now().UnixNano()
 
-		q = qOrig.Clone(end)
-		q.AddTimeFilter(start, end)
-		// q.Optimize() call is needed for converting '*' into filterNoop.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
-		q.Optimize()
-		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, tp.writeBlock); err != nil {
+		qCopy := q.Clone()
+		qCopy.AddTimeFilter(start, end)
+		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
 			httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
 			return
 		}
@@ -570,212 +559,9 @@ func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
 	return tailRows, nil
 }
 
-// ProcessStatsQueryRangeRequest handles /select/logsql/stats_query_range request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats
-func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Obtain step
-	stepStr := r.FormValue("step")
-	if stepStr == "" {
-		stepStr = "1d"
-	}
-	step, err := promutils.ParseDuration(stepStr)
-	if err != nil {
-		err = fmt.Errorf("cannot parse 'step' arg: %s", err)
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-	if step <= 0 {
-		err := fmt.Errorf("'step' must be bigger than zero")
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Obtain `by(...)` fields from the last `| stats` pipe in q.
-	// Add `_time:step` to the `by(...)` list.
-	byFields, err := q.GetStatsByFieldsAddGroupingByTime(int64(step))
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	q.Optimize()
-
-	m := make(map[string]*statsSeries)
-	var mLock sync.Mutex
-
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
-		for i := range timestamps {
-			timestamp := q.GetTimestamp()
-			labels := make([]logstorage.Field, 0, len(byFields))
-			for j, c := range columns {
-				if c.Name == "_time" {
-					nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
-					if ok {
-						timestamp = nsec
-						continue
-					}
-				}
-				if slices.Contains(byFields, c.Name) {
-					labels = append(labels, logstorage.Field{
-						Name:  clonedColumnNames[j],
-						Value: strings.Clone(c.Values[i]),
-					})
-				}
-			}
-
-			var dst []byte
-			for j, c := range columns {
-				if !slices.Contains(byFields, c.Name) {
-					name := clonedColumnNames[j]
-					dst = dst[:0]
-					dst = append(dst, name...)
-					dst = logstorage.MarshalFieldsToJSON(dst, labels)
-					key := string(dst)
-					p := statsPoint{
-						Timestamp: timestamp,
-						Value:     strings.Clone(c.Values[i]),
-					}
-
-					mLock.Lock()
-					ss := m[key]
-					if ss == nil {
-						ss = &statsSeries{
-							key:    key,
-							Name:   name,
-							Labels: labels,
-						}
-						m[key] = ss
-					}
-					ss.Points = append(ss.Points, p)
-					mLock.Unlock()
-				}
-			}
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
-		err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Sort the collected stats by time
-	rows := make([]*statsSeries, 0, len(m))
-	for _, ss := range m {
-		points := ss.Points
-		sort.Slice(points, func(i, j int) bool {
-			return points[i].Timestamp < points[j].Timestamp
-		})
-		rows = append(rows, ss)
-	}
-	sort.Slice(rows, func(i, j int) bool {
-		return rows[i].key < rows[j].key
-	})
-
-	w.Header().Set("Content-Type", "application/json")
-	WriteStatsQueryRangeResponse(w, rows)
-}
-
-type statsSeries struct {
-	key string
-
-	Name   string
-	Labels []logstorage.Field
-	Points []statsPoint
-}
-
-type statsPoint struct {
-	Timestamp int64
-	Value     string
-}
-
-// ProcessStatsQueryRequest handles /select/logsql/stats_query request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats
-func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Obtain `by(...)` fields from the last `| stats` pipe in q.
-	byFields, err := q.GetStatsByFields()
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	q.Optimize()
-
-	var rows []statsRow
-	var rowsLock sync.Mutex
-
-	timestamp := q.GetTimestamp()
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
-		for i := range timestamps {
-			labels := make([]logstorage.Field, 0, len(byFields))
-			for j, c := range columns {
-				if slices.Contains(byFields, c.Name) {
-					labels = append(labels, logstorage.Field{
-						Name:  clonedColumnNames[j],
-						Value: strings.Clone(c.Values[i]),
-					})
-				}
-			}
-
-			for j, c := range columns {
-				if !slices.Contains(byFields, c.Name) {
-					r := statsRow{
-						Name:      clonedColumnNames[j],
-						Labels:    labels,
-						Timestamp: timestamp,
-						Value:     strings.Clone(c.Values[i]),
-					}
-
-					rowsLock.Lock()
-					rows = append(rows, r)
-					rowsLock.Unlock()
-				}
-			}
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
-		err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	WriteStatsQueryResponse(w, rows)
-}
-
-type statsRow struct {
-	Name      string
-	Labels    []logstorage.Field
-	Timestamp int64
-	Value     string
-}
-
 // ProcessQueryRequest handles /select/logsql/query request.
 //
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-logs
+// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
 func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	q, tenantIDs, err := parseCommonArgs(r)
 	if err != nil {
@@ -850,7 +636,6 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
 	q.Optimize()
-
 	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
 	if err != nil {
 		return nil, err
@@ -861,62 +646,32 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 		return rows, nil
 	}
 
-	// Slow path - adjust time range for selecting up to limitUpper rows
+	// Slow path - search for the time range containing up to limitUpper rows.
 	start, end := q.GetFilterTimeRange()
 	d := end/2 - start/2
 	start += d
 
 	qOrig := q
 	for {
-		timestamp := qOrig.GetTimestamp()
-		q = qOrig.Clone(timestamp)
+		q = qOrig.Clone()
 		q.AddTimeFilter(start, end)
-		// q.Optimize() call is needed for converting '*' into filterNoop.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
-		q.Optimize()
 		rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
 		if err != nil {
 			return nil, err
 		}
 
-		if d == 0 || start >= end {
-			// The [start ... end] time range equals one nanosecond.
-			// Just return up to limit rows.
-			if len(rows) > limit {
-				rows = rows[:limit]
-			}
-			return rows, nil
-		}
-
-		dLastBit := d & 1
-		d /= 2
-
-		if len(rows) >= limitUpper {
-			// The number of found rows on the [start ... end] time range exceeds limitUpper,
-			// so reduce the time range to [start+d ... end].
-			start += d
-			continue
-		}
-		if len(rows) >= limit {
-			// The number of found rows is in the range [limit ... limitUpper).
-			// This means that found rows contains the needed limit rows with the biggest timestamps.
+		if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
 			rows = getLastNRows(rows, limit)
 			return rows, nil
 		}
 
-		// The number of found rows on [start ... end] time range is below the limit.
-		// This means the time range doesn't cover the needed logs, so it must be extended.
-
-		if len(rows) == 0 {
-			// The [start ... end] time range doesn't contain any rows, so change it to [start-d ... start).
-			end = start - 1
-			start -= d + dLastBit
-			continue
+		lastBit := d & 1
+		d /= 2
+		if len(rows) > limit {
+			start += d
+		} else {
+			start -= d + lastBit
 		}
-
-		// The number of found rows on [start ... end] time range is bigger than 0 but smaller than limit.
-		// Increase the time range to [start-d ... end].
-		start -= d + dLastBit
 	}
 }
 
@@ -937,25 +692,20 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 	var rows []row
 	var rowsLock sync.Mutex
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
+		rowsLock.Lock()
+		defer rowsLock.Unlock()
 
 		for i, timestamp := range timestamps {
 			fields := make([]logstorage.Field, len(columns))
 			for j := range columns {
 				f := &fields[j]
-				f.Name = clonedColumnNames[j]
+				f.Name = strings.Clone(columns[j].Name)
 				f.Value = strings.Clone(columns[j].Values[i])
 			}
-
-			rowsLock.Lock()
 			rows = append(rows, row{
 				timestamp: timestamp,
 				fields:    fields,
 			})
-			rowsLock.Unlock()
 		}
 
 		if len(rows) >= limit {
@@ -977,23 +727,9 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	}
 	tenantIDs := []logstorage.TenantID{tenantID}
 
-	// Parse optional time arg
-	timestamp, okTime, err := getTimeNsec(r, "time")
-	if err != nil {
-		return nil, nil, err
-	}
-	if !okTime {
-		// If time arg is missing, then evaluate query at the current timestamp
-		timestamp = time.Now().UnixNano()
-	}
-
-	// decrease timestamp by one nanosecond in order to avoid capturing logs belonging
-	// to the first nanosecond at the next period of time (month, week, day, hour, etc.)
-	timestamp--
-
 	// Parse query
 	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
+	q, err := logstorage.ParseQuery(qStr)
 	if err != nil {
 		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
 	}

app/vlselect/logsql/query_response.qtpl

@@ -6,31 +6,15 @@
 
 // JSONRow creates JSON row from the given fields.
 {% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-	{% code
-		i := 0
-		for i < len(columns) && columns[i].Values[rowIdx] == "" {
-			i++
-		}
-		columns = columns[i:]
-	%}
-	{% if len(columns) == 0 %}
-		{% return %}
-	{% endif %}
-	{
-	{% code	c := &columns[0] %}
+{
+	{% code c := &columns[0] %}
 	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
 	{% code columns = columns[1:] %}
 	{% for colIdx := range columns %}
-		{% code
-			c := &columns[colIdx]
-			v := c.Values[rowIdx]
-		%}
-		{% if v == "" %}
-			{% continue %}
-		{% endif %}
+		{% code c := &columns[colIdx] %}
 		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
 	{% endfor %}
-	}{% newline %}
+}{% newline %}
 {% endfunc %}
 
 // JSONRows prints formatted rows
@@ -39,11 +23,7 @@
 		{% return %}
 	{% endif %}
 	{% for _, fields := range rows %}
-		{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
-		{% if len(fields) == 0 %}
-			{% continue %}
-		{% endif %}
-		{
+	{
 		{% if len(fields) > 0 %}
 			{% code
 				f := fields[0]
@@ -51,13 +31,10 @@
 			%}
 			{%q= f.Name %}:{%q= f.Value %}
 			{% for _, f := range fields %}
-				{% if f.Value == "" %}
-					{% continue %}
-				{% endif %}
 				,{%q= f.Name %}:{%q= f.Value %}
 			{% endfor %}
 		{% endif %}
-		}{% newline %}
+	}{% newline %}
 	{% endfor %}
 {% endfunc %}
 

app/vlselect/logsql/query_response.qtpl.go

@@ -26,176 +26,141 @@ var (
 
 //line app/vlselect/logsql/query_response.qtpl:8
 func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:10
-	i := 0
-	for i < len(columns) && columns[i].Values[rowIdx] == "" {
-		i++
-	}
-	columns = columns[i:]
-
-//line app/vlselect/logsql/query_response.qtpl:16
-	if len(columns) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:17
-		return
-//line app/vlselect/logsql/query_response.qtpl:18
-	}
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:8
 	qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:20
+//line app/vlselect/logsql/query_response.qtpl:10
 	c := &columns[0]
 
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:11
 	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:11
 	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:11
 	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:22
+//line app/vlselect/logsql/query_response.qtpl:12
 	columns = columns[1:]
 
-//line app/vlselect/logsql/query_response.qtpl:23
+//line app/vlselect/logsql/query_response.qtpl:13
 	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:25
+//line app/vlselect/logsql/query_response.qtpl:14
 		c := &columns[colIdx]
-		v := c.Values[rowIdx]
 
-//line app/vlselect/logsql/query_response.qtpl:28
-		if v == "" {
-//line app/vlselect/logsql/query_response.qtpl:29
-			continue
-//line app/vlselect/logsql/query_response.qtpl:30
-		}
-//line app/vlselect/logsql/query_response.qtpl:30
+//line app/vlselect/logsql/query_response.qtpl:14
 		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:31
+//line app/vlselect/logsql/query_response.qtpl:15
 		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:31
+//line app/vlselect/logsql/query_response.qtpl:15
 		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:31
+//line app/vlselect/logsql/query_response.qtpl:15
 		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:16
 	}
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:16
 	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:33
+//line app/vlselect/logsql/query_response.qtpl:17
 	qw422016.N().S(`
 `)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 }
 
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 }
 
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 }
 
 // JSONRows prints formatted rows
 
-//line app/vlselect/logsql/query_response.qtpl:37
+//line app/vlselect/logsql/query_response.qtpl:21
 func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:38
+//line app/vlselect/logsql/query_response.qtpl:22
 	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:23
 		return
-//line app/vlselect/logsql/query_response.qtpl:40
+//line app/vlselect/logsql/query_response.qtpl:24
 	}
-//line app/vlselect/logsql/query_response.qtpl:41
+//line app/vlselect/logsql/query_response.qtpl:25
 	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:42
-		fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
-
-//line app/vlselect/logsql/query_response.qtpl:43
-		if len(fields) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:44
-			continue
-//line app/vlselect/logsql/query_response.qtpl:45
-		}
-//line app/vlselect/logsql/query_response.qtpl:45
+//line app/vlselect/logsql/query_response.qtpl:25
 		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:47
+//line app/vlselect/logsql/query_response.qtpl:27
 		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:49
+//line app/vlselect/logsql/query_response.qtpl:29
 			f := fields[0]
 			fields = fields[1:]
 
-//line app/vlselect/logsql/query_response.qtpl:52
+//line app/vlselect/logsql/query_response.qtpl:32
 			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:52
+//line app/vlselect/logsql/query_response.qtpl:32
 			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:52
+//line app/vlselect/logsql/query_response.qtpl:32
 			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:53
+//line app/vlselect/logsql/query_response.qtpl:33
 			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:54
-				if f.Value == "" {
-//line app/vlselect/logsql/query_response.qtpl:55
-					continue
-//line app/vlselect/logsql/query_response.qtpl:56
-				}
-//line app/vlselect/logsql/query_response.qtpl:56
+//line app/vlselect/logsql/query_response.qtpl:33
 				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:57
+//line app/vlselect/logsql/query_response.qtpl:34
 				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:57
+//line app/vlselect/logsql/query_response.qtpl:34
 				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:57
+//line app/vlselect/logsql/query_response.qtpl:34
 				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:58
+//line app/vlselect/logsql/query_response.qtpl:35
 			}
-//line app/vlselect/logsql/query_response.qtpl:59
+//line app/vlselect/logsql/query_response.qtpl:36
 		}
-//line app/vlselect/logsql/query_response.qtpl:59
+//line app/vlselect/logsql/query_response.qtpl:36
 		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:60
+//line app/vlselect/logsql/query_response.qtpl:37
 		qw422016.N().S(`
 `)
-//line app/vlselect/logsql/query_response.qtpl:61
+//line app/vlselect/logsql/query_response.qtpl:38
 	}
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 }
 
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 }
 
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 }

app/vlselect/logsql/stats_query_range_response.qtpl

@@ -1,52 +0,0 @@
-{% stripspace %}
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-{% func StatsQueryRangeResponse(rows []*statsSeries) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"matrix",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsSeries(rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsSeries(rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsSeries(ss *statsSeries) %}
-{
-	"metric":{
-		"__name__":{%q= ss.Name %}
-		{% if len(ss.Labels) > 0 %}
-			{% for _, label := range ss.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"values":[
-		{% code points := ss.Points %}
-		{% if len(points) > 0 %}
-			{%= formatStatsPoint(&points[0]) %}
-			{% code points = points[1:] %}
-			{% for i := range points %}
-				,{%= formatStatsPoint(&points[i]) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func formatStatsPoint(p *statsPoint) %}
-[
-	{%f= float64(p.Timestamp)/1e9 %},
-	{%q= p.Value %}
-]
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_range_response.qtpl.go

@@ -1,188 +0,0 @@
-// Code generated by qtc from "stats_query_range_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-func StreamStatsQueryRangeResponse(qw422016 *qt422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:11
-		streamformatStatsSeries(qw422016, rows[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:14
-			streamformatStatsSeries(qw422016, rows[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func WriteStatsQueryRangeResponse(qq422016 qtio422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	StreamStatsQueryRangeResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func StatsQueryRangeResponse(rows []*statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	WriteStatsQueryRangeResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-func streamformatStatsSeries(qw422016 *qt422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:25
-	qw422016.N().Q(ss.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:26
-	if len(ss.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-		for _, label := range ss.Labels {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	qw422016.N().S(`},"values":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:33
-	points := ss.Points
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:34
-	if len(points) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:35
-		streamformatStatsPoint(qw422016, &points[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:36
-		points = points[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-		for i := range points {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:38
-			streamformatStatsPoint(qw422016, &points[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:39
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func writeformatStatsSeries(qq422016 qtio422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	streamformatStatsSeries(qw422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func formatStatsSeries(ss *statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	writeformatStatsSeries(qb422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-func streamformatStatsPoint(qw422016 *qt422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-	qw422016.N().S(`[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().F(float64(p.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().Q(p.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().S(`]`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func writeformatStatsPoint(qq422016 qtio422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	streamformatStatsPoint(qw422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func formatStatsPoint(p *statsPoint) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	writeformatStatsPoint(qb422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}

app/vlselect/logsql/stats_query_response.qtpl

@@ -1,36 +0,0 @@
-{% stripspace %}
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-{% func StatsQueryResponse(rows []statsRow) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"vector",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsRow(&rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsRow(&rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsRow(r *statsRow) %}
-{
-	"metric":{
-		"__name__":{%q= r.Name %}
-		{% if len(r.Labels) > 0 %}
-			{% for _, label := range r.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"value":[{%f= float64(r.Timestamp)/1e9 %},{%q= r.Value %}]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_response.qtpl.go

@@ -1,133 +0,0 @@
-// Code generated by qtc from "stats_query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-func StreamStatsQueryResponse(qw422016 *qt422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:11
-		streamformatStatsRow(qw422016, &rows[0])
-//line app/vlselect/logsql/stats_query_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:14
-			streamformatStatsRow(qw422016, &rows[i])
-//line app/vlselect/logsql/stats_query_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func WriteStatsQueryResponse(qq422016 qtio422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	StreamStatsQueryResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func StatsQueryResponse(rows []statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	WriteStatsQueryResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-func streamformatStatsRow(qw422016 *qt422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_response.qtpl:25
-	qw422016.N().Q(r.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:26
-	if len(r.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-		for _, label := range r.Labels {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	qw422016.N().S(`},"value":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().F(float64(r.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().Q(r.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func writeformatStatsRow(qq422016 qtio422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	streamformatStatsRow(qw422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func formatStatsRow(r *statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	writeformatStatsRow(qb422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}

app/vlselect/main.go

@@ -193,14 +193,6 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 		logsqlQueryRequests.Inc()
 		logsql.ProcessQueryRequest(ctx, w, r)
 		return true
-	case "/select/logsql/stats_query":
-		logsqlStatsQueryRequests.Inc()
-		logsql.ProcessStatsQueryRequest(ctx, w, r)
-		return true
-	case "/select/logsql/stats_query_range":
-		logsqlStatsQueryRangeRequests.Inc()
-		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
-		return true
 	case "/select/logsql/stream_field_names":
 		logsqlStreamFieldNamesRequests.Inc()
 		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
@@ -240,8 +232,6 @@ var (
 	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
 	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
 	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlStatsQueryRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
-	logsqlStatsQueryRangeRequests   = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
 	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
 	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
 	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.faf86aa5.css",
-    "main.js": "./static/js/main.2810cc52.js",
-    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
+    "main.css": "./static/css/main.1041c3d4.css",
+    "main.js": "./static/js/main.8988988c.js",
+    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.faf86aa5.css",
-    "static/js/main.2810cc52.js"
+    "static/css/main.1041c3d4.css",
+    "static/js/main.8988988c.js"
   ]
 }

app/vlselect/vmui/config.json

@@ -1,5 +0,0 @@
-{
-  "license": {
-    "type": "opensource"
-  }
-}

app/vlselect/vmui/favicon.svg

@@ -1 +0,0 @@
-<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.2810cc52.js"></script><link href="./static/css/main.faf86aa5.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8988988c.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/manifest.json

@@ -1,11 +1,16 @@
 {
-  "short_name": "vmui",
-  "name": "vmui",
+  "short_name": "Victoria Metrics UI",
+  "name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
   "icons": [
     {
-      "src": "favicon.svg",
-      "sizes": "any",
-      "type": "image/svg+xml"
+      "src": "favicon-32x32.png",
+      "sizes": "32x32",
+      "type": "image/png"
+    },
+    {
+      "src": "apple-touch-icon.png",
+      "type": "image/png",
+      "sizes": "192x192"
     }
   ],
   "start_url": ".",

app/vlselect/vmui/static/js/main.8988988c.js.LICENSE.txt

@@ -5,7 +5,7 @@
 */
 
 /**
- * @remix-run/router v1.19.2
+ * @remix-run/router v1.17.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.26.2
+ * React Router DOM v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +27,7 @@
  */
 
 /**
- * React Router v6.26.2
+ * React Router v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlselect/vmui/static/media/MetricsQL.aaabf95f2c9bf356bde4.md

@@ -1,4 +1,5 @@
 ---
+sort: 23
 weight: 23
 title: MetricsQL
 menu:
@@ -9,6 +10,9 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
+
+# MetricsQL
+
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -103,7 +107,7 @@ The list of MetricsQL features on top of PromQL:
 * Trailing commas on all the lists are allowed - label filters, function args and with expressions.
   For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
   This simplifies maintenance of multi-line queries.
-* Metric names and label names may contain any unicode letter. For example `ტემპერატურა{πόλη="Київ"}` is a valid MetricsQL expression.
+* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a valid MetricsQL expression.
 * Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
   It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
   Additionally, the following escape sequences are supported:
@@ -216,11 +220,9 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
+See also [median_over_time](#median_over_time).
 
 #### changes
 
@@ -260,8 +262,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -272,8 +272,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -284,8 +282,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -296,9 +292,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
-See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
+See also [count_over_time](#count_over_time).
 
 #### count_over_time
 
@@ -319,8 +313,6 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -440,8 +432,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -453,15 +443,11 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -469,8 +455,6 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -478,9 +462,8 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`.
-
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
+returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -512,14 +495,13 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -528,13 +510,12 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -543,10 +524,6 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
-
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -563,18 +540,15 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -613,8 +587,6 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -622,11 +594,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
+See also [tmax_over_time](#tmax_over_time).
 
 #### median_over_time
 
@@ -634,8 +604,6 @@ See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -643,11 +611,9 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
+See also [tmin_over_time](#tmin_over_time).
 
 #### mode_over_time
 
@@ -656,8 +622,6 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -668,8 +632,6 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -697,8 +659,6 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -710,8 +670,6 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -722,12 +680,11 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -746,8 +703,6 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -755,11 +710,10 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 #### rollup
@@ -771,10 +725,6 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
-See also [rollup_rate](#rollup_rate).
-
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -786,8 +736,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -814,8 +762,6 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
-
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -828,17 +774,12 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
-
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -846,12 +787,10 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
-
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -885,8 +824,6 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -901,8 +838,6 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -914,8 +849,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -933,8 +866,6 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -946,43 +877,35 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
 
 #### sum_eq_over_time
 
-`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
 
-`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
 
-`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -992,8 +915,6 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -1003,8 +924,6 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1082,8 +1001,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2313,8 +2230,7 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2347,6 +2263,5 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
-  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
+  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.

app/vlstorage/main.go

@@ -18,12 +18,12 @@ import (
 )
 
 var (
-	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
+	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
 		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
 		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
 	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
 		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
-	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
+	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
 		"see https://docs.victoriametrics.com/victorialogs/#retention")
 	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
 		"see https://docs.victoriametrics.com/victorialogs/#storage")
@@ -37,8 +37,6 @@ var (
 		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
 		"the storage stops accepting new data")
-
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 )
 
 // Init initializes vlstorage.
@@ -82,35 +80,13 @@ func Init() {
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics, true)
+	metrics.UnregisterSet(storageMetrics)
 	storageMetrics = nil
 
 	strg.MustClose()
 	strg = nil
 }
 
-// RequestHandler is a storage request handler.
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if path == "/internal/force_merge" {
-		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-			return true
-		}
-		// Run force merge in background
-		partitionNamePrefix := r.FormValue("partition_prefix")
-		go func() {
-			activeForceMerges.Inc()
-			defer activeForceMerges.Dec()
-			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-			startTime := time.Now()
-			strg.MustForceMerge(partitionNamePrefix)
-			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-		}()
-		return true
-	}
-	return false
-}
-
 var strg *logstorage.Storage
 var storageMetrics *metrics.Set
 
@@ -229,5 +205,3 @@ func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
 	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
 }
-
-var activeForceMerges = metrics.NewCounter("vl_active_force_merges")

app/vmagent/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 8429
 
 ENTRYPOINT ["/vmagent-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./vmagent-prod

app/vmagent/influx/request_handler.go

@@ -36,7 +36,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, true, isGzipped, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, nil)
 	})
 }
@@ -50,12 +50,11 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	isStreamMode := req.Header.Get("Stream-Mode") == "1"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }

app/vmagent/main.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
@@ -44,7 +42,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -72,8 +70,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -116,7 +114,7 @@ func main() {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
+			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
@@ -318,10 +316,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -440,7 +434,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -449,17 +443,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		var bb bytesutil.ByteBuffer
 		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		promscrapeConfigReloadRequests.Inc()
@@ -568,10 +562,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -682,8 +672,7 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
-	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/vmagent/prometheusimport/request_handler_test.go

@@ -26,11 +26,11 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unxepected error %s", err)
+		t.Errorf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 

app/vmagent/remotewrite/client.go

@@ -7,22 +7,22 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -34,10 +34,8 @@ var (
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
-	sendTimeout      = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxTime")
-	retryMaxTime     = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
-	proxyURL         = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
 	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
@@ -92,9 +90,6 @@ type client struct {
 	fq *persistentqueue.FastQueue
 	hc *http.Client
 
-	retryMinInterval time.Duration
-	retryMaxTime     time.Duration
-
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
@@ -125,7 +120,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
-		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
+		DialContext:         httputils.GetStatDialFunc("vmagent_remotewrite"),
 		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
@@ -148,15 +143,13 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
-		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
-		authCfg:          authCfg,
-		awsCfg:           awsCfg,
-		fq:               fq,
-		hc:               hc,
-		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
-		retryMaxTime:     retryMaxTime.GetOptionalArg(argIdx),
-		stopCh:           make(chan struct{}),
+		sanitizedURL:   sanitizedURL,
+		remoteWriteURL: remoteWriteURL,
+		authCfg:        authCfg,
+		awsCfg:         awsCfg,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
 
@@ -403,11 +396,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
 //
 // The function returns false only if c.stopCh is closed.
-// Otherwise, it tries sending the block to remote storage indefinitely.
+// Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
-	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
+	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
+	retryDuration := timeutil.AddJitterToDuration(time.Second)
 	retriesCount := 0
 
 again:
@@ -463,10 +456,10 @@ again:
 
 	// Unexpected status code returned
 	retriesCount++
-	retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
-	retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
-
-	// Handle response
+	retryDuration *= 2
+	if retryDuration > maxRetryDuration {
+		retryDuration = maxRetryDuration
+	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
@@ -488,49 +481,3 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
-
-// getRetryDuration returns retry duration.
-// retryAfterDuration has the highest priority.
-// If retryAfterDuration is not specified, retryDuration gets doubled.
-// retryDuration can't exceed maxRetryDuration.
-//
-// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
-func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
-	// retryAfterDuration has the highest priority duration
-	if retryAfterDuration > 0 {
-		return timeutil.AddJitterToDuration(retryAfterDuration)
-	}
-
-	// default backoff retry policy
-	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
-	}
-
-	return retryDuration
-}
-
-// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
-// retryAfterString should be in either HTTP-date or a number of seconds.
-// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
-func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
-	if retryAfterString == "" {
-		return retryAfterDuration
-	}
-
-	defer func() {
-		v := retryAfterDuration.Seconds()
-		logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
-	}()
-
-	// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
-	if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
-		return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
-	}
-	// Retry-After could be in seconds.
-	if seconds, err := strconv.Atoi(retryAfterString); err == nil {
-		return time.Duration(seconds) * time.Second
-	}
-
-	return 0
-}

app/vmagent/remotewrite/client_test.go

@@ -1,99 +0,0 @@
-package remotewrite
-
-import (
-	"math"
-	"net/http"
-	"testing"
-	"time"
-)
-
-func TestCalculateRetryDuration(t *testing.T) {
-	// `testFunc` call `calculateRetryDuration` for `n` times
-	// and evaluate if the result of `calculateRetryDuration` is
-	// 1. >= expectMinDuration
-	// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
-	f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
-		t.Helper()
-
-		for i := 0; i < n; i++ {
-			retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
-		}
-
-		expectMaxDuration := helper(expectMinDuration)
-		expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
-
-		if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
-			t.Fatalf(
-				"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
-				expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
-				retryDuration.Milliseconds(),
-			)
-		}
-	}
-
-	// Call calculateRetryDuration for 1 time.
-	{
-		// default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-		// default backoff policy exceed max limit"
-		f(0, 10*time.Minute, 1, time.Minute)
-
-		// retry after > default backoff policy
-		f(10*time.Second, 1*time.Second, 1, 10*time.Second)
-		// retry after < default backoff policy
-		f(1*time.Second, 10*time.Second, 1, 1*time.Second)
-		// retry after invalid and < default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-
-	}
-
-	// Call calculateRetryDuration for multiple times.
-	{
-		// default backoff policy 2 times
-		f(0, time.Second, 2, 4*time.Second)
-		// default backoff policy 3 times
-		f(0, time.Second, 3, 8*time.Second)
-		// default backoff policy N times exceed max limit
-		f(0, time.Second, 10, time.Minute)
-
-		// retry after 120s 1 times
-		f(120*time.Second, time.Second, 1, 120*time.Second)
-		// retry after 120s 2 times
-		f(120*time.Second, time.Second, 2, 120*time.Second)
-	}
-}
-
-func TestParseRetryAfterHeader(t *testing.T) {
-	f := func(retryAfterString string, expectResult time.Duration) {
-		t.Helper()
-
-		result := parseRetryAfterHeader(retryAfterString)
-		// expect `expectResult == result` when retryAfterString is in seconds or invalid
-		// expect the difference between result and expectResult to be lower than 10%
-		if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
-			t.Fatalf(
-				"incorrect retry after duration, want (ms): %d, got (ms): %d",
-				expectResult.Milliseconds(), result.Milliseconds(),
-			)
-		}
-	}
-
-	// retry after header in seconds
-	f("10", 10*time.Second)
-	// retry after header in date time
-	f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
-	// retry after header invalid
-	f("invalid-retry-after", 0)
-	// retry after header not in GMT
-	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
-}
-
-// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
-func helper(d time.Duration) time.Duration {
-	dv := d / 10
-	if dv > 10*time.Second {
-		dv = 10 * time.Second
-	}
-
-	return d + dv
-}

app/vmagent/remotewrite/relabel.go

@@ -181,7 +181,7 @@ func (rctx *relabelCtx) reset() {
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		return &relabelCtx{}
 	},
 }

app/vmagent/remotewrite/remotewrite.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
-	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -89,15 +88,15 @@ var (
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
-		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence ."+
+		"See also -remoteWrite.dropSamplesOnOverload")
+	dropSamplesOnOverload = flagutil.NewArrayBool("remoteWrite.dropSamplesOnOverload", "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
 )
 
 var (
-	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
-	rwctxsGlobal []*remoteWriteCtx
+	// rwctxs contains statically populated entries when -remoteWrite.url is specified.
+	rwctxs []*remoteWriteCtx
 
 	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
 	defaultAuthToken = &auth.Token{}
@@ -110,11 +109,8 @@ var (
 		StatusCode: http.StatusTooManyRequests,
 	}
 
-	// disableOnDiskQueueAny is set to true if at least a single -remoteWrite.url is configured with -remoteWrite.disableOnDiskQueue
-	disableOnDiskQueueAny bool
-
-	// dropSamplesOnFailureGlobal is set to true if -remoteWrite.dropSamplesOnOverload is set or if multiple -remoteWrite.disableOnDiskQueue options are set.
-	dropSamplesOnFailureGlobal bool
+	// disableOnDiskQueueAll is set to true if all remoteWrite.urls were configured to disable persistent queue via disableOnDiskQueue
+	disableOnDiskQueueAll bool
 )
 
 // MultitenancyEnabled returns true if -enableMultitenantHandlers is specified.
@@ -207,18 +203,28 @@ func Init() {
 	relabelConfigSuccess.Set(1)
 	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
-	initStreamAggrConfigGlobal()
+	sasFile, sasOpts := getStreamAggrOpts(-1)
+	if sasFile != "" {
+		sas, err := newStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -streamAggr.config=%q: %s", sasFile, err)
+		}
+		sasGlobal.Store(sas)
+	} else if sasOpts.DedupInterval > 0 {
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesDropFailed, sasOpts.DedupInterval, sasOpts.DropInputLabels, sasOpts.Alias)
+	}
 
-	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	if len(*remoteWriteURLs) > 0 {
+		rwctxs = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	}
 
-	disableOnDiskQueues := []bool(*disableOnDiskQueue)
-	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
-
-	// Samples must be dropped if multiple -remoteWrite.disableOnDiskQueue options are configured and at least a single is set to true.
-	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
-	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
-	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
-	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
+	disableOnDiskQueueAll = true
+	for _, v := range *disableOnDiskQueue {
+		if !v {
+			disableOnDiskQueueAll = false
+			break
+		}
+	}
 
 	dropDanglingQueues()
 
@@ -228,9 +234,9 @@ func Init() {
 		defer configReloaderWG.Done()
 		for {
 			select {
+			case <-sighupCh:
 			case <-configReloaderStopCh:
 				return
-			case <-sighupCh:
 			}
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
@@ -249,8 +255,8 @@ func dropDanglingQueues() {
 	// In case if there were many persistent queues with identical *remoteWriteURLs
 	// the queue with the last index will be dropped.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
-	for _, rwctx := range rwctxsGlobal {
+	existingQueues := make(map[string]struct{}, len(rwctxs))
+	for _, rwctx := range rwctxs {
 		existingQueues[rwctx.fq.Dirname()] = struct{}{}
 	}
 
@@ -267,7 +273,7 @@ func dropDanglingQueues() {
 		}
 	}
 	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
 	}
 }
 
@@ -376,10 +382,10 @@ func Stop() {
 		deduplicatorGlobal = nil
 	}
 
-	for _, rwctx := range rwctxsGlobal {
+	for _, rwctx := range rwctxs {
 		rwctx.MustStop()
 	}
-	rwctxsGlobal = nil
+	rwctxs = nil
 
 	if sl := hourlySeriesLimiter; sl != nil {
 		sl.MustStop()
@@ -391,8 +397,6 @@ func Stop() {
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
 //
-// PushDropSamplesOnFailure drops wr samples if they cannot be sent to -remoteWrite.url by any reason.
-//
 // PushDropSamplesOnFailure can modify wr contents.
 func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
@@ -405,7 +409,7 @@ func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
 func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, dropSamplesOnFailureGlobal)
+	return tryPush(at, wr, false)
 }
 
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
@@ -422,30 +426,33 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		tenantRctx = getRelabelCtx()
 		defer putRelabelCtx(tenantRctx)
 	}
+	rowsCount := getRowsCount(tss)
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
 	// if some of remote storage systems cannot keep up with the data ingestion rate.
-	rwctxs, ok := getEligibleRemoteWriteCtxs(tss, forceDropSamplesOnFailure)
-	if !ok {
-		// At least a single remote write queue is blocked and dropSamplesOnFailure isn't set.
-		// Return false to the caller, so it could re-send samples again.
-		return false
-	}
-	if len(rwctxs) == 0 {
-		// All the remote write queues are skipped because they are blocked and dropSamplesOnFailure is set to true.
-		// Return true to the caller, so it doesn't re-send the samples again.
-		return true
+	// this shortcut is only applicable if all remote writes have disableOnDiskQueue = true
+	if disableOnDiskQueueAll {
+		for _, rwctx := range rwctxs {
+			if rwctx.fq.IsWriteBlocked() {
+				rwctx.pushFailures.Inc()
+				if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
+					// Just drop samples
+					rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+					continue
+				}
+				return false
+			}
+		}
 	}
 
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
+	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}
-	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
@@ -494,51 +501,24 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 				tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
 			}
 			matchIdxsPool.Put(matchIdxs)
-		}
-		if deduplicatorGlobal != nil {
+		} else if deduplicatorGlobal != nil {
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
-		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushBlockToRemoteStorages(tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
 	return true
 }
 
-func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
-	if !disableOnDiskQueueAny {
-		return rwctxsGlobal, true
-	}
-
-	// This code is applicable if at least a single remote storage has -disableOnDiskQueue
-	rwctxs := make([]*remoteWriteCtx, 0, len(rwctxsGlobal))
-	for _, rwctx := range rwctxsGlobal {
-		if !rwctx.fq.IsWriteBlocked() {
-			rwctxs = append(rwctxs, rwctx)
-		} else {
-			rwctx.pushFailures.Inc()
-			if !forceDropSamplesOnFailure {
-				return nil, false
-			}
-			rowsCount := getRowsCount(tss)
-			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-		}
-	}
-	return rwctxs, true
-}
-
-func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
-	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
-	if len(rwctxs) == 0 {
+func pushToRemoteStoragesDropFailed(tss []prompbmarshal.TimeSeries) {
+	if tryPushBlockToRemoteStorages(tss, true) {
 		return
 	}
-	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
-		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
-	}
 }
 
-func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -557,7 +537,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingBlockAmongRemoteStorages(tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
@@ -578,7 +558,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 	return !anyPushFailed.Load()
 }
 
-func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingBlockAmongRemoteStorages(tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -765,8 +745,10 @@ type remoteWriteCtx struct {
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
 
-	streamAggrKeepInput bool
-	streamAggrDropInput bool
+	streamAggrKeepInput   bool
+	streamAggrDropInput   bool
+	disableOnDiskQueue    bool
+	dropSamplesOnOverload bool
 
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
@@ -791,7 +773,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
-
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -836,13 +817,31 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		c:   c,
 		pss: pss,
 
-		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		dropSamplesOnOverload: dropSamplesOnOverload.GetOptionalArg(argIdx),
+		disableOnDiskQueue:    isPQDisabled,
 
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+
+		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+	}
+
+	// Initialize sas
+	sasFile, sasOpts := getStreamAggrOpts(argIdx)
+	if sasFile != "" {
+		sas, err := newStreamAggrConfig(argIdx, rwctx.pushInternalTrackDropped)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
+		}
+		rwctx.sas.Store(sas)
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
+	} else if sasOpts.DedupInterval > 0 {
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, sasOpts.DedupInterval, sasOpts.DropInputLabels, sasOpts.Alias)
 	}
-	rwctx.initStreamAggrConfig()
 
 	return rwctx
 }
@@ -923,8 +922,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		}
 		matchIdxsPool.Put(matchIdxs)
-	}
-	if rwctx.deduplicator != nil {
+	} else if rwctx.deduplicator != nil {
 		rwctx.deduplicator.Push(tss)
 		return true
 	}
@@ -936,9 +934,8 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 
 	// Couldn't push tss to remote storage
 	rwctx.pushFailures.Inc()
-	if forceDropSamplesOnFailure {
-		rowsCount := getRowsCount(tss)
-		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
+		rwctx.rowsDroppedOnPushFailure.Add(len(tss))
 		return true
 	}
 	return false
@@ -965,12 +962,14 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSe
 	if rwctx.tryPushInternal(tss) {
 		return
 	}
-	if !rwctx.fq.IsPersistentQueueDisabled() {
+	if !rwctx.disableOnDiskQueue {
 		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
 	}
 	rwctx.pushFailures.Inc()
-	rowsCount := getRowsCount(tss)
-	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	if dropSamplesOnOverload.GetOptionalArg(rwctx.idx) {
+		rowsCount := getRowsCount(tss)
+		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	}
 }
 
 func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
@@ -1001,7 +1000,7 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 }
 
 var tssPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}
 		return &a
 	},

app/vmagent/remotewrite/remotewrite_test.go

@@ -77,16 +77,14 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "global")
 		}
 
-		if streamAggrConfig != "" {
-			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
+		if len(streamAggrConfig) > 0 {
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), nil, streamaggr.Options{})
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
-			defer sas.MustStop()
 			rwctx.sas.Store(sas)
 		}
 
@@ -96,9 +94,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 
 		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		if !rwctx.TryPush(inputTss, false) {
-			t.Fatalf("cannot push samples to rwctx")
-		}
+		rwctx.TryPush(inputTss, false)
 
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)

app/vmagent/remotewrite/streamaggr.go

@@ -3,7 +3,6 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -24,7 +23,7 @@ var (
 	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
+	streamAggrGlobalDedupInterval = flagutil.NewDuration("streamAggr.dedupInterval", "0s", "Input samples are de-duplicated with this interval on "+
 		"aggregator before optional aggregation with -streamAggr.config . "+
 		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
 	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
@@ -51,200 +50,115 @@ var (
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
 		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
 		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
-		"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
 func CheckStreamAggrConfigs() error {
-	// Check global config
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		return err
-	}
-	sas.MustStop()
-
-	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
-		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
-	}
-
 	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+
+	if _, err := newStreamAggrConfig(-1, pushNoop); err != nil {
+		return fmt.Errorf("could not load -streamAggr.config stream aggregation config: %w", err)
+	}
+	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
+		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
+			len(*streamAggrConfig), len(*remoteWriteURLs))
+	}
 	for idx := range *streamAggrConfig {
-		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
-		if err != nil {
+		if _, err := newStreamAggrConfig(idx, pushNoop); err != nil {
 			return err
 		}
-		sas.MustStop()
 	}
 	return nil
 }
 
 func reloadStreamAggrConfigs() {
-	reloadStreamAggrConfigGlobal()
-	for _, rwctx := range rwctxsGlobal {
-		rwctx.reloadStreamAggrConfig()
+	reloadStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
+	for idx, rwctx := range rwctxs {
+		reloadStreamAggrConfig(idx, rwctx.pushInternalTrackDropped)
 	}
 }
 
-func reloadStreamAggrConfigGlobal() {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
-		return
-	}
-
-	sas := sasGlobal.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := sasGlobal.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
-}
-
-func initStreamAggrConfigGlobal() {
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		sasGlobal.Store(sas)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	}
-	dedupInterval := *streamAggrGlobalDedupInterval
-	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
-	}
-}
-
-func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
-	idx := rwctx.idx
-
-	sas, err := rwctx.newStreamAggrConfig()
-	if err != nil {
-		logger.Fatalf("cannot initialize stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	}
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-	if dedupInterval > 0 {
-		alias := fmt.Sprintf("dedup-%d", idx+1)
-		var dropLabels []string
-		if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-			dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
-		}
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, dropLabels, alias)
-	}
-}
-
-func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
-	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if path == "" {
-		return
-	}
-
+func reloadStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) {
+	path, opts := getStreamAggrOpts(idx)
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
 
-	sasNew, err := rwctx.newStreamAggrConfig()
+	sasNew, err := newStreamAggrConfigWithOpts(pushFunc, path, opts)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		logger.Errorf("cannot reload stream aggregation config at %q; continue using the previously loaded config; error: %s", path, err)
 		return
 	}
 
-	sas := rwctx.sas.Load()
+	var sas *streamaggr.Aggregators
+	if idx < 0 {
+		sas = sasGlobal.Load()
+	} else {
+		sas = rwctxs[idx].sas.Load()
+	}
+
 	if !sasNew.Equal(sas) {
-		sasOld := rwctx.sas.Swap(sasNew)
+		var sasOld *streamaggr.Aggregators
+		if idx < 0 {
+			sasOld = sasGlobal.Swap(sasNew)
+		} else {
+			sasOld = rwctxs[idx].sas.Swap(sasNew)
+		}
 		sasOld.MustStop()
-		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
+		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
 	} else {
 		sasNew.MustStop()
-		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
+		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
 }
 
-func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return nil, nil
+func getStreamAggrOpts(idx int) (string, streamaggr.Options) {
+	if idx < 0 {
+		return *streamAggrGlobalConfig, streamaggr.Options{
+			DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
+			DropInputLabels:      *streamAggrGlobalDropInputLabels,
+			IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
+			IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+			Alias:                "global",
+		}
 	}
-
-	opts := &streamaggr.Options{
-		DedupInterval:        *streamAggrGlobalDedupInterval,
-		DropInputLabels:      *streamAggrGlobalDropInputLabels,
-		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
-		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
-		KeepInput:            *streamAggrGlobalKeepInput,
-	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
-	}
-	return sas, nil
-}
-
-func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
-	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
-}
-
-func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path := streamAggrConfig.GetOptionalArg(idx)
-	if path == "" {
-		return nil, nil
-	}
-
-	alias := fmt.Sprintf("%d:secret-url", idx+1)
+	url := fmt.Sprintf("%d:secret-url", idx+1)
 	if *showRemoteWriteURL {
-		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
+		url = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
 	}
-	var dropLabels []string
-	if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-		dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
-	}
-	opts := &streamaggr.Options{
+	opts := streamaggr.Options{
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
-		DropInputLabels:      dropLabels,
+		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
-		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
+		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+		Alias:                url,
 	}
 
-	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
+	if len(*streamAggrConfig) == 0 {
+		return "", opts
 	}
-	return sas, nil
+	return streamAggrConfig.GetOptionalArg(idx), opts
+}
+
+func newStreamAggrConfigWithOpts(pushFunc streamaggr.PushFunc, path string, opts streamaggr.Options) (*streamaggr.Aggregators, error) {
+	if len(path) == 0 {
+		// Skip empty stream aggregation config.
+		return nil, nil
+	}
+	return streamaggr.LoadFromFile(path, pushFunc, opts)
+}
+
+func newStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
+	path, opts := getStreamAggrOpts(idx)
+	return newStreamAggrConfigWithOpts(pushFunc, path, opts)
 }

app/vmalert-tool/Makefile

@@ -81,9 +81,6 @@ vmalert-tool-linux-ppc64le:
 vmalert-tool-linux-s390x:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-vmalert-tool-linux-loong64:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 vmalert-tool-linux-386:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert-tool/main.go

@@ -41,19 +41,9 @@ Examples:
 						Usage:    "disable adding group's Name as label to generated alerts and time series.",
 						Required: false,
 					},
-					&cli.StringSliceFlag{
-						Name:     "external.label",
-						Usage:    `Optional label in the form 'name=value' to add to all generated recording rules and alerts. Supports an array of values separated by comma or specified via multiple flags.`,
-						Required: false,
-					},
-					&cli.StringFlag{
-						Name:     "external.url",
-						Usage:    `Optional external URL to template in rule's labels or annotations.`,
-						Required: false,
-					},
 				},
 				Action: func(c *cli.Context) error {
-					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url")); failed {
+					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
 						return fmt.Errorf("unittest failed")
 					}
 					return nil

app/vmalert-tool/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/vmalert-tool/unittest/input.go

@@ -43,33 +43,18 @@ func httpWrite(address string, r io.Reader) {
 // writeInputSeries send input series to vmstorage and flush them
 func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error {
 	r := testutil.WriteRequest{}
-	var err error
-	r.Timeseries, err = parseInputSeries(input, interval, startStamp)
-	if err != nil {
-		return err
-	}
-
-	data := testutil.Compress(r)
-	// write input series to vm
-	httpWrite(dst, bytes.NewBuffer(data))
-	vmstorage.Storage.DebugFlush()
-	return nil
-}
-
-func parseInputSeries(input []series, interval *promutils.Duration, startStamp time.Time) ([]testutil.TimeSeries, error) {
-	var res []testutil.TimeSeries
 	for _, data := range input {
 		expr, err := metricsql.Parse(data.Series)
 		if err != nil {
-			return res, fmt.Errorf("failed to parse series %s: %v", data.Series, err)
+			return fmt.Errorf("failed to parse series %s: %v", data.Series, err)
 		}
 		promvals, err := parseInputValue(data.Values, true)
 		if err != nil {
-			return res, fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
+			return fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
 		}
 		metricExpr, ok := expr.(*metricsql.MetricExpr)
-		if !ok || len(metricExpr.LabelFilterss) != 1 {
-			return res, fmt.Errorf("got invalid input series %s: %v", data.Series, err)
+		if !ok {
+			return fmt.Errorf("failed to parse series %s to metric expr: %v", data.Series, err)
 		}
 		samples := make([]testutil.Sample, 0, len(promvals))
 		ts := startStamp
@@ -86,9 +71,17 @@ func parseInputSeries(input []series, interval *promutils.Duration, startStamp t
 		for _, filter := range metricExpr.LabelFilterss[0] {
 			ls = append(ls, testutil.Label{Name: filter.Label, Value: filter.Value})
 		}
-		res = append(res, testutil.TimeSeries{Labels: ls, Samples: samples})
+		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
 	}
-	return res, nil
+
+	data, err := testutil.Compress(r)
+	if err != nil {
+		return fmt.Errorf("failed to compress data: %v", err)
+	}
+	// write input series to vm
+	httpWrite(dst, bytes.NewBuffer(data))
+	vmstorage.Storage.DebugFlush()
+	return nil
 }
 
 // parseInputValue support input like "1", "1+1x1 _ -4 3+20x1", see more examples in test.

app/vmalert-tool/unittest/input_test.go

@@ -2,98 +2,92 @@ package unittest
 
 import (
 	"testing"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-func TestParseInputValue_Failure(t *testing.T) {
-	f := func(input string) {
-		t.Helper()
-
-		_, err := parseInputValue(input, true)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
+func TestParseInputValue(t *testing.T) {
+	testCases := []struct {
+		input  string
+		exp    []sequenceValue
+		failed bool
+	}{
+		{
+			"",
+			nil,
+			true,
+		},
+		{
+			"testfailed",
+			nil,
+			true,
+		},
+		// stale doesn't support operations
+		{
+			"stalex3",
+			nil,
+			true,
+		},
+		{
+			"-4",
+			[]sequenceValue{{Value: -4}},
+			false,
+		},
+		{
+			"_",
+			[]sequenceValue{{Omitted: true}},
+			false,
+		},
+		{
+			"stale",
+			[]sequenceValue{{Value: decimal.StaleNaN}},
+			false,
+		},
+		{
+			"-4x1",
+			[]sequenceValue{{Value: -4}, {Value: -4}},
+			false,
+		},
+		{
+			"_x1",
+			[]sequenceValue{{Omitted: true}},
+			false,
+		},
+		{
+			"1+1x2 0.1 0.1+0.3x2 3.14",
+			[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}},
+			false,
+		},
+		{
+			"2-1x4",
+			[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
+			false,
+		},
+		{
+			"1+1x1 _ -4 stale 3+20x1",
+			[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
+			false,
+		},
 	}
 
-	f("")
-	f("testfailed")
-
-	// stale doesn't support operations
-	f("stalex3")
-}
-
-func TestParseInputValue_Success(t *testing.T) {
-	f := func(input string, outputExpected []sequenceValue) {
-		t.Helper()
-
-		output, err := parseInputValue(input, true)
-		if err != nil {
-			t.Fatalf("unexpected error in parseInputValue: %s", err)
+	for _, tc := range testCases {
+		output, err := parseInputValue(tc.input, true)
+		if err != nil != tc.failed {
+			t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
 		}
-
-		if len(outputExpected) != len(output) {
-			t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
+		if len(tc.exp) != len(output) {
+			t.Fatalf("expect %v, got %v", tc.exp, output)
 		}
-		for i := 0; i < len(outputExpected); i++ {
-			if outputExpected[i].Omitted != output[i].Omitted {
-				t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
+		for i := 0; i < len(tc.exp); i++ {
+			if tc.exp[i].Omitted != output[i].Omitted {
+				t.Fatalf("expect %v, got %v", tc.exp, output)
 			}
-			if outputExpected[i].Value != output[i].Value {
-				if decimal.IsStaleNaN(outputExpected[i].Value) && decimal.IsStaleNaN(output[i].Value) {
+			if tc.exp[i].Value != output[i].Value {
+				if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
 					continue
 				}
-				t.Fatalf("unexpected Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
+				t.Fatalf("expect %v, got %v", tc.exp, output)
 			}
 		}
 	}
-
-	f("-4", []sequenceValue{{Value: -4}})
-
-	f("_", []sequenceValue{{Omitted: true}})
-
-	f("stale", []sequenceValue{{Value: decimal.StaleNaN}})
-
-	f("-4x1", []sequenceValue{{Value: -4}, {Value: -4}})
-
-	f("_x1", []sequenceValue{{Omitted: true}})
-
-	f("1+1x2 0.1 0.1+0.3x2 3.14", []sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}})
-
-	f("2-1x4", []sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}})
-
-	f("1+1x1 _ -4 stale 3+20x1", []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}})
-}
-
-func TestParseInputSeries_Success(t *testing.T) {
-	f := func(input []series) {
-		t.Helper()
-		var interval promutils.Duration
-		_, err := parseInputSeries(input, &interval, time.Now())
-		if err != nil {
-			t.Fatalf("expect to see no error: %v", err)
-		}
-	}
-
-	f([]series{{Series: "test", Values: "1"}})
-	f([]series{{Series: "test{}", Values: "1"}})
-	f([]series{{Series: "test{env=\"prod\",job=\"a\" }", Values: "1"}})
-	f([]series{{Series: "{__name__=\"test\",env=\"prod\",job=\"a\" }", Values: "1"}})
-}
-
-func TestParseInputSeries_Fail(t *testing.T) {
-	f := func(input []series) {
-		t.Helper()
-		var interval promutils.Duration
-		_, err := parseInputSeries(input, &interval, time.Now())
-		if err == nil {
-			t.Fatalf("expect to see error: %v", err)
-		}
-	}
-
-	f([]series{{Series: "", Values: "1"}})
-	f([]series{{Series: "{}", Values: "1"}})
-	f([]series{{Series: "{env=\"prod\",job=\"a\" or env=\"dev\",job=\"b\"}", Values: "1"}})
 }

app/vmalert-tool/unittest/recording.go

@@ -57,18 +57,16 @@ Outer:
 					continue Outer
 				}
 				metricsqlMetricExpr, ok := metricsqlExpr.(*metricsql.MetricExpr)
-				if !ok || len(metricsqlMetricExpr.LabelFilterss) > 1 {
+				if !ok {
 					checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s, err: %v", mt.Expr,
-						mt.EvalTime.Duration().String(), fmt.Errorf("got invalid exp_samples: %q", s.Labels)))
+						mt.EvalTime.Duration().String(), fmt.Errorf("got unsupported metricsql type")))
 					continue Outer
 				}
-				if len(metricsqlMetricExpr.LabelFilterss) > 0 {
-					for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
-						expLb = append(expLb, datasource.Label{
-							Name:  l.Label,
-							Value: l.Value,
-						})
-					}
+				for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
+					expLb = append(expLb, datasource.Label{
+						Name:  l.Label,
+						Value: l.Value,
+					})
 				}
 			}
 			sort.Slice(expLb, func(i, j int) bool {

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -24,16 +24,20 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
+              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 in cluster  has been down for more than 5 minutes."
-              dashboard: "/d/dashboard?orgId=1"
+              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
 
       - eval_time: 0
         alertname: AlwaysFiring
         exp_alerts:
-          - {}
+          - exp_labels:
+              datacenter: dc-123
 
       - eval_time: 0
         alertname: InstanceDown
         exp_alerts: []
+
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -8,8 +8,7 @@ groups:
           severity: page
         annotations:
           summary: "Instance {{ $labels.instance }} down"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} in cluster {{ $externalLabels.cluster }} has been down for more than 5 minutes."
-          dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
       - alert: AlwaysFiring
         expr: 1
       - alert: SameAlertNameWithDifferentGroup

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -16,8 +16,7 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - exp_labels:
-              cluster: prod
+          - {}
       - eval_time: 1m
         groupname: group2
         alertname: SameAlertNameWithDifferentGroup
@@ -26,8 +25,7 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - exp_labels:
-              cluster: prod
+          - {}
       - eval_time: 6m
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
@@ -63,18 +61,18 @@ tests:
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t1", cluster="prod"}'
+            labels: '{__name__="t1", datacenter="dc-123"}'
       - expr: t2
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t2", cluster="prod"}'
+            labels: '{__name__="t2", datacenter="dc-123"}'
       - expr: t3
         eval_time: 4m
         exp_samples:
           # t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
           - value: 3
-            labels: '{__name__="t3", cluster="prod"}'
+            labels: '{__name__="t3", datacenter="dc-123"}'
 
     alert_rule_test:
       - eval_time: 10m
@@ -85,21 +83,22 @@ tests:
               job: vmagent1
               severity: page
               instance: localhost:9090
-              cluster: prod
+              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent1 in cluster prod has been down for more than 5 minutes."
-              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
+              description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              cluster: prod
+              datacenter: dc-123
 
       - eval_time: 0
         groupname: alerts
         alertname: InstanceDown
         exp_alerts: []
 
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test", cluster="prod", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -25,21 +25,22 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
-              cluster: prod
+              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 in cluster prod has been down for more than 5 minutes."
-              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
+              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              cluster: prod
+              datacenter: dc-123
 
       - eval_time: 0
         groupname: group1
         alertname: InstanceDown
         exp_alerts: []
 
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/unittest.go

@@ -55,7 +55,7 @@ const (
 )
 
 // UnitTest runs unittest for files
-func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL string) bool {
+func UnitTest(files []string, disableGroupLabel bool) bool {
 	if err := templates.Load([]string{}, true); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
@@ -71,34 +71,14 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 
 	testfiles, err := config.ReadFromFS(files)
 	if err != nil {
-		logger.Fatalf("failed to load test files %q: %v", files, err)
+		fmt.Println("  FAILED")
+		fmt.Printf("\nfailed to read test files: \n%v", err)
 	}
-	if len(testfiles) == 0 {
-		fmt.Println("no test file found")
-		return false
-	}
-
-	labels := make(map[string]string)
-	for _, s := range externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		labels[s[:n]] = s[n+1:]
-	}
-	_, err = notifier.Init(nil, labels, externalURL)
-	if err != nil {
-		logger.Fatalf("failed to init notifier: %v", err)
-	}
-
 	var failed bool
 	for fileName, file := range testfiles {
-		if err := ruleUnitTest(fileName, file, labels); err != nil {
+		if err := ruleUnitTest(fileName, file); err != nil {
 			fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", fileName, err)
+			fmt.Printf("\nfailed to run unit test for file %q: \n%v", file, err)
 			failed = true
 		} else {
 			fmt.Println("  SUCCESS")
@@ -108,7 +88,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	return failed
 }
 
-func ruleUnitTest(filename string, content []byte, externalLabels map[string]string) []error {
+func ruleUnitTest(filename string, content []byte) []error {
 	fmt.Println("\nUnit Testing: ", filename)
 	var unitTestInp unitTestFile
 	if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
@@ -146,7 +126,7 @@ func ruleUnitTest(filename string, content []byte, externalLabels map[string]str
 			errs = append(errs, err)
 			continue
 		}
-		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups, externalLabels)
+		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
 		errs = append(errs, testErrs...)
 	}
 
@@ -183,10 +163,6 @@ func verifyTestGroup(group testGroup) error {
 			return fmt.Errorf("\n%s    missing required field \"eval_time\"", testGroupName)
 		}
 	}
-	if group.ExternalLabels != nil {
-		fmt.Printf("\n%s    warning: filed `external_labels` will be deprecated soon, please use `-external.label` cmd-line flag instead. "+
-			"Check https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735 for details.\n", testGroupName)
-	}
 	return nil
 }
 
@@ -203,7 +179,6 @@ func processFlags() {
 		{flag: "retentionPeriod", value: "100y"},
 		{flag: "datasource.url", value: testDataSourcePath},
 		{flag: "remoteWrite.url", value: testRemoteWritePath},
-		{flag: "notifier.blackhole", value: "true"},
 	} {
 		// panics if flag doesn't exist
 		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
@@ -250,7 +225,7 @@ checkCheck:
 			if readyCheckFunc() {
 				break checkCheck
 			}
-			time.Sleep(100 * time.Millisecond)
+			time.Sleep(3 * time.Second)
 		}
 	}
 }
@@ -264,7 +239,7 @@ func tearDown() {
 	fs.MustRemoveAll(storagePath)
 }
 
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group, externalLabels map[string]string) (checkErrs []error) {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
 	// set up vmstorage and http server for ingest and read queries
 	setUp()
 	// tear down vmstorage and clean the data dir
@@ -313,14 +288,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	// create groups with given rule
 	var groups []*rule.Group
 	for _, group := range testGroups {
-		mergedExternalLabels := make(map[string]string)
-		for k, v := range tg.ExternalLabels {
-			mergedExternalLabels[k] = v
-		}
-		for k, v := range externalLabels {
-			mergedExternalLabels[k] = v
-		}
-		ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
+		ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
 		groups = append(groups, ng)
 	}
 

app/vmalert-tool/unittest/unittest_test.go

@@ -14,34 +14,34 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestUnitTest_Failure(t *testing.T) {
-	f := func(files []string) {
-		t.Helper()
-
-		failed := UnitTest(files, false, nil, "")
-		if !failed {
-			t.Fatalf("expecting failed test")
+func TestUnitRule(t *testing.T) {
+	testCases := []struct {
+		name              string
+		disableGroupLabel bool
+		files             []string
+		failed            bool
+	}{
+		{
+			name:   "run multi files",
+			files:  []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
+			failed: false,
+		},
+		{
+			name:              "disable group label",
+			disableGroupLabel: true,
+			files:             []string{"./testdata/disable-group-label.yaml"},
+			failed:            false,
+		},
+		{
+			name:   "failing test",
+			files:  []string{"./testdata/failed-test.yaml"},
+			failed: true,
+		},
+	}
+	for _, tc := range testCases {
+		fail := UnitTest(tc.files, tc.disableGroupLabel)
+		if fail != tc.failed {
+			t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
 		}
 	}
-
-	// failing test
-	f([]string{"./testdata/failed-test.yaml"})
-}
-
-func TestUnitTest_Success(t *testing.T) {
-	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
-		t.Helper()
-
-		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL)
-		if failed {
-			t.Fatalf("unexpected failed test")
-		}
-	}
-
-	// run multi files
-	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
-
-	// disable group label
-	// template with null external values
-	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
 }

app/vmalert/config/config.go

@@ -1,20 +1,19 @@
 package config
 
 import (
-	"bytes"
 	"crypto/md5"
 	"fmt"
 	"hash/fnv"
-	"io"
 	"net/url"
 	"sort"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"gopkg.in/yaml.v2"
 )
 
 // Group contains list of Rules grouped into
@@ -46,11 +45,11 @@ type Group struct {
 	// EvalAlignment will make the timestamp of group query requests be aligned with interval
 	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
+func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type group Group
 	if err := unmarshal((*group)(g)); err != nil {
 		return err
@@ -143,11 +142,11 @@ type Rule struct {
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (r *Rule) UnmarshalYAML(unmarshal func(any) error) error {
+func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rule Rule
 	if err := unmarshal((*rule)(r)); err != nil {
 		return err
@@ -299,33 +298,19 @@ func parseConfig(data []byte) ([]Group, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
-
-	var result []Group
-	type cfgFile struct {
+	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.
-		XXX map[string]any `yaml:",inline"`
+		XXX map[string]interface{} `yaml:",inline"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	if err != nil {
+		return nil, err
 	}
-
-	decoder := yaml.NewDecoder(bytes.NewReader(data))
-	for {
-		var cf cfgFile
-		if err = decoder.Decode(&cf); err != nil {
-			if err == io.EOF { // EOF indicates no more documents to read
-				break
-			}
-			return nil, err
-		}
-		if err = checkOverflow(cf.XXX, "config"); err != nil {
-			return nil, err
-		}
-		result = append(result, cf.Groups...)
-	}
-
-	return result, nil
+	return g.Groups, checkOverflow(g.XXX, "config")
 }
 
-func checkOverflow(m map[string]any, ctx string) error {
+func checkOverflow(m map[string]interface{}, ctx string) error {
 	if len(m) > 0 {
 		var keys []string
 		for k := range m {

app/vmalert/config/config_test.go

@@ -9,10 +9,11 @@ import (
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
@@ -22,6 +23,12 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
 func TestParseFromURL(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/bad", func(w http.ResponseWriter, _ *http.Request) {
@@ -39,34 +46,6 @@ groups:
 		w.Write([]byte(`
 groups:
   - name: TestGroup
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)`))
-	})
-	mux.HandleFunc("/good-multi-doc", func(w http.ResponseWriter, _ *http.Request) {
-		w.Write([]byte(`
-groups:
-  - name: foo
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)
----
-groups:
-  - name: bar
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)`))
-	})
-	mux.HandleFunc("/bad-multi-doc", func(w http.ResponseWriter, _ *http.Request) {
-		w.Write([]byte(`
-bad_field:
-  - name: foo
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)
----
-groups:
-  - name: bar
     rules:
       - record: conns
         expr: max(vm_tcplistener_conns)`))
@@ -75,366 +54,439 @@ groups:
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	f := func(urls []string, expErr bool) {
-		for i, u := range urls {
-			urls[i] = srv.URL + u
-		}
-		_, err := Parse(urls, notifier.ValidateTemplates, true)
-		if err != nil && !expErr {
-			t.Fatalf("error parsing URLs %s", err)
-		}
-		if err == nil && expErr {
-			t.Fatalf("expecting error parsing URLs but got none")
-		}
+	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
+		t.Errorf("error parsing URLs %s", err)
 	}
 
-	f([]string{"/good-alert", "/good-rr", "/good-multi-doc"}, false)
-	f([]string{"/bad"}, true)
-	f([]string{"/bad-multi-doc"}, true)
-	f([]string{"/good-alert", "/bad"}, true)
-}
-
-func TestParse_Success(t *testing.T) {
-	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
-	if err != nil {
-		t.Fatalf("error parsing files %s", err)
+	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
+		t.Errorf("expected parsing error: %s", err)
 	}
 }
 
-func TestParse_Failure(t *testing.T) {
-	f := func(paths []string, errStrExpected string) {
-		t.Helper()
-
-		_, err := Parse(paths, notifier.ValidateTemplates, true)
+func TestParseBad(t *testing.T) {
+	testCases := []struct {
+		path   []string
+		expErr string
+	}{
+		{
+			[]string{"testdata/rules/rules_interval_bad.rules"},
+			"eval_offset should be smaller than interval",
+		},
+		{
+			[]string{"testdata/rules/rules0-bad.rules"},
+			"unexpected token",
+		},
+		{
+			[]string{"testdata/dir/rules0-bad.rules"},
+			"error parsing annotation",
+		},
+		{
+			[]string{"testdata/dir/rules1-bad.rules"},
+			"duplicate in file",
+		},
+		{
+			[]string{"testdata/dir/rules2-bad.rules"},
+			"function \"unknown\" not defined",
+		},
+		{
+			[]string{"testdata/dir/rules3-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/dir/rules4-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/rules/rules1-bad.rules"},
+			"bad graphite expr",
+		},
+		{
+			[]string{"testdata/dir/rules6-bad.rules"},
+			"missing ':' in header",
+		},
+		{
+			[]string{"http://unreachable-url"},
+			"failed to",
+		},
+	}
+	for _, tc := range testCases {
+		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
 		if err == nil {
-			t.Fatalf("expected to get error")
+			t.Errorf("expected to get error")
+			return
 		}
-		if !strings.Contains(err.Error(), errStrExpected) {
-			t.Fatalf("expected err to contain %q; got %q instead", errStrExpected, err)
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
 		}
 	}
-
-	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
-	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
-	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
-	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
-	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
-	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
-	f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields")
-	f([]string{"testdata/rules/rules-multi-doc-duplicates-bad.rules"}, "duplicate")
-	f([]string{"http://unreachable-url"}, "failed to")
 }
 
-func TestRuleValidate(t *testing.T) {
+func TestRule_Validate(t *testing.T) {
 	if err := (&Rule{}).Validate(); err == nil {
-		t.Fatalf("expected empty name error")
+		t.Errorf("expected empty name error")
 	}
 	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
-		t.Fatalf("expected empty expr error")
+		t.Errorf("expected empty expr error")
 	}
 	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
-		t.Fatalf("expected valid rule; got %s", err)
+		t.Errorf("expected valid rule; got %s", err)
 	}
 }
 
-func TestGroupValidate_Failure(t *testing.T) {
-	f := func(group *Group, validateExpressions bool, errStrExpected string) {
-		t.Helper()
-
-		err := group.Validate(nil, validateExpressions)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		errStr := err.Error()
-		if !strings.Contains(errStr, errStrExpected) {
-			t.Fatalf("missing %q in the returned error %q", errStrExpected, errStr)
-		}
-	}
-
-	f(&Group{}, false, "group name must be set")
-
-	f(&Group{
-		Name:     "negative interval",
-		Interval: promutils.NewDuration(-1),
-	}, false, "interval shouldn't be lower than 0")
-
-	f(&Group{
-		Name:       "wrong eval_offset",
-		Interval:   promutils.NewDuration(time.Minute),
-		EvalOffset: promutils.NewDuration(2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
-
-	f(&Group{
-		Name:  "wrong limit",
-		Limit: -1,
-	}, false, "invalid limit")
-
-	f(&Group{
-		Name:        "wrong concurrency",
-		Concurrency: -1,
-	}, false, "invalid concurrency")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
+func TestGroup_Validate(t *testing.T) {
+	testCases := []struct {
+		group               *Group
+		rules               []Rule
+		validateAnnotations bool
+		validateExpressions bool
+		expErr              string
+	}{
+		{
+			group:  &Group{},
+			expErr: "group name must be set",
+		},
+		{
+			group: &Group{
+				Name:     "negative interval",
+				Interval: promutils.NewDuration(-1),
 			},
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
+			expErr: "interval shouldn't be lower than 0",
+		},
+		{
+			group: &Group{
+				Name:       "wrong eval_offset",
+				Interval:   promutils.NewDuration(time.Minute),
+				EvalOffset: promutils.NewDuration(2 * time.Minute),
 			},
+			expErr: "eval_offset should be smaller than interval",
 		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test graphite prometheus bad expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
+		{
+			group: &Group{
+				Name:  "wrong limit",
+				Limit: -1,
 			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
+			expErr: "invalid limit",
+		},
+		{
+			group: &Group{
+				Name:        "wrong concurrency",
+				Concurrency: -1,
 			},
+			expErr: "invalid concurrency",
 		},
-	}, false, "invalid rule")
-
-	f(&Group{
-		Name: "test graphite inherit",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sum(up == 0 ) by (host)",
-			},
-		},
-	}, false, "either `record` or `alert` must be set")
-
-	// validate expressions
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "invalid expression")
-
-	f(&Group{
-		Name: "test thanos",
-		Type: NewRawType("thanos"),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, true, "unknown datasource type")
-
-	f(&Group{
-		Name: "test graphite",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "some-description",
-			}},
-		},
-	}, true, "bad graphite expr")
-}
-
-func TestGroupValidate_Success(t *testing.T) {
-	f := func(group *Group, validateAnnotations, validateExpressions bool) {
-		t.Helper()
-
-		var validateTplFn ValidateTplFn
-		if validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
-		}
-		err := group.Validate(validateTplFn, validateExpressions)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, false, false)
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": "{{ value|query }}",
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
 				},
 			},
+			expErr: "",
 		},
-	}, false, false)
-
-	// validate annotiations
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": `
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr:              "invalid expression",
+			validateExpressions: true,
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": `
 {{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
   {{ . | first | value | humanize1024 }}B
 {{ end }}`,
+						},
+					},
+				},
+			},
+			validateAnnotations: true,
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test thanos",
+				Type: NewRawType("thanos"),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "unknown datasource type",
+		},
+		{
+			group: &Group{
+				Name: "test graphite",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "some-description",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{
+				Name: "test prometheus",
+				Type: NewPrometheusType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{
+				Name: "test graphite inherit",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sum(up == 0 ) by (host)",
+					},
 				},
 			},
 		},
-	}, true, false)
-
-	// validate expressions
-	f(&Group{
-		Name: "test prometheus",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
+		{
+			group: &Group{
+				Name: "test graphite prometheus bad expr",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sum(up == 0 ) by (host)",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+					},
+				},
+			},
+			expErr: "invalid rule",
 		},
-	}, false, true)
-}
-
-func TestHashRule_NotEqual(t *testing.T) {
-	f := func(a, b Rule) {
-		t.Helper()
-
-		aID, bID := HashRule(a), HashRule(b)
-		if aID == bID {
-			t.Fatalf("rule hashes mustn't be equal; got %d", aID)
-		}
 	}
 
-	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
-
-	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 2"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-		"foo": "baz",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1"})
-}
-
-func TestHashRule_Equal(t *testing.T) {
-	f := func(a, b Rule) {
-		t.Helper()
-
-		aID, bID := HashRule(a), HashRule(b)
-		if aID != bID {
-			t.Fatalf("rule hashes must be equal; got %d and %d", aID, bID)
+	for _, tc := range testCases {
+		var validateTplFn ValidateTplFn
+		if tc.validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
+		if err == nil {
+			if tc.expErr != "" {
+				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
+			}
+			continue
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
 		}
 	}
+}
 
-	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1"}, Rule{Alert: "alert", Expr: "up == 1"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-		"foo": "bar",
-	}})
-
-	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Alert: "record", Expr: "up == 1"})
-
-	f(Rule{
-		Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute),
-	}, Rule{Alert: "alert", Expr: "up == 1"})
+func TestHashRule(t *testing.T) {
+	testCases := []struct {
+		a, b  Rule
+		equal bool
+	}{
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1"},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "bar",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Alert: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			false,
+		},
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 2"},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "baz",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			false,
+		},
+	}
+	for i, tc := range testCases {
+		aID, bID := HashRule(tc.a), HashRule(tc.b)
+		if tc.equal != (aID == bID) {
+			t.Fatalf("missmatch for rule %d", i)
+		}
+	}
 }
 
 func TestGroupChecksum(t *testing.T) {

app/vmalert/config/log/logger.go

@@ -29,7 +29,7 @@ func (l *Logger) isDisabled() bool {
 }
 
 // Errorf logs error message.
-func (l *Logger) Errorf(format string, args ...any) {
+func (l *Logger) Errorf(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -37,7 +37,7 @@ func (l *Logger) Errorf(format string, args ...any) {
 }
 
 // Warnf logs warning message.
-func (l *Logger) Warnf(format string, args ...any) {
+func (l *Logger) Warnf(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -45,7 +45,7 @@ func (l *Logger) Warnf(format string, args ...any) {
 }
 
 // Infof logs info message.
-func (l *Logger) Infof(format string, args ...any) {
+func (l *Logger) Infof(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -54,6 +54,6 @@ func (l *Logger) Infof(format string, args ...any) {
 
 // Panicf logs panic message and panics.
 // Panicf can't be suppressed
-func (l *Logger) Panicf(format string, args ...any) {
+func (l *Logger) Panicf(format string, args ...interface{}) {
 	logger.Panicf(format, args...)
 }

app/vmalert/config/log/logger_test.go

@@ -18,14 +18,14 @@ func TestOutput(t *testing.T) {
 
 	mustMatch := func(exp string) {
 		t.Helper()
-
 		if exp == "" {
 			if testOutput.String() != "" {
-				t.Fatalf("expected output to be empty; got %q", testOutput.String())
+				t.Errorf("expected output to be empty; got %q", testOutput.String())
+				return
 			}
 		}
 		if !strings.Contains(testOutput.String(), exp) {
-			t.Fatalf("output %q should contain %q", testOutput.String(), exp)
+			t.Errorf("output %q should contain %q", testOutput.String(), exp)
 		}
 		fmt.Println(testOutput.String())
 		testOutput.Reset()

app/vmalert/config/testdata/rules/rules-multi-doc-bad.rules

@@ -1,29 +0,0 @@
-groups:
-  - name: groupTest
-    rules:
-      - alert: VMRows
-        for: 1ms
-        expr: vm_rows > 0
-        labels:
-          label: bar
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
-invalid-field-1: invalid-value-1
-invalid-field-2: invalid-value-2
----
-groups:
-  - name: TestGroup
-    interval: 2s
-    concurrency: 2
-    type: graphite
-    rules:
-      - alert: Conns
-        expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500)
-        for: 3m
-
-        annotations:
-          summary: Too high connection number for {{$labels.instance}}
-          description: "It is {{ $value }} connections for {{$labels.instance}}"
-invalid-field-2: invalid-value-2
-invalid-field-3: invalid-value-3

app/vmalert/config/testdata/rules/rules-multi-doc-duplicates-bad.rules

@@ -1,11 +0,0 @@
-groups:
-  - name: foo
-    rules:
-      - alert: VMRows
-        expr: vm_rows > 0
----
-groups:
-  - name: foo
-    rules:
-      - alert: VMRows
-        expr: vm_rows > 0

app/vmalert/config/testdata/rules/rules-multi-doc-good.rules

@@ -1,15 +0,0 @@
-
----
-groups:
-  - name: groupTest
-    rules:
-      - alert: VMRows
-        for: 1ms
-        expr: vm_rows > 0
-        labels:
-          label: bar
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
----
-groups:

app/vmalert/config/testdata/rules/rules-multi-doc2-good.rules

@@ -1,46 +0,0 @@
----
-groups:
-  - name: groupTest
-    rules:
-      - alert: VMRows
-        for: 1ms
-        expr: vm_rows > 0
-        labels:
-          label: bar
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
-  - name: groupTest-2
-    rules:
-      - alert: VMRows-2
-        for: 1ms
-        expr: vm_rows_2 > 0
-        labels:
-          label: bar2
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "\n markdown result is : \n---\n # header\n body: \n text \n----\n"
----
-groups:
-  - name: groupTest-3
-    rules:
-      - alert: VMRows-3
-        for: 1ms
-        expr: vm_rows_3 > 0
-        labels:
-          label: bar_3
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
-  - name: groupTest-4
-    rules:
-      - alert: VMRows-4
-        for: 1ms
-        expr: vm_rows_4 > 0
-        labels:
-          label: bar4
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
----
-groups: