Implement SBOM generation

Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -5,10 +5,10 @@ body:
   - type: markdown
     attributes:
       value: |
-        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade) 
+        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
         to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
         and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) first.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/troubleshooting/) first.
   - type: textarea
     id: describe-the-bug
     attributes:
@@ -60,12 +60,12 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
         
         See how to setup monitoring here:
-          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
+          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/#monitoring)
     validations:
       required: false
   - type: textarea

.github/ISSUE_TEMPLATE/question.yml

@@ -24,9 +24,9 @@ body:
       label: Troubleshooting docs
       description: I am familiar with the following troubleshooting docs
       options:
-        - label: General - https://docs.victoriametrics.com/victoriametrics/troubleshooting/
+        - label: General - https://docs.victoriametrics.com/troubleshooting/
           required: false
-        - label: vmagent - https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
+        - label: vmagent - https://docs.victoriametrics.com/vmagent/#troubleshooting
           required: false
-        - label: vmalert - https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting
+        - label: vmalert - https://docs.victoriametrics.com/vmalert/#troubleshooting
           required: false

.github/pull_request_template.md

@@ -6,4 +6,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 
 The following checks are **mandatory**:
 
-- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
+- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).

.github/workflows/docs.yaml

@@ -1,57 +0,0 @@
-name: publish-docs
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'docs/**'
-      - '.github/workflows/docs.yaml'
-  workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-        with:
-          path: __vm
-
-      - name: Checkout private code
-        uses: actions/checkout@v4
-        with:
-          repository: VictoriaMetrics/vmdocs
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: __vm-docs
-
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v6
-        id: import-gpg
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-          git_config_global: true
-
-      - name: Copy docs
-        id: update
-        run: |
-          find docs -type d -maxdepth 1 -mindepth 1 -exec \
-            sh -c 'rsync -zarvh --delete {}/ ../__vm-docs/content/$(basename {})/' \;
-          echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
-        working-directory: __vm
-
-      - name: Push to vmdocs
-        run: |
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          if [[ -n $(git status --porcelain) ]]; then
-            git add .
-            git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.update.outputs.SHORT_SHA }}"
-            git push
-          fi
-        working-directory: __vm-docs

.github/workflows/main.yml

@@ -85,38 +85,9 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
 
       - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
+        run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v4
         with:
-          files: ./coverage.txt
-
-  integration-test:
-    name: integration-test
-    needs: [lint, test]
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: Run integration tests
-        run: make integration-test
+          file: ./coverage.txt

.github/workflows/sync-docs.yml

@@ -0,0 +1,51 @@
+name: publish-docs
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'docs/**'
+  workflow_dispatch: {}
+permissions:
+  contents: read  # This is required for actions/checkout and to commit back image update
+  deployments: write
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+        with:
+          path: main
+      - name: Checkout private code
+        uses: actions/checkout@v4
+        with:
+          repository: VictoriaMetrics/vmdocs
+          token: ${{ secrets.VM_BOT_GH_TOKEN }}
+          path: docs
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@v6
+        with:
+          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+          workdir: docs
+      - name: Set short git commit SHA
+        id: vars
+        run: |
+          calculatedSha=$(git rev-parse --short ${{ github.sha }})
+          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
+        working-directory: main
+      - name: update code and commit
+        run: |
+          rm -rf content
+          cp -r ../main/docs content
+          make clean-after-copy
+          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
+          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
+          git add .
+          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
+          git push
+        working-directory: docs

.github/workflows/test_package.yml

@@ -0,0 +1,59 @@
+name: Package tests
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '.github/workflows/test_package.yml'
+      - '**/Dockerfile*'
+      - '**/Makefile'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '.github/workflows/test_package.yml'
+      - '**/Dockerfile*'
+      - '**/Makefile'
+
+permissions:
+  contents: read
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  test_release:
+    name: Test Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-artifacts-${{ runner.os }}-test-release-${{ steps.go.outputs.go-version }}-${{ hashFiles('Makefile', '**/Dockerfile*', '**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-test-release-
+
+      - name: Run make release
+        run: |
+          make clean
+          make release
+
+      - name: Run release tests
+        run: make test-release-victoria-metrics

.gitignore

@@ -7,7 +7,6 @@
 .vscode
 *.test
 *.swp
-/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
@@ -26,5 +25,3 @@ _site
 /docs/.jekyll-metadata
 coverage.txt
 cspell.json
-*~
-deployment/docker/provisioning/plugins/

.wwhrd.yml

@@ -4,4 +4,3 @@ allowlist:
   - BSD-3-Clause
   - BSD-2-Clause
   - ISC
-  - MPL-2.0

CONTRIBUTING.md

@@ -1 +1 @@
-The document has been moved [here](https://docs.victoriametrics.com/victoriametrics/contributing/).
+The document has been moved [here](https://docs.victoriametrics.com/contributing/).

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2024 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -5,6 +5,7 @@ MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
+LATEST_TAG ?= latest
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -12,21 +13,90 @@ PKG_TAG := $(BUILDINFO_TAG)
 endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
-TAR_OWNERSHIP ?= --owner=1000 --group=1000
+
+VICTORIA_LOGS_COMPONENTS = victoria-logs
+VICTORIA_METRICS_COMPONENTS = victoria-metrics
+VICTORIA_METRICS_UTILS_COMPONENTS = vmagent vmalert vmalert-tool vmauth vmbackup vmrestore vmctl
 
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include codespell/Makefile
+include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
 include package/release/Makefile
 
+define RELEASE_GOOS_GOARCH
+	$(eval PKG_NAME := $(1))
+	$(eval PKG_COMPONENTS := $(2))
+
+	# Build
+	$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+
+	# Generate SBOM
+	mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cyclonedx-gomod app -assert-licenses -json -licenses -packages \
+			-main app/$(pkg_component) \
+			-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
+	
+	# Pack and compress
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar \
+			$(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+	cd bin && gzip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar
+	cd bin && tar -czf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+
+	# Generate checksums
+	cd bin && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ >> \
+			$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
+
+	# Clean up
+	cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+	$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+endef
+
+define RELEASE_WINDOWS_GOARCH
+	$(eval PKG_NAME := $(1))
+	$(eval PKG_COMPONENTS := $(2))
+
+	# Build
+	$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+
+	# Generate SBOM
+	mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cyclonedx-gomod app -assert-licenses -json -licenses -packages \
+			-main app/$(pkg_component) \
+			-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
+	
+	# Pack and compress
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && zip -u $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip \
+			$(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
+	cd bin && zip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip -r $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+
+	# Generate checksums
+	cd bin && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
+
+	# Clean up
+	cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+	$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
+endef
+
+
 all: \
 	victoria-metrics-prod \
 	victoria-logs-prod \
-	vlogscli-prod \
 	vmagent-prod \
 	vmalert-prod \
 	vmalert-tool-prod \
@@ -51,7 +121,6 @@ publish: \
 package: \
 	package-victoria-metrics \
 	package-victoria-logs \
-	package-vlogscli \
 	package-vmagent \
 	package-vmalert \
 	package-vmalert-tool \
@@ -195,31 +264,12 @@ vmutils-crossbuild: \
 	vmutils-openbsd-amd64 \
 	vmutils-windows-amd64
 
-publish-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
-
-publish-victoria-logs-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-logs $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG) APP_NAME=vlogscli $(MAKE) publish-via-docker-latest
-
 publish-release:
 	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && $(MAKE) publish
+	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
 
 release:
 	$(MAKE_PARALLEL) \
@@ -262,34 +312,13 @@ release-victoria-metrics-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
 release-victoria-metrics-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
+	GOOS=windows GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
 
-release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
+release-victoria-metrics-goos-goarch:
+	$(call RELEASE_GOOS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
 
-release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-			> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		victoria-metrics-windows-$(GOARCH)-prod.exe
-
-release-victoria-logs-bundle: \
-	release-victoria-logs \
-	release-vlogscli
-
-publish-victoria-logs-bundle: \
-	publish-victoria-logs \
-	publish-vlogscli
+release-victoria-metrics-windows-goarch:
+	$(call RELEASE_WINDOWS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
 
 release-victoria-logs:
 	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
@@ -331,7 +360,7 @@ release-victoria-logs-windows-amd64:
 
 release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
@@ -348,63 +377,6 @@ release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-logs-windows-$(GOARCH)-prod.exe
 
-release-vlogscli:
-	$(MAKE_PARALLEL) release-vlogscli-linux-386 \
-		release-vlogscli-linux-amd64 \
-		release-vlogscli-linux-arm \
-		release-vlogscli-linux-arm64 \
-		release-vlogscli-darwin-amd64 \
-		release-vlogscli-darwin-arm64 \
-		release-vlogscli-freebsd-amd64 \
-		release-vlogscli-openbsd-amd64 \
-		release-vlogscli-windows-amd64
-
-release-vlogscli-linux-386:
-	GOOS=linux GOARCH=386 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-amd64:
-	GOOS=linux GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-arm:
-	GOOS=linux GOARCH=arm $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-arm64:
-	GOOS=linux GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-darwin-arm64:
-	GOOS=darwin GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-freebsd-amd64:
-	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-openbsd-amd64:
-	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vlogscli-windows-goarch
-
-release-vlogscli-goos-goarch: vlogscli-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vlogscli-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vlogscli-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf vlogscli-$(GOOS)-$(GOARCH)-prod
-
-release-vlogscli-windows-goarch: vlogscli-windows-$(GOARCH)-prod
-	cd bin && \
-		zip vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vlogscli-windows-$(GOARCH)-prod.exe \
-		&& sha256sum vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vlogscli-windows-$(GOARCH)-prod.exe \
-			> vlogscli-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vlogscli-windows-$(GOARCH)-prod.exe
-
 release-vmutils: \
 	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
@@ -418,7 +390,7 @@ release-vmutils: \
 
 release-vmutils-linux-386:
 	GOOS=linux GOARCH=386 $(MAKE) release-vmutils-goos-goarch
-
+	
 release-vmutils-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
@@ -441,77 +413,13 @@ release-vmutils-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
+	GOOS=windows GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
 
-release-vmutils-goos-goarch: \
-	vmagent-$(GOOS)-$(GOARCH)-prod \
-	vmalert-$(GOOS)-$(GOARCH)-prod \
-	vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-	vmauth-$(GOOS)-$(GOARCH)-prod \
-	vmbackup-$(GOOS)-$(GOARCH)-prod \
-	vmrestore-$(GOOS)-$(GOARCH)-prod \
-	vmctl-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOOS)-$(GOARCH)-prod \
-			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-			vmauth-$(GOOS)-$(GOARCH)-prod \
-			vmbackup-$(GOOS)-$(GOARCH)-prod \
-			vmrestore-$(GOOS)-$(GOARCH)-prod \
-			vmctl-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOOS)-$(GOARCH)-prod \
-			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-			vmauth-$(GOOS)-$(GOARCH)-prod \
-			vmbackup-$(GOOS)-$(GOARCH)-prod \
-			vmrestore-$(GOOS)-$(GOARCH)-prod \
-			vmctl-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vmagent-$(GOOS)-$(GOARCH)-prod \
-		vmalert-$(GOOS)-$(GOARCH)-prod \
-		vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-		vmauth-$(GOOS)-$(GOARCH)-prod \
-		vmbackup-$(GOOS)-$(GOARCH)-prod \
-		vmrestore-$(GOOS)-$(GOARCH)-prod \
-		vmctl-$(GOOS)-$(GOARCH)-prod
+release-vmutils-goos-goarch:
+	$(call RELEASE_GOOS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
 
-release-vmutils-windows-goarch: \
-	vmagent-windows-$(GOARCH)-prod \
-	vmalert-windows-$(GOARCH)-prod \
-	vmalert-tool-windows-$(GOARCH)-prod \
-	vmauth-windows-$(GOARCH)-prod \
-	vmbackup-windows-$(GOARCH)-prod \
-	vmrestore-windows-$(GOARCH)-prod \
-	vmctl-windows-$(GOARCH)-prod
-	cd bin && \
-		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vmagent-windows-$(GOARCH)-prod.exe \
-			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
-			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
-			vmctl-windows-$(GOARCH)-prod.exe \
-		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vmagent-windows-$(GOARCH)-prod.exe \
-			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
-			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
-			vmctl-windows-$(GOARCH)-prod.exe \
-			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vmagent-windows-$(GOARCH)-prod.exe \
-		vmalert-windows-$(GOARCH)-prod.exe \
-		vmalert-tool-windows-$(GOARCH)-prod.exe \
-		vmauth-windows-$(GOARCH)-prod.exe \
-		vmbackup-windows-$(GOARCH)-prod.exe \
-		vmrestore-windows-$(GOARCH)-prod.exe \
-		vmctl-windows-$(GOARCH)-prod.exe
+release-vmutils-windows-goarch:
+	$(call RELEASE_WINDOWS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
 
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
@@ -519,47 +427,42 @@ pprof-cpu:
 fmt:
 	gofmt -l -w -s ./lib
 	gofmt -l -w -s ./app
-	gofmt -l -w -s ./apptest
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
+	go vet ./lib/...
 	go vet ./app/...
-	go vet ./apptest/...
 
 check-all: fmt vet golangci-lint govulncheck
 
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
-
-integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore victoria-logs
-	go test ./apptest/... -skip="^TestCluster.*"
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
+	go test -bench=. ./lib/...
 	go test -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	go get -u ./lib/...
-	go get -u ./app/...
-	go mod tidy -compat=1.24
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.22
 	go mod vendor
 
 app-local:
@@ -582,10 +485,10 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`
@@ -604,3 +507,37 @@ install-wwhrd:
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
+
+cyclonedx-gomod-install:
+	which cyclonedx-gomod || go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest
+
+copy-docs:
+# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
+# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
+	echo "---" > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "sort: ${ORDER}" >> ${DST}; \
+		echo "weight: ${ORDER}" >> ${DST}; \
+		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
+	fi
+
+	echo "title: ${TITLE}" >> ${DST}
+	@if [ ${OLD_URL} ]; then \
+		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
+	fi
+	echo "---" >> ${DST}
+	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"/' ${DST}
+	rm -rf docs/*.tmp
+
+# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
+# For ORDER=0 it adds no order tag/weight tag.
+# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
+# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
+# Images starting with <img src="docs/ are replaced with <img src="
+# Cluster docs are supposed to be ordered as 2nd.
+# The rest of docs is ordered manually.
+docs-sync:
+	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
+	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs

SECURITY.md

@@ -6,13 +6,13 @@ The following versions of VictoriaMetrics receive regular security fixes:
 
 | Version | Supported          |
 |---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
+| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to <security@victoriametrics.com>
+Please report any security issues to security@victoriametrics.com

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 9428
 
 ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -45,13 +44,9 @@ func main() {
 
 	vlstorage.Init()
 	vlselect.Init()
-
-	insertutil.SetLogRowsStorage(&vlstorage.Storage{})
 	vlinsert.Init()
 
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -97,9 +92,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if vlselect.RequestHandler(w, r) {
 		return true
 	}
-	if vlstorage.RequestHandler(w, r) {
-		return true
-	}
 	return false
 }
 

app/victoria-logs/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/victoria-metrics/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 8428
 
 ENTRYPOINT ["/victoria-metrics-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./victoria-metrics-prod

app/victoria-metrics/main.go

@@ -14,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -32,7 +31,7 @@ var (
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication")
+		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/#deduplication")
 	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
 		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
@@ -40,24 +39,9 @@ var (
 		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
 		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
 		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
-	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmsingle can receive per second. Data ingestion is paused when the limit is exceeded. "+
-		"By default there are no limits on samples ingestion rate.")
-	finalDedupScheduleInterval = flag.Duration("storage.finalDedupScheduleCheckInterval", time.Hour, "The interval for checking when final deduplication process should be started."+
-		"Storage unconditionally adds 25% jitter to the interval value on each check evaluation."+
-		" Changing the interval to the bigger values may delay downsampling, deduplication for historical data."+
-		" See also https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication")
 )
 
 func main() {
-	// VictoriaMetrics is optimized for reduced memory allocations,
-	// so it can run with the reduced GOGC in order to reduce the used memory,
-	// while keeping CPU usage spent in GC at low levels.
-	//
-	// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
-	// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
-	// exceeds 0.05 for extended periods of time.
-	cgroup.SetGOGC(30)
-
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
@@ -90,20 +74,13 @@ func main() {
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
-	if *finalDedupScheduleInterval < time.Hour {
-		logger.Fatalf("-dedup.finalDedupScheduleCheckInterval cannot be smaller than 1 hour; got %s", *finalDedupScheduleInterval)
-	}
-	storage.SetFinalDedupScheduleInterval(*finalDedupScheduleInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
-	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()
 
 	startSelfScraper()
 
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -120,7 +97,6 @@ func main() {
 	}
 	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	vminsert.Stop()
-	vminsertcommon.StopIngestionRateLimiter()
 
 	vmstorage.Stop()
 	vmselect.Stop()

app/victoria-metrics/main_test.go

@@ -0,0 +1,619 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+const (
+	testFixturesDir            = "testdata"
+	testStorageSuffix          = "vm-test-storage"
+	testHTTPListenAddr         = ":7654"
+	testStatsDListenAddr       = ":2003"
+	testOpenTSDBListenAddr     = ":4242"
+	testOpenTSDBHTTPListenAddr = ":4243"
+	testLogLevel               = "INFO"
+)
+
+const (
+	testReadHTTPPath           = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath  = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath      = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testImportCSVWriteHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/import/csv"
+
+	testHealthHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+)
+
+const (
+	testStorageInitTimeout = 10 * time.Second
+)
+
+var (
+	storagePath   string
+	insertionTime = time.Now().UTC()
+)
+
+type test struct {
+	Name                     string   `json:"name"`
+	Data                     []string `json:"data"`
+	InsertQuery              string   `json:"insert_query"`
+	Query                    []string `json:"query"`
+	ResultMetrics            []Metric `json:"result_metrics"`
+	ResultSeries             Series   `json:"result_series"`
+	ResultQuery              Query    `json:"result_query"`
+	Issue                    string   `json:"issue"`
+	ExpectedResultLinesCount int      `json:"expected_result_lines_count"`
+}
+
+type Metric struct {
+	Metric     map[string]string `json:"metric"`
+	Values     []float64         `json:"values"`
+	Timestamps []int64           `json:"timestamps"`
+}
+
+func (r *Metric) UnmarshalJSON(b []byte) error {
+	type plain Metric
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+type Series struct {
+	Status string              `json:"status"`
+	Data   []map[string]string `json:"data"`
+}
+
+type Query struct {
+	Status string `json:"status"`
+	Data   struct {
+		ResultType string          `json:"resultType"`
+		Result     json.RawMessage `json:"result"`
+	} `json:"data"`
+}
+
+const rtVector, rtMatrix = "vector", "matrix"
+
+func (q *Query) metrics() ([]Metric, error) {
+	switch q.Data.ResultType {
+	case rtVector:
+		var r QueryInstant
+		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
+			return nil, err
+		}
+		return r.metrics()
+	case rtMatrix:
+		var r QueryRange
+		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
+			return nil, err
+		}
+		return r.metrics()
+	default:
+		return nil, fmt.Errorf("unknown result type %q", q.Data.ResultType)
+	}
+}
+
+type QueryInstant struct {
+	Result []struct {
+		Labels map[string]string `json:"metric"`
+		TV     [2]any            `json:"value"`
+	} `json:"result"`
+}
+
+func (q QueryInstant) metrics() ([]Metric, error) {
+	result := make([]Metric, len(q.Result))
+	for i, res := range q.Result {
+		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
+		}
+		var m Metric
+		m.Metric = res.Labels
+		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
+		m.Values = append(m.Values, f)
+		result[i] = m
+	}
+	return result, nil
+}
+
+type QueryRange struct {
+	Result []struct {
+		Metric map[string]string `json:"metric"`
+		Values [][]any           `json:"values"`
+	} `json:"result"`
+}
+
+func (q QueryRange) metrics() ([]Metric, error) {
+	var result []Metric
+	for i, res := range q.Result {
+		var m Metric
+		for _, tv := range res.Values {
+			f, err := strconv.ParseFloat(tv[1].(string), 64)
+			if err != nil {
+				return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, tv[1], err)
+			}
+			m.Values = append(m.Values, f)
+			m.Timestamps = append(m.Timestamps, int64(tv[0].(float64)))
+		}
+		if len(m.Values) < 1 || len(m.Timestamps) < 1 {
+			return nil, fmt.Errorf("metric %v contains no values", res)
+		}
+		m.Metric = q.Result[i].Metric
+		result = append(result, m)
+	}
+	return result, nil
+}
+
+func (q *Query) UnmarshalJSON(b []byte) error {
+	type plain Query
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(q))
+}
+
+func TestMain(m *testing.M) {
+	setUp()
+	code := m.Run()
+	tearDown()
+	os.Exit(code)
+}
+
+func setUp() {
+	storagePath = filepath.Join(os.TempDir(), testStorageSuffix)
+	processFlags()
+	logger.Init()
+	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
+	vmselect.Init()
+	vminsert.Init()
+	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
+	readyStorageCheckFunc := func() bool {
+		resp, err := http.Get(testHealthHTTPPath)
+		if err != nil {
+			return false
+		}
+		_ = resp.Body.Close()
+		return resp.StatusCode == 200
+	}
+	if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil {
+		log.Fatalf("http server can't start for %s seconds, err %s", testStorageInitTimeout, err)
+	}
+}
+
+func processFlags() {
+	flag.Parse()
+	for _, fv := range []struct {
+		flag  string
+		value string
+	}{
+		{flag: "storageDataPath", value: storagePath},
+		{flag: "httpListenAddr", value: testHTTPListenAddr},
+		{flag: "graphiteListenAddr", value: testStatsDListenAddr},
+		{flag: "opentsdbListenAddr", value: testOpenTSDBListenAddr},
+		{flag: "loggerLevel", value: testLogLevel},
+		{flag: "opentsdbHTTPListenAddr", value: testOpenTSDBHTTPListenAddr},
+	} {
+		// panics if flag doesn't exist
+		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
+			log.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
+		}
+	}
+}
+
+func waitFor(timeout time.Duration, f func() bool) error {
+	fraction := timeout / 10
+	for i := fraction; i < timeout; i += fraction {
+		if f() {
+			return nil
+		}
+		time.Sleep(fraction)
+	}
+	return fmt.Errorf("timeout")
+}
+
+func tearDown() {
+	if err := httpserver.Stop(*httpListenAddrs); err != nil {
+		log.Printf("cannot stop the webservice: %s", err)
+	}
+	vminsert.Stop()
+	vmstorage.Stop()
+	vmselect.Stop()
+	fs.MustRemoveAll(storagePath)
+}
+
+func TestWriteRead(t *testing.T) {
+	t.Run("write", testWrite)
+	time.Sleep(500 * time.Millisecond)
+	vmstorage.Storage.DebugFlush()
+	time.Sleep(1500 * time.Millisecond)
+	t.Run("read", testRead)
+}
+
+func testWrite(t *testing.T) {
+	t.Run("prometheus", func(t *testing.T) {
+		for _, test := range readIn("prometheus", insertionTime) {
+			if test.Data == nil {
+				continue
+			}
+			r := testutil.WriteRequest{}
+			testData := strings.Join(test.Data, "\n")
+			if err := json.Unmarshal([]byte(testData), &r.Timeseries); err != nil {
+				panic(fmt.Errorf("BUG: cannot unmarshal TimeSeries: %s\ntest data\n%s", err, testData))
+			}
+			if n := len(r.Timeseries); n <= 0 {
+				panic(fmt.Errorf("BUG: expecting non-empty Timeseries in test data:\n%s", testData))
+			}
+			data := testutil.Compress(r)
+			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
+		}
+	})
+	t.Run("csv", func(t *testing.T) {
+		for _, test := range readIn("csv", insertionTime) {
+			if test.Data == nil {
+				continue
+			}
+			httpWrite(t, testImportCSVWriteHTTPPath, test.InsertQuery, bytes.NewBuffer([]byte(strings.Join(test.Data, "\n"))))
+		}
+	})
+
+	t.Run("influxdb", func(t *testing.T) {
+		for _, x := range readIn("influxdb", insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				httpWrite(t, testWriteHTTPPath, test.InsertQuery, bytes.NewBufferString(strings.Join(test.Data, "\n")))
+			})
+		}
+	})
+	t.Run("graphite", func(t *testing.T) {
+		for _, x := range readIn("graphite", insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				tcpWrite(t, "127.0.0.1"+testStatsDListenAddr, strings.Join(test.Data, "\n"))
+			})
+		}
+	})
+	t.Run("opentsdb", func(t *testing.T) {
+		for _, x := range readIn("opentsdb", insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				tcpWrite(t, "127.0.0.1"+testOpenTSDBListenAddr, strings.Join(test.Data, "\n"))
+			})
+		}
+	})
+	t.Run("opentsdbhttp", func(t *testing.T) {
+		for _, x := range readIn("opentsdbhttp", insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				logger.Infof("writing %s", test.Data)
+				httpWrite(t, testOpenTSDBWriteHTTPPath, test.InsertQuery, bytes.NewBufferString(strings.Join(test.Data, "\n")))
+			})
+		}
+	})
+}
+
+func testRead(t *testing.T) {
+	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+		t.Run(engine, func(t *testing.T) {
+			for _, x := range readIn(engine, insertionTime) {
+				test := x
+				t.Run(test.Name, func(t *testing.T) {
+					t.Parallel()
+					for _, q := range test.Query {
+						q = testutil.PopulateTimeTplString(q, insertionTime)
+						if test.Issue != "" {
+							test.Issue = "\nRegression in " + test.Issue
+						}
+						switch {
+						case strings.HasPrefix(q, "/api/v1/export/csv"):
+							data := strings.Split(string(httpReadData(t, testReadHTTPPath, q)), "\n")
+							if len(data) == test.ExpectedResultLinesCount {
+								t.Fatalf("not expected number of csv lines want=%d\ngot=%d test=%s.%s\n\response=%q", len(data), test.ExpectedResultLinesCount, q, test.Issue, strings.Join(data, "\n"))
+							}
+						case strings.HasPrefix(q, "/api/v1/export"):
+							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
+								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/series"):
+							s := Series{}
+							httpReadStruct(t, testReadHTTPPath, q, &s)
+							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
+								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/query"):
+							queryResult := Query{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							gotMetrics, err := queryResult.metrics()
+							if err != nil {
+								t.Fatalf("failed to parse query response: %s", err)
+							}
+							expMetrics, err := test.ResultQuery.metrics()
+							if err != nil {
+								t.Fatalf("failed to parse expected response: %s", err)
+							}
+							if err := checkMetricsResult(gotMetrics, expMetrics); err != nil {
+								t.Fatalf("%q fails with error %s.%s", q, err, test.Issue)
+							}
+						default:
+							t.Fatalf("unsupported read query %s", q)
+						}
+					}
+				})
+			}
+		})
+	}
+}
+
+func readIn(readFor string, insertTime time.Time) []test {
+	testDir := filepath.Join(testFixturesDir, readFor)
+	var tt []test
+	err := filepath.Walk(testDir, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Ext(path) != ".json" {
+			return nil
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			panic(fmt.Errorf("BUG: cannot read %s: %s", path, err))
+		}
+		item := test{}
+		if err := json.Unmarshal(b, &item); err != nil {
+			panic(fmt.Errorf("cannot parse %T from %s: %s; data:\n%s", &item, path, err, b))
+		}
+		for i := range item.Data {
+			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
+		}
+		tt = append(tt, item)
+		return nil
+	})
+	if err != nil {
+		panic(fmt.Errorf("BUG: cannot read test data at %s: %w", testDir, err))
+	}
+	if len(tt) == 0 {
+		panic(fmt.Errorf("BUG: no tests found in %s", testDir))
+	}
+	return tt
+}
+
+func httpWrite(t *testing.T, address, query string, r io.Reader) {
+	t.Helper()
+
+	requestURL := address + query
+	resp, err := http.Post(requestURL, "", r)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != 204 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 204", requestURL, resp.StatusCode)
+	}
+}
+
+func tcpWrite(t *testing.T, address, data string) {
+	t.Helper()
+
+	conn, err := net.Dial("tcp", address)
+	if err != nil {
+		t.Fatalf("cannot dial %s: %s", address, err)
+	}
+	defer func() {
+		_ = conn.Close()
+	}()
+	n, err := conn.Write([]byte(data))
+	if err != nil {
+		t.Fatalf("cannot write %d bytes to %s: %s", len(data), address, err)
+	}
+	if n != len(data) {
+		panic(fmt.Errorf("BUG: conn.Write() returned unexpected number of written bytes to %s; got %d; want %d", address, n, len(data)))
+	}
+}
+
+func httpReadMetrics(t *testing.T, address, query string) []Metric {
+	t.Helper()
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+
+	var rows []Metric
+	dec := json.NewDecoder(resp.Body)
+	for {
+		var row Metric
+		err := dec.Decode(&row)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return rows
+			}
+			t.Fatalf("cannot decode %T from response received from %s: %s", &row, requestURL, err)
+		}
+		rows = append(rows, row)
+	}
+}
+
+func httpReadStruct(t *testing.T, address, query string, dst any) {
+	t.Helper()
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+	err = json.NewDecoder(resp.Body).Decode(dst)
+	if err != nil {
+		t.Fatalf("cannot decode %T from response received from %s: %s", dst, requestURL, err)
+	}
+}
+
+func httpReadData(t *testing.T, address, query string) []byte {
+	t.Helper()
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("cannot read response from %s: %s", requestURL, err)
+	}
+	return data
+}
+
+func checkMetricsResult(got, want []Metric) error {
+	for _, r := range append([]Metric(nil), got...) {
+		want = removeIfFoundMetrics(r, want)
+	}
+	if len(want) > 0 {
+		return fmt.Errorf("expected metrics %+v not found in %+v", want, got)
+	}
+	return nil
+}
+
+func removeIfFoundMetrics(r Metric, contains []Metric) []Metric {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) &&
+			reflect.DeepEqual(r.Timestamps, item.Timestamps) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkSeriesResult(got, want Series) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	wantData := append([]map[string]string(nil), want.Data...)
+	for _, r := range got.Data {
+		wantData = removeIfFoundSeries(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected seria(s) %+v not found in %+v", wantData, got.Data)
+	}
+	return nil
+}
+
+func removeIfFoundSeries(r map[string]string, contains []map[string]string) []map[string]string {
+	for i, item := range contains {
+		if reflect.DeepEqual(r, item) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func TestImportJSONLines(t *testing.T) {
+	f := func(labelsCount, labelLen int) {
+		t.Helper()
+
+		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
+		line := generateJSONLine(labelsCount, labelLen)
+		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
+		if err != nil {
+			t.Fatalf("cannot create request: %s", err)
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
+		}
+		if resp.StatusCode != 204 {
+			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
+		}
+	}
+
+	// labels with various lengths
+	for i := 0; i < 500; i++ {
+		f(10, i*5)
+	}
+
+	// Too many labels
+	f(1000, 100)
+
+	// Too long labels
+	f(1, 100_000)
+	f(10, 100_000)
+	f(10, 10_000)
+}
+
+func generateJSONLine(labelsCount, labelLen int) string {
+	m := make(map[string]string, labelsCount)
+	m["__name__"] = generateSizedRandomString(labelLen)
+	for j := 1; j < labelsCount; j++ {
+		labelName := generateSizedRandomString(labelLen)
+		labelValue := generateSizedRandomString(labelLen)
+		m[labelName] = labelValue
+	}
+
+	type jsonLine struct {
+		Metric     map[string]string `json:"metric"`
+		Values     []float64         `json:"values"`
+		Timestamps []int64           `json:"timestamps"`
+	}
+	line := &jsonLine{
+		Metric:     m,
+		Values:     []float64{1.34},
+		Timestamps: []int64{time.Now().UnixNano() / 1e6},
+	}
+	data, err := json.Marshal(&line)
+	if err != nil {
+		panic(fmt.Errorf("cannot marshal JSON: %w", err))
+	}
+	data = append(data, '\n')
+	return string(data)
+}
+
+const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
+
+func generateSizedRandomString(size int) string {
+	dst := make([]byte, size)
+	for i := range dst {
+		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
+	}
+	return string(dst)
+}

app/victoria-metrics/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
@@ -9,5 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]
 ARG TARGETARCH
-ARG BINARY_SUFFIX=non-existing
-COPY victoria-metrics-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./victoria-metrics-prod
+COPY victoria-metrics-linux-${TARGETARCH}-prod ./victoria-metrics-prod

app/victoria-metrics/self_scraper.go

@@ -10,10 +10,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
 var (
@@ -49,7 +48,7 @@ func selfScraper(scrapeInterval time.Duration) {
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
 	var mrs []storage.MetricRow
-	var labels []prompbmarshal.Label
+	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
 	f := func(currentTime time.Time, sendStaleMarkers bool) {
 		currentTimestamp := currentTime.UnixNano() / 1e6
@@ -69,10 +68,6 @@ func selfScraper(scrapeInterval time.Duration) {
 				t := &r.Tags[j]
 				labels = addLabel(labels, t.Key, t.Value)
 			}
-			if timeserieslimits.IsExceeding(labels) {
-				// Skip metric with exceeding labels.
-				continue
-			}
 			if len(mrs) < cap(mrs) {
 				mrs = mrs[:len(mrs)+1]
 			} else {
@@ -104,11 +99,11 @@ func selfScraper(scrapeInterval time.Duration) {
 	}
 }
 
-func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label {
+func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
 	if len(dst) < cap(dst) {
 		dst = dst[:len(dst)+1]
 	} else {
-		dst = append(dst, prompbmarshal.Label{})
+		dst = append(dst, prompb.Label{})
 	}
 	lb := &dst[len(dst)-1]
 	lb.Name = key

app/victoria-metrics/testdata/csv/basic.json

@@ -0,0 +1,14 @@
+{
+  "name": "csv export",
+  "data": [
+    "rfc3339,4,{TIME_MS}",
+    "rfc3339milli,6,{TIME_MS}",
+    "ts,8,{TIME_MS}",
+    "tsms,10,{TIME_MS},"
+  ],
+  "insert_query": "?format=1:label:tfmt,2:metric:test_csv,3:time:unix_ms",
+  "query": [
+    "/api/v1/export/csv?format=__name__,tfmt,__value__,__timestamp__:rfc3339&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
+  ],
+  "expected_result_lines_count": 4
+}

app/victoria-metrics/testdata/csv/with_extra_labels.json

@@ -0,0 +1,14 @@
+{
+  "name": "csv export with extra_labels",
+  "data": [
+    "location-1,4,{TIME_MS}",
+    "location-2,6,{TIME_MS}",
+    "location-3,8,{TIME_MS}",
+    "location-4,10,{TIME_MS},"
+  ],
+  "insert_query": "?format=1:label:location,2:metric:test_csv_labels,3:time:unix_ms&extra_label=location=location-1",
+  "query": [
+    "/api/v1/export/csv?format=__name__,location,__value__,__timestamp__:unix_ms&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
+  ],
+  "expected_result_lines_count": 4
+}

app/victoria-metrics/testdata/graphite/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["graphite.foo.bar.baz;tag1=value1;tag2=value2 123 {TIME_S}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"graphite.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -0,0 +1,16 @@
+{
+  "name": "comparison-not-inf-not-nan",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150",
+  "data": [
+    "not_nan_not_inf;item=x 1 {TIME_S-1m}",
+    "not_nan_not_inf;item=x 1 {TIME_S-2m}",
+    "not_nan_not_inf;item=y 3 {TIME_S-1m}",
+    "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
+  "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"], ["{TIME_S}","0.5"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -0,0 +1,16 @@
+{
+  "name": "empty-label-match",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/395",
+  "data": [
+    "empty_label_match 1 {TIME_S-1m}",
+    "empty_label_match;foo=bar 2 {TIME_S-1m}",
+    "empty_label_match;foo=baz 3 {TIME_S-1m}"],
+  "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S-1m}&end={TIME_S}&step=60"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"__name__":"empty_label_match"},"values":[["{TIME_S-1m}","1"],["{TIME_S}","1"]]},
+	      {"metric":{"__name__":"empty_label_match","foo":"bar"},"values":[["{TIME_S-1m}","2"],["{TIME_S}","2"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/graphite-selector.json

@@ -0,0 +1,17 @@
+{
+  "name": "graphite-selector",
+  "issue": "",
+  "data": [
+    "graphite-selector.bar.baz 1 {TIME_S-1m}",
+    "graphite-selector.xxx.yy  2 {TIME_S-1m}",
+    "graphite-selector.bb.cc 3 {TIME_S-1m}",
+    "graphite-selector.a.baz 4 {TIME_S-1m}"],
+  "query": ["/api/v1/query?query=sort({__graphite__='graphite-selector.*.baz'})&time={TIME_S-1m}"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"vector","result":[
+	    {"metric":{"__name__":"graphite-selector.bar.baz"},"value":["{TIME_S-1m}","1"]},
+	    {"metric":{"__name__":"graphite-selector.a.baz"},"value":["{TIME_S-1m}","4"]}
+    ]}
+  }
+}

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -0,0 +1,23 @@
+{
+  "name": "max_lookback_set",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_set 1 {TIME_S-30s}",
+    "max_lookback_set 2 {TIME_S-60s}",
+    "max_lookback_set 3 {TIME_S-120s}",
+    "max_lookback_set 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_set"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"],
+	      ["{TIME_S-10s}","1"],
+	      ["{TIME_S-0s}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -0,0 +1,31 @@
+{
+  "name": "max_lookback_unset",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_unset 1 {TIME_S-30s}",
+    "max_lookback_unset 2 {TIME_S-60s}",
+    "max_lookback_unset 3 {TIME_S-120s}",
+    "max_lookback_unset 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-140s}","4"],
+	      ["{TIME_S-130s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-110s}","3"],
+	      ["{TIME_S-100s}","3"],
+	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-80s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-50s}","2"],
+	      ["{TIME_S-40s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"],
+	      ["{TIME_S-10s}","1"],
+	      ["{TIME_S-0s}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/name-plus-negative-filter.json

@@ -0,0 +1,16 @@
+{
+  "name": "name-plus-negative-filter",
+  "issue": "",
+  "data": [
+    "name-plus-negative-filter;foo=123 1 {TIME_S-1m}",
+    "name-plus-negative-filter;bar=123 2 {TIME_S-1m}",
+    "name-plus-negative-filter;foo=qwe 3 {TIME_S-1m}"
+  ],
+  "query": ["/api/v1/query?query={__name__='name-plus-negative-filter',foo!='123'}&time={TIME_S-1m}"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"vector","result":[
+	    {"metric":{"__name__":"name-plus-negative-filter","foo":"qwe"},"value":["{TIME_S-1m}","3"]}
+    ]}
+  }
+}

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -0,0 +1,18 @@
+{
+  "name": "not-nan-as-missing-data",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153",
+  "data": [
+    "not_nan_as_missing_data;item=x 2 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=x 1 {TIME_S-1m}",
+    "not_nan_as_missing_data;item=y 4 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
+  ],
+  "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"x"},"values":[["{TIME_S-2m}","2"]]},
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"],["{TIME_S}", "3"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -0,0 +1,14 @@
+{
+  "name": "subquery-aggregation",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/184",
+  "data": [
+    "forms_daily_count;item=x 1 {TIME_S-1m}",
+    "forms_daily_count;item=x 2 {TIME_S-2m}",
+    "forms_daily_count;item=y 3 {TIME_S-1m}",
+    "forms_daily_count;item=y 4 {TIME_S-2m}"],
+  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}&latency_offset=1ms"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","2"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","4"]}]}
+  }
+}

app/victoria-metrics/testdata/influxdb/basic.json

@@ -0,0 +1,9 @@
+{
+  "name": "basic_insertion",
+  "data": ["measurement,tag1=value1,tag2=value2 field1=1.23,field2=123 {TIME_NS}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[1.23], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/influxdb/with_extra_labels.json

@@ -0,0 +1,10 @@
+{
+  "name": "insert_with_extra_labels",
+  "data": ["measurement,tag1=value1,tag2=value2 field6=1.23,field5=123 {TIME_NS}"],
+  "insert_query": "?extra_label=job=test&extra_label=tag2=value10",
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"measurement_field5","tag1":"value1","job": "test","tag2":"value10"},"values":[123], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"measurement_field6","tag1":"value1","job": "test","tag2":"value10"},"values":[1.23], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdb/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["put openstdb.foo.bar.baz {TIME_S} 123 tag1=value1 tag2=value2"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"openstdb.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["{\"metric\": \"opentsdbhttp.foo\", \"value\": 1001, \"timestamp\": {TIME_S}, \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.foo","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/multi_line.json

@@ -0,0 +1,9 @@
+{
+  "name": "multiline",
+  "data": ["[{\"metric\": \"opentsdbhttp.multiline1\", \"value\": 1001, \"timestamp\": \"{TIME_S}\", \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}, {\"metric\": \"opentsdbhttp.multiline2\", \"value\": 1002, \"timestamp\": {TIME_S}}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.multiline1","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]},
+    {"metric":{"__name__":"opentsdbhttp.multiline2"},"values":[1002], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/with_extra_labels.json

@@ -0,0 +1,9 @@
+{
+  "name": "insert_with_extra_labels",
+  "data": ["{\"metric\": \"opentsdbhttp.foobar\", \"value\": 1001, \"timestamp\": {TIME_S}, \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}"],
+  "insert_query": "?extra_label=job=open-test&extra_label=x=z",
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.foobar","bar":"baz","x":"z","job": "open-test"},"values":[1001], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.bar\"},{\"name\":\"baz\",\"value\":\"qux\"}],\"samples\":[{\"value\":100000,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.bar","baz":"qux"},"values":[100000], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/case-sensitive-regex.json

@@ -0,0 +1,10 @@
+{
+  "name": "case-sensitive-regex",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/161",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"sensitiveRegex\"}],\"samples\":[{\"value\":2,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"SensitiveRegex\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={label=~'(?i)sensitiveregex'}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"sensitiveRegex"},"values":[2], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"SensitiveRegex"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/duplicate-label.json

@@ -0,0 +1,9 @@
+{
+  "name": "duplicate_label",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/172",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.duplicate_label\"},{\"name\":\"duplicate\",\"value\":\"label\"},{\"name\":\"duplicate\",\"value\":\"label\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.duplicate_label","duplicate":"label"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/instant-matrix.json

@@ -0,0 +1,12 @@
+{
+  "name": "instant query with look-behind window",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
+  "query": ["/api/v1/query?query=foo[5m]"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[{"metric":{"__name__":"foo"},"values":[["{TIME_S-60s}", "1"]]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/instant-scalar.json

@@ -0,0 +1,11 @@
+{
+  "name": "instant scalar query",
+  "query": ["/api/v1/query?query=42&time={TIME_S}"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"vector",
+        "result":[{"metric":{},"value":["{TIME_S}", "42"]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/issue-5553-too-big-lookback.json

@@ -0,0 +1,13 @@
+{
+  "name": "too big look-behind window",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"},{\"name\":\"issue\",\"value\":\"5553\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
+  "query": ["/api/v1/query?query=foo{issue=\"5553\"}[100y]"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[{"metric":{"__name__":"foo", "issue": "5553"},"values":[["{TIME_S-60s}", "1"]]}]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/match-series.json

@@ -0,0 +1,15 @@
+{
+  "name": "match_series",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/155",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"1\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"2\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"3\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"4\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/series?match[]={__name__='MatchSeries'}", "/api/v1/series?match[]={__name__=~'MatchSeries.*'}"],
+  "result_series": {
+    "status": "success",
+    "data": [
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"1","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"2","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"3","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"4","TurbineType":"V112"}
+    ]
+  }
+}

app/victoria-metrics/testdata/prometheus/query-range.json

@@ -0,0 +1,18 @@
+{
+  "name": "query range",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"bar\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}, {\"value\":2,\"timestamp\":\"{TIME_MS-120s}\"}, {\"value\":1,\"timestamp\":\"{TIME_MS-180s}\"}]}]"],
+  "query": ["/api/v1/query_range?query=bar&step=30s&start={TIME_MS-180s}"],
+  "result_query": {
+      "status": "success",
+      "data":{
+        "resultType":"matrix",
+        "result":[
+          {
+            "metric":{"__name__":"bar"},
+            "values":[["{TIME_S-180s}", "1"],["{TIME_S-150s}", "1"],["{TIME_S-120s}", "2"],["{TIME_S-90s}", "2"], ["{TIME_S-60s}", "1"], ["{TIME_S-30s}", "1"], ["{TIME_S}", "1"]]
+          }
+        ]
+      }
+    }
+}

app/victoria-metrics/testdata/prometheus/with_extra_labels.json

@@ -0,0 +1,9 @@
+{
+  "name": "basic_insertion_with_extra_labels",
+  "insert_query": "?extra_label=job=prom-test&extra_label=baz=bar",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.foobar\"},{\"name\":\"baz\",\"value\":\"qux\"}],\"samples\":[{\"value\":100000,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.foobar","baz":"bar","job": "prom-test"},"values":[100000], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/with_request_extra_filter.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_select_with_extra_labels",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.tenant.limits\"},{\"name\":\"baz\",\"value\":\"qux\"},{\"name\":\"tenant\",\"value\":\"dev\"}],\"samples\":[{\"value\":100000,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.up\"},{\"name\":\"baz\",\"value\":\"qux\"}],\"samples\":[{\"value\":100000,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}&extra_label=tenant=dev"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.tenant.limits","baz":"qux","tenant": "dev"},"values":[100000], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/vlinsert/datadog/datadog.go

@@ -1,260 +0,0 @@
-package datadog
-
-import (
-	"bytes"
-	"fmt"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#stream-fields")
-	datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Comma-separated list of fields to ignore for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#dropping-fields")
-
-	maxRequestSize = flagutil.NewBytes("datadog.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single DataDog request")
-)
-
-var parserPool fastjson.ParserPool
-
-// RequestHandler processes Datadog insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/datadog/api/v1/validate":
-		fmt.Fprintf(w, `{}`)
-		return true
-	case "/insert/datadog/api/v2/logs":
-		return datadogLogsIngestion(w, r)
-	default:
-		return false
-	}
-}
-
-func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	startTime := time.Now()
-	v2LogsRequestsTotal.Inc()
-
-	var ts int64
-	if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
-		var err error
-		ts, err = strconv.ParseInt(tsValue, 10, 64)
-		if err != nil {
-			httpserver.Errorf(w, r, "could not parse dd-message-timestamp header value: %s", err)
-			return true
-		}
-		ts *= 1e6
-	} else {
-		ts = startTime.UnixNano()
-	}
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = *datadogStreamFields
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *datadogIgnoreFields
-	}
-
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("datadog", false)
-		err := readLogsRequest(ts, data, lmp)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read DataDog protocol data: %s", err)
-		return true
-	}
-
-	// update v2LogsRequestDuration only for successfully parsed requests
-	// There is no need in updating v2LogsRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	v2LogsRequestDuration.UpdateDuration(startTime)
-	w.WriteHeader(http.StatusAccepted)
-	fmt.Fprintf(w, `{}`)
-	return true
-}
-
-var (
-	v2LogsRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
-	v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
-)
-
-// datadog message field has two formats:
-//   - regular log message with string text
-//   - nested json format for serverless plugins
-//     which has the following format:
-//     {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
-//
-// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
-func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage.Field, error) {
-	switch v.Type() {
-	case fastjson.TypeString:
-		val := v.GetStringBytes()
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: bytesutil.ToUnsafeString(val),
-		})
-	case fastjson.TypeObject:
-		var firstErr error
-		v.GetObject().Visit(func(k []byte, v *fastjson.Value) {
-			if firstErr != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "_msg",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "status":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "status",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "lamdba":
-				obj, err := v.Object()
-				if err != nil {
-					firstErr = err
-					firstErr = fmt.Errorf("unexpected lambda value type for %q:%q; want object", k, v)
-					return
-				}
-				obj.Visit(func(k []byte, v *fastjson.Value) {
-					if firstErr != nil {
-						return
-					}
-					val, err := v.StringBytes()
-					if err != nil {
-						firstErr = fmt.Errorf("unexpected lambda label value type for %q:%q; want string", k, v)
-						return
-					}
-					fields = append(fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(val),
-					})
-				})
-
-			}
-		})
-	default:
-		return fields, fmt.Errorf("unsupported message type %q", v.Type().String())
-	}
-	return fields, nil
-}
-
-// readLogsRequest parses data according to DataDog logs format
-// https://docs.datadoghq.com/api/latest/logs/#send-logs
-func readLogsRequest(ts int64, data []byte, lmp insertutil.LogMessageProcessor) error {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-	records, err := v.Array()
-	if err != nil {
-		return fmt.Errorf("cannot extract array from parsed JSON: %w", err)
-	}
-
-	var fields []logstorage.Field
-	for _, r := range records {
-		o, err := r.Object()
-		if err != nil {
-			return fmt.Errorf("could not extract log record: %w", err)
-		}
-		o.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				fields, err = appendMsgFields(fields, v)
-				if err != nil {
-					return
-				}
-			case "timestamp":
-				val, e := v.Int64()
-				if e != nil {
-					err = fmt.Errorf("failed to parse timestamp for %q:%q", k, v)
-				}
-				if val > 0 {
-					ts = val * 1e6
-				}
-			case "ddtags":
-				// https://docs.datadoghq.com/getting_started/tagging/
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				var pair []byte
-				idx := 0
-				for idx >= 0 {
-					idx = bytes.IndexByte(val, ',')
-					if idx < 0 {
-						pair = val
-					} else {
-						pair = val[:idx]
-						val = val[idx+1:]
-					}
-					if len(pair) > 0 {
-						n := bytes.IndexByte(pair, ':')
-						if n < 0 {
-							// No tag value.
-							fields = append(fields, logstorage.Field{
-								Name:  bytesutil.ToUnsafeString(pair),
-								Value: "no_label_value",
-							})
-						}
-						fields = append(fields, logstorage.Field{
-							Name:  bytesutil.ToUnsafeString(pair[:n]),
-							Value: bytesutil.ToUnsafeString(pair[n+1:]),
-						})
-					}
-				}
-			default:
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				fields = append(fields, logstorage.Field{
-					Name:  bytesutil.ToUnsafeString(k),
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			}
-		})
-		if err != nil {
-			return err
-		}
-		lmp.AddRow(ts, fields, nil)
-		fields = fields[:0]
-	}
-	return nil
-}

app/vlinsert/datadog/datadog_test.go

@@ -1,104 +0,0 @@
-package datadog
-
-import (
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestReadLogsRequestFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-
-		lmp := &insertutil.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if err := lmp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`["create":{}]`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadLogsRequestSuccess(t *testing.T) {
-	f := func(data string, rowsExpected int, resultExpected string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-		var timestampsExpected []int64
-		for i := 0; i < rowsExpected; i++ {
-			timestampsExpected = append(timestampsExpected, ts)
-		}
-		lmp := &insertutil.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := lmp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	// Verify non-empty data
-	data := `[
-		{
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"bar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-                        "ddtags":"tag1:value1,tag2:value2",
-                        "hostname":"127.0.0.1",
-                        "message":{"message": "nested"},
-                        "service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"foobar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"baz",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"xyz",
-			"service":"test"
-		}, {
-			"ddsource": "nginx",
-			"ddtags":"tag1:value1,tag2:value2,",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-                }, {
-			"ddsource":"nginx",
-			"ddtags":",tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-		}
-	]`
-	rowsExpected := 7
-	resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"nested","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}`
-	f(data, rowsExpected, resultExpected)
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -1,22 +1,27 @@
 package elasticsearch
 
 import (
+	"bufio"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 )
 
@@ -30,38 +35,30 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	// This header is needed for Logstash
 	w.Header().Set("X-Elastic-Product", "Elasticsearch")
 
-	if strings.HasPrefix(path, "/insert/elasticsearch/_ilm/policy") {
+	if strings.HasPrefix(path, "/_ilm/policy") {
 		// Return fake response for Elasticsearch ilm request.
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_index_template") {
+	if strings.HasPrefix(path, "/_index_template") {
 		// Return fake response for Elasticsearch index template request.
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_ingest") {
+	if strings.HasPrefix(path, "/_ingest") {
 		// Return fake response for Elasticsearch ingest pipeline request.
 		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_nodes") {
+	if strings.HasPrefix(path, "/_nodes") {
 		// Return fake response for Elasticsearch nodes discovery request.
 		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/logstash") || strings.HasPrefix(path, "/insert/elasticsearch/_logstash") {
-		// Return fake response for Logstash APIs requests.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
 	switch path {
-	// some clients may omit trailing slash
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
-	case "/insert/elasticsearch/", "/insert/elasticsearch":
+	case "/":
 		switch r.Method {
 		case http.MethodGet:
 			// Return fake response for Elasticsearch ping request.
@@ -76,7 +73,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		}
 
 		return true
-	case "/insert/elasticsearch/_license":
+	case "/_license":
 		// Return fake response for Elasticsearch license request.
 		fmt.Fprintf(w, `{
 			"license": {
@@ -87,23 +84,22 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			}
 		}`)
 		return true
-	case "/insert/elasticsearch/_bulk":
+	case "/_bulk":
 		startTime := time.Now()
 		bulkRequestsTotal.Inc()
 
-		cp, err := insertutil.GetCommonParams(r)
+		cp, err := insertutils.GetCommonParams(r)
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		if err := insertutil.CanWriteData(); err != nil {
+		if err := vlstorage.CanWriteData(); err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk", true)
-		encoding := r.Header.Get("Content-Encoding")
-		streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-		n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeFields, cp.MsgFields, lmp)
+		lmp := cp.NewLogMessageProcessor()
+		isGzip := r.Header.Get("Content-Encoding") == "gzip"
+		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, lmp)
 		lmp.MustClose()
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
@@ -129,44 +125,67 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 
 var (
 	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
-	bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
+	rowsIngestedTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
+	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
+func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp insertutils.LogMessageProcessor) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return 0, fmt.Errorf("cannot decode Elasticsearch protocol data: %w", err)
+	if isGzip {
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
+		}
+		defer common.PutGzipReader(zr)
+		r = zr
 	}
-	defer protoparserutil.PutUncompressedReader(reader)
 
-	wcr := writeconcurrencylimiter.GetReader(reader)
+	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
-	lr := insertutil.NewLineReader(streamName, wcr)
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
 
 	n := 0
+	nCheckpoint := 0
 	for {
-		ok, err := readBulkLine(lr, timeFields, msgFields, lmp)
+		ok, err := readBulkLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil || !ok {
+			rowsIngestedTotal.Add(n - nCheckpoint)
 			return n, err
 		}
 		n++
+		if batchSize := n - nCheckpoint; n >= 1000 {
+			rowsIngestedTotal.Add(batchSize)
+			nCheckpoint = n
+		}
 	}
 }
 
-func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
+var lineBufferPool bytesutil.ByteBufferPool
+
+func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 
 	// Read the command, must be "create" or "index"
 	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
+						insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
 		}
-		line = lr.Line
+		line = sc.Bytes()
 	}
 	lineStr := bytesutil.ToUnsafeString(line)
 	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
@@ -174,51 +193,47 @@ func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp
 	}
 
 	// Decode log message
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
+	if !sc.Scan() {
+		if err := sc.Err(); err != nil {
+			if errors.Is(err, bufio.ErrTooLong) {
+				return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
+			}
 			return false, err
 		}
 		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
 	}
-	line = lr.Line
-	if len(line) == 0 {
-		// Special case - the line could be too long, so it was skipped.
-		// Continue parsing next lines.
-		return true, nil
-	}
+	line = sc.Bytes()
 	p := logstorage.GetJSONParser()
 	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
 
-	ts, err := extractTimestampFromFields(timeFields, p.Fields)
+	ts, err := extractTimestampFromFields(timeField, p.Fields)
 	if err != nil {
 		return false, fmt.Errorf("cannot parse timestamp: %w", err)
 	}
 	if ts == 0 {
 		ts = time.Now().UnixNano()
 	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
 	logstorage.PutJSONParser(p)
 
 	return true, nil
 }
 
-func extractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			timestamp, err := parseElasticsearchTimestamp(f.Value)
-			if err != nil {
-				return 0, err
-			}
-			f.Value = ""
-			return timestamp, nil
+func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
 		}
+		timestamp, err := parseElasticsearchTimestamp(f.Value)
+		if err != nil {
+			return 0, err
+		}
+		f.Value = ""
+		return timestamp, nil
 	}
 	return 0, nil
 }
@@ -230,8 +245,19 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
 		return 0, nil
 	}
 	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in seconds or milliseconds
-		return insertutil.ParseUnixTimestamp(s)
+		// Try parsing timestamp in milliseconds
+		n, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
+		}
+		if n > int64(math.MaxInt64)/1e6 {
+			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+		}
+		if n < int64(math.MinInt64)/1e6 {
+			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+		}
+		n *= 1e6
+		return n, nil
 	}
 	if len(s) == len("YYYY-MM-DD") {
 		t, err := time.Parse("2006-01-02", s)

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -2,24 +2,20 @@ package elasticsearch
 
 import (
 	"bytes"
+	"compress/gzip"
 	"fmt"
-	"github.com/golang/snappy"
-	"github.com/klauspost/compress/gzip"
-	"github.com/klauspost/compress/zlib"
-	"github.com/klauspost/compress/zstd"
-	"io"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestReadBulkRequest_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", []string{"_time"}, []string{"_msg"}, tlp)
+		rows, err := readBulkRequest(r, false, "_time", "_msg", tlp)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -37,87 +33,66 @@ foobar`)
 }
 
 func TestReadBulkRequest_Success(t *testing.T) {
-	f := func(data, encoding, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		timeFields := []string{"non_existing_foo", timeField, "non_existing_bar"}
-		msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", timeFields, msgFields, tlp)
+		rows, err := readBulkRequest(r, false, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 
 		// Read the request with compression
-		tlp = &insertutil.TestLogMessageProcessor{}
-		if encoding != "" {
-			data = compressData(data, encoding)
-		}
-		r = bytes.NewBufferString(data)
-		rows, err = readBulkRequest("test", r, encoding, timeFields, msgFields, tlp)
+		tlp = &insertutils.TestLogMessageProcessor{}
+		compressedData := compressData(data)
+		r = bytes.NewBufferString(compressedData)
+		rows, err = readBulkRequest(r, true, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatalf("verification failure after compression: %s", err)
 		}
 	}
 
 	// Verify an empty data
-	f("", "gzip", "_time", "_msg", nil, "")
-	f("\n", "gzip", "_time", "_msg", nil, "")
-	f("\n\n", "gzip", "_time", "_msg", nil, "")
+	f("", "_time", "_msg", 0, nil, "")
+	f("\n", "_time", "_msg", 0, nil, "")
+	f("\n\n", "_time", "_msg", 0, nil, "")
 
 	// Verify non-empty data
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
 {"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06 04:48:12.735+01:00","message":"baz"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
 {"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"1686026893735","x":"y"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty","@timestamp":"1686026893"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty float","@timestamp":"1686026123.62"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000, 1686026123620000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}
-{"_msg":"qwe rty"}
-{"_msg":"qwe rty float"}`
-	f(data, "zstd", timeField, msgField, timestampsExpected, resultExpected)
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 
-func compressData(s string, encoding string) string {
+func compressData(s string) string {
 	var bb bytes.Buffer
-	var zw io.WriteCloser
-	switch encoding {
-	case "gzip":
-		zw = gzip.NewWriter(&bb)
-	case "zstd":
-		zw, _ = zstd.NewWriter(&bb)
-	case "snappy":
-		return string(snappy.Encode(nil, []byte(s)))
-	case "deflate":
-		zw = zlib.NewWriter(&bb)
-	default:
-		panic(fmt.Errorf("%q encoding is not supported", encoding))
-	}
+	zw := gzip.NewWriter(&bb)
 	if _, err := zw.Write([]byte(s)); err != nil {
 		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
 	}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -5,29 +5,20 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("encoding:none", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "")
+	b.Run("gzip:off", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, false)
 	})
-	b.Run("encoding:gzip", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "gzip")
-	})
-	b.Run("encoding:zstd", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "zstd")
-	})
-	b.Run("encoding:deflate", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "deflate")
-	})
-	b.Run("encoding:snappy", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "snappy")
+	b.Run("gzip:on", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, true)
 	})
 }
 
-func benchmarkReadBulkRequest(b *testing.B, encoding string) {
+func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
 {"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"create":{"_index":"filebeat-8.8.0"}}
@@ -35,14 +26,14 @@ func benchmarkReadBulkRequest(b *testing.B, encoding string) {
 {"create":{"_index":"filebeat-8.8.0"}}
 {"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
-	if encoding != "" {
-		data = compressData(data, encoding)
+	if isGzip {
+		data = compressData(data)
 	}
 	dataBytes := bytesutil.ToUnsafeBytes(data)
 
-	timeFields := []string{"@timestamp"}
-	msgFields := []string{"message"}
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
+	timeField := "@timestamp"
+	msgField := "message"
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))
@@ -50,7 +41,7 @@ func benchmarkReadBulkRequest(b *testing.B, encoding string) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest("test", r, encoding, timeFields, msgFields, blp)
+			_, err := readBulkRequest(r, isGzip, timeField, msgField, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutil/common_params.go

@@ -1,335 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	defaultMsgValue = flag.String("defaultMsgValue", "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field",
-		"Default value for _msg field if the ingested log entry doesn't contain it; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field")
-)
-
-// CommonParams contains common HTTP parameters used by log ingestion APIs.
-//
-// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
-type CommonParams struct {
-	TenantID         logstorage.TenantID
-	TimeFields       []string
-	MsgFields        []string
-	StreamFields     []string
-	IgnoreFields     []string
-	DecolorizeFields []string
-	ExtraFields      []logstorage.Field
-
-	IsTimeFieldSet  bool
-	Debug           bool
-	DebugRequestURI string
-	DebugRemoteAddr string
-}
-
-// GetCommonParams returns CommonParams from r.
-func GetCommonParams(r *http.Request) (*CommonParams, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, err
-	}
-
-	var isTimeFieldSet bool
-	timeFields := []string{"_time"}
-	if tfs := httputil.GetArray(r, "_time_field", "VL-Time-Field"); len(tfs) > 0 {
-		isTimeFieldSet = true
-		timeFields = tfs
-	}
-
-	msgFields := httputil.GetArray(r, "_msg_field", "VL-Msg-Field")
-	streamFields := httputil.GetArray(r, "_stream_fields", "VL-Stream-Fields")
-	ignoreFields := httputil.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
-	decolorizeFields := httputil.GetArray(r, "decolorize_fields", "VL-Decolorize-Fields")
-
-	extraFields, err := getExtraFields(r)
-	if err != nil {
-		return nil, err
-	}
-
-	debug := false
-	if dv := httputil.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
-		debug, err = strconv.ParseBool(dv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
-		}
-	}
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-	if debug {
-		debugRequestURI = httpserver.GetRequestURI(r)
-		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
-	}
-
-	cp := &CommonParams{
-		TenantID:         tenantID,
-		TimeFields:       timeFields,
-		MsgFields:        msgFields,
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-
-		IsTimeFieldSet:  isTimeFieldSet,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
-	}
-
-	return cp, nil
-}
-
-func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
-	efs := httputil.GetArray(r, "extra_fields", "VL-Extra-Fields")
-	if len(efs) == 0 {
-		return nil, nil
-	}
-
-	extraFields := make([]logstorage.Field, len(efs))
-	for i, ef := range efs {
-		n := strings.Index(ef, "=")
-		if n <= 0 || n == len(ef)-1 {
-			return nil, fmt.Errorf(`invalid extra_field format: %q; must be in the form "field=value"`, ef)
-		}
-		extraFields[i] = logstorage.Field{
-			Name:  ef[:n],
-			Value: ef[n+1:],
-		}
-	}
-	return extraFields, nil
-}
-
-// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) *CommonParams {
-	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
-	if streamFields == nil {
-		streamFields = []string{
-			"hostname",
-			"app_name",
-			"proc_id",
-		}
-	}
-	cp := &CommonParams{
-		TenantID: tenantID,
-		TimeFields: []string{
-			"timestamp",
-		},
-		MsgFields: []string{
-			"message",
-		},
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-	}
-
-	return cp
-}
-
-// LogRowsStorage is an interface for ingesting logs into the storage.
-type LogRowsStorage interface {
-	// MustAddRows must add lr to the underlying storage.
-	MustAddRows(lr *logstorage.LogRows)
-
-	// CanWriteData must returns non-nil error if logs cannot be added to the underlying storage.
-	CanWriteData() error
-}
-
-var logRowsStorage LogRowsStorage
-
-// SetLogRowsStorage sets the storage for writing data to via LogMessageProcessor.
-//
-// This function must be called before using LogMessageProcessor and CanWriteData from this package.
-func SetLogRowsStorage(storage LogRowsStorage) {
-	logRowsStorage = storage
-}
-
-// CanWriteData returns non-nil error if data cannot be written to the underlying storage.
-func CanWriteData() error {
-	return logRowsStorage.CanWriteData()
-}
-
-// LogMessageProcessor is an interface for log message processors.
-type LogMessageProcessor interface {
-	// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
-	//
-	// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
-	//
-	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can reuse them.
-	AddRow(timestamp int64, fields, streamFields []logstorage.Field)
-
-	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
-	MustClose()
-}
-
-type logMessageProcessor struct {
-	mu            sync.Mutex
-	wg            sync.WaitGroup
-	stopCh        chan struct{}
-	lastFlushTime time.Time
-
-	cp *CommonParams
-	lr *logstorage.LogRows
-
-	rowsIngestedTotal  *metrics.Counter
-	bytesIngestedTotal *metrics.Counter
-}
-
-func (lmp *logMessageProcessor) initPeriodicFlush() {
-	lmp.lastFlushTime = time.Now()
-
-	lmp.wg.Add(1)
-	go func() {
-		defer lmp.wg.Done()
-
-		d := timeutil.AddJitterToDuration(time.Second)
-		ticker := time.NewTicker(d)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-lmp.stopCh:
-				return
-			case <-ticker.C:
-				lmp.mu.Lock()
-				if time.Since(lmp.lastFlushTime) >= d {
-					lmp.flushLocked()
-				}
-				lmp.mu.Unlock()
-			}
-		}
-	}()
-}
-
-// AddRow adds new log message to lmp with the given timestamp and fields.
-//
-// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
-func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// InsertRowProcessor is used by native data ingestion protocol parser.
-type InsertRowProcessor interface {
-	// AddInsertRow must add r to the underlying storage.
-	AddInsertRow(r *logstorage.InsertRow)
-}
-
-// AddInsertRow adds r to lmp.
-func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(r.Fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(r.Fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAddInsertRow(r)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// flushLocked must be called under locked lmp.mu.
-func (lmp *logMessageProcessor) flushLocked() {
-	lmp.lastFlushTime = time.Now()
-	logRowsStorage.MustAddRows(lmp.lr)
-	lmp.lr.ResetKeepSettings()
-}
-
-// MustClose flushes the remaining data to the underlying storage and closes lmp.
-func (lmp *logMessageProcessor) MustClose() {
-	close(lmp.stopCh)
-	lmp.wg.Wait()
-
-	lmp.flushLocked()
-	logstorage.PutLogRows(lmp.lr)
-	lmp.lr = nil
-}
-
-// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
-//
-// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
-func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode bool) LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.DecolorizeFields, cp.ExtraFields, *defaultMsgValue)
-	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
-	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
-	lmp := &logMessageProcessor{
-		cp: cp,
-		lr: lr,
-
-		rowsIngestedTotal:  rowsIngestedTotal,
-		bytesIngestedTotal: bytesIngestedTotal,
-
-		stopCh: make(chan struct{}),
-	}
-
-	if isStreamMode {
-		lmp.initPeriodicFlush()
-	}
-
-	return lmp
-}
-
-var (
-	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
-)

app/vlinsert/insertutil/flags.go

@@ -1,17 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-)
-
-var (
-	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
-
-	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
-)

app/vlinsert/insertutil/line_reader.go

@@ -1,158 +0,0 @@
-package insertutil
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-)
-
-// LineReader reads newline-delimited lines from the underlying reader
-type LineReader struct {
-	// Line contains the next line read after the call to NextLine
-	//
-	// The Line contents is valid until the next call to NextLine.
-	Line []byte
-
-	// name is the LineReader name
-	name string
-
-	// r is the underlying reader to read data from
-	r io.Reader
-
-	// buf is a buffer for reading the next line
-	buf []byte
-
-	// bufOffset is the offset at buf to read the next line from
-	bufOffset int
-
-	// err is the last error when reading data from r
-	err error
-
-	// eofReached is set to true when all the data is read from r
-	eofReached bool
-}
-
-// NewLineReader returns LineReader for r.
-func NewLineReader(name string, r io.Reader) *LineReader {
-	return &LineReader{
-		name: name,
-		r:    r,
-	}
-}
-
-// NextLine reads the next line from the underlying reader.
-//
-// It returns true if the next line is successfully read into Line.
-// If the line length exceeds MaxLineSizeBytes, then this line is skipped
-// and an empty line is returned instead.
-//
-// If false is returned, then no more lines left to read from r.
-// Check for Err in this case.
-func (lr *LineReader) NextLine() bool {
-	for {
-		lr.Line = nil
-		if lr.bufOffset >= len(lr.buf) {
-			if lr.err != nil || lr.eofReached {
-				return false
-			}
-			if !lr.readMoreData() {
-				return false
-			}
-			if lr.bufOffset >= len(lr.buf) && lr.eofReached {
-				return false
-			}
-		}
-
-		buf := lr.buf[lr.bufOffset:]
-		if n := bytes.IndexByte(buf, '\n'); n >= 0 {
-			lr.Line = buf[:n]
-			lr.bufOffset += n + 1
-			return true
-		}
-		if lr.eofReached {
-			lr.Line = buf
-			lr.bufOffset += len(buf)
-			return true
-		}
-		if !lr.readMoreData() {
-			return false
-		}
-	}
-}
-
-// Err returns the last error after NextLine call.
-func (lr *LineReader) Err() error {
-	if lr.err == nil {
-		return nil
-	}
-	return fmt.Errorf("%s: %s", lr.name, lr.err)
-}
-
-func (lr *LineReader) readMoreData() bool {
-	if lr.bufOffset > 0 {
-		lr.buf = append(lr.buf[:0], lr.buf[lr.bufOffset:]...)
-		lr.bufOffset = 0
-	}
-
-	bufLen := len(lr.buf)
-	if bufLen >= MaxLineSizeBytes.IntN() {
-		ok, skippedBytes := lr.skipUntilNextLine()
-		logger.Warnf("%s: the line length exceeds -insert.maxLineSizeBytes=%d; skipping it; total skipped bytes=%d",
-			lr.name, MaxLineSizeBytes.IntN(), skippedBytes)
-		tooLongLinesSkipped.Inc()
-		return ok
-	}
-
-	lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-	n, err := lr.r.Read(lr.buf[bufLen:])
-	lr.buf = lr.buf[:bufLen+n]
-	if err != nil {
-		if errors.Is(err, io.EOF) {
-			lr.eofReached = true
-			return true
-		}
-		lr.err = fmt.Errorf("cannot read the next line: %s", err)
-	}
-	return n > 0
-}
-
-var tooLongLinesSkipped = metrics.NewCounter("vl_too_long_lines_skipped_total")
-
-func (lr *LineReader) skipUntilNextLine() (bool, int) {
-
-	// Initialize skipped bytes count with MaxLineSizeBytes because
-	// we've already read that many bytes without encountering a newline,
-	// indicating the line size exceeds the maximum allowed limit.
-	skipSizeBytes := MaxLineSizeBytes.IntN()
-
-	for {
-		lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-		n, err := lr.r.Read(lr.buf)
-		skipSizeBytes += n
-		lr.buf = lr.buf[:n]
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				lr.eofReached = true
-				lr.buf = lr.buf[:0]
-				return true, skipSizeBytes
-			}
-			lr.err = fmt.Errorf("cannot skip the current line: %s", err)
-			return false, skipSizeBytes
-		}
-		if n := bytes.IndexByte(lr.buf, '\n'); n >= 0 {
-			// Include skipped bytes before \n, including the newline itself.
-			skipSizeBytes += n + 1 - len(lr.buf)
-			// Include \n in the buf, so too long line is replaced with an empty line.
-			// This is needed for maintaining synchorinzation consistency between lines
-			// in protocols such as Elasticsearch bulk import.
-			lr.buf = append(lr.buf[:0], lr.buf[n:]...)
-			return true, skipSizeBytes
-		}
-	}
-}

app/vlinsert/insertutil/line_reader_test.go

@@ -1,164 +0,0 @@
-package insertutil
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"reflect"
-	"testing"
-)
-
-func TestLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if len(lr.Line) > 0 {
-			t.Fatalf("unexpected non-empty line after failed NextLine(): %q", lr.Line)
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", []string{""})
-	f("\n\n", []string{"", ""})
-	f("foo", []string{"foo"})
-	f("foo\n", []string{"foo"})
-	f("\nfoo", []string{"", "foo"})
-	f("foo\n\n", []string{"foo", ""})
-	f("foo\nbar", []string{"foo", "bar"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\n\nbar\n\n", []string{"", "foo", "", "bar", ""})
-}
-
-func TestLineReader_SkipUntilNextLine(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error for data=%q: %s", data, err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines for data=%q\ngot\n%q\nwant\n%q", data, lines, linesExpected)
-		}
-	}
-
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLineLen := MaxLineSizeBytes.IntN() + overflow
-		longLine := string(make([]byte, longLineLen))
-
-		// Single long line
-		data := longLine
-		f(data, nil)
-
-		// Multiple long lines
-		data = longLine + "\n" + longLine
-		f(data, []string{""})
-
-		data = longLine + "\n" + longLine + "\n"
-		f(data, []string{"", ""})
-
-		// Long line in the middle
-		data = "foo\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "bar"})
-
-		// Multiple long lines in the middle
-		data = "foo\n" + longLine + "\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "", "bar"})
-
-		// Long line in the end
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		// Long line in the end
-		data = "foo\n" + longLine + "\n"
-		f(data, []string{"foo", ""})
-	}
-}
-
-func TestLineReader_Failure(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		fr := &failureReader{
-			r: bytes.NewBufferString(data),
-		}
-		lr := NewLineReader("foo", fr)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error on the second call")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("foo", nil)
-	f("foo\n", []string{"foo"})
-	f("\n", []string{""})
-	f("foo\nbar", []string{"foo"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\nbar\n\n", []string{"", "foo", "bar", ""})
-
-	// long line
-	longLineLen := MaxLineSizeBytes.IntN()
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLine := string(make([]byte, longLineLen+overflow))
-
-		data := longLine
-		f(data, nil)
-
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		data = longLine + "\nfoo"
-		f(data, []string{""})
-
-		data = longLine + "\nfoo\n"
-		f(data, []string{"", "foo"})
-	}
-}
-
-type failureReader struct {
-	r io.Reader
-}
-
-func (r *failureReader) Read(p []byte) (int, error) {
-	n, _ := r.r.Read(p)
-	if n > 0 {
-		return n, nil
-	}
-	return 0, fmt.Errorf("some error")
-}

app/vlinsert/insertutil/testutils.go

@@ -1,56 +0,0 @@
-package insertutil
-
-import (
-	"fmt"
-	"reflect"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// TestLogMessageProcessor implements LogMessageProcessor for testing.
-type TestLogMessageProcessor struct {
-	timestamps []int64
-	rows       []string
-}
-
-// AddRow adds row with the given timestamp and fields to tlp
-func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("BUG: streamFields must be nil; got %v", streamFields))
-	}
-	tlp.timestamps = append(tlp.timestamps, timestamp)
-	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
-}
-
-// MustClose closes tlp.
-func (tlp *TestLogMessageProcessor) MustClose() {
-}
-
-// Verify verifies the number of rows, timestamps and results after AddRow calls.
-func (tlp *TestLogMessageProcessor) Verify(timestampsExpected []int64, resultExpected string) error {
-	result := strings.Join(tlp.rows, "\n")
-	if len(tlp.rows) != len(timestampsExpected) {
-		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), len(timestampsExpected), result, resultExpected)
-	}
-
-	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
-		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
-	}
-	if result != resultExpected {
-		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-	}
-
-	return nil
-}
-
-// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
-type BenchmarkLogMessageProcessor struct{}
-
-// AddRow implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _, _ []logstorage.Field) {
-}
-
-// MustClose implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) MustClose() {
-}

app/vlinsert/insertutil/timestamp.go

@@ -1,106 +0,0 @@
-package insertutil
-
-import (
-	"fmt"
-	"math"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ExtractTimestampFromFields extracts timestamp in nanoseconds from the first field the name from timeFields at fields.
-//
-// The value for the corresponding timeFields is set to empty string after returning from the function,
-// so it could be ignored during data ingestion.
-//
-// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			nsecs, err := parseTimestamp(f.Value)
-			if err != nil {
-				return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", f.Name, err)
-			}
-			f.Value = ""
-			if nsecs == 0 {
-				nsecs = time.Now().UnixNano()
-			}
-			return nsecs, nil
-		}
-	}
-	return time.Now().UnixNano(), nil
-}
-
-func parseTimestamp(s string) (int64, error) {
-	// "-" is a nil timestamp value, if the syslog
-	// application is incapable of obtaining system time
-	// https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
-	if s == "" || s == "0" || s == "-" {
-		return time.Now().UnixNano(), nil
-	}
-	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
-		return ParseUnixTimestamp(s)
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
-	}
-	return nsecs, nil
-}
-
-// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
-func ParseUnixTimestamp(s string) (int64, error) {
-	if strings.IndexByte(s, '.') >= 0 {
-		// Parse timestamp as floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-		}
-		if f < (1<<31) && f >= (-1<<31) {
-			// The timestamp is in seconds.
-			return int64(f * 1e9), nil
-		}
-		if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
-			// The timestamp is in milliseconds.
-			return int64(f * 1e6), nil
-		}
-		if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
-			// The timestamp is in microseconds.
-			return int64(f * 1e3), nil
-		}
-		// The timestamp is in nanoseconds
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		return int64(f), nil
-	}
-
-	// Parse timestamp as integer
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-	}
-	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds.
-		return n * 1e9, nil
-	}
-	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
-		// The timestamp is in milliseconds.
-		return n * 1e6, nil
-	}
-	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
-		// The timestamp is in microseconds.
-		return n * 1e3, nil
-	}
-	// The timestamp is in nanoseconds
-	return n, nil
-}

app/vlinsert/insertutil/timestamp_test.go

@@ -1,185 +0,0 @@
-package insertutil
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestParseUnixTimestamp_Success(t *testing.T) {
-	f := func(s string, timestampExpected int64) {
-		t.Helper()
-
-		timestamp, err := ParseUnixTimestamp(s)
-		if err != nil {
-			t.Fatalf("unexpected error in ParseUnixTimestamp(%q): %s", s, err)
-		}
-		if timestamp != timestampExpected {
-			t.Fatalf("unexpected timestamp returned from ParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
-		}
-	}
-
-	f("0", 0)
-
-	// nanoseconds
-	f("-1234567890123456789", -1234567890123456789)
-	f("1234567890123456789", 1234567890123456789)
-
-	// microseconds
-	f("-1234567890123456", -1234567890123456000)
-	f("1234567890123456", 1234567890123456000)
-	f("1234567890123456.789", 1234567890123456768)
-
-	// milliseconds
-	f("-1234567890123", -1234567890123000000)
-	f("1234567890123", 1234567890123000000)
-	f("1234567890123.456", 1234567890123456000)
-
-	// seconds
-	f("-1234567890", -1234567890000000000)
-	f("1234567890", 1234567890000000000)
-	f("-1234567890.123456", -1234567890123456000)
-}
-
-func TestParseUnixTimestamp_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := ParseUnixTimestamp(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error in ParseUnixTimestamp(%q)", s)
-		}
-	}
-
-	// non-numeric timestamp
-	f("")
-	f("foobar")
-	f("foo.bar")
-
-	// too big timestamp
-	f("12345678901234567890")
-	f("-12345678901234567890")
-	f("12345678901234567890.235424")
-	f("-12345678901234567890.235424")
-}
-
-func TestExtractTimestampFromFields_Success(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs != nsecsExpected {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
-		}
-
-		for _, f := range fields {
-			if f.Name == timeField {
-				if f.Value != "" {
-					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
-				}
-			}
-		}
-	}
-
-	// UTC time
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20Z"},
-	}, 1718753840000000000)
-
-	// Time with timezone
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
-	}, 1718725040000000000)
-
-	// SQL datetime format
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
-	}, 1718773640123000000)
-
-	// Time with nanosecond precision
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
-		{Name: "foo", Value: "bar"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in nanoseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456789"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in microseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456"},
-	}, 1718773640123456000)
-
-	// Unix timestamp in milliseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123"},
-	}, 1718773640123000000)
-
-	// Unix timestamp in seconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640"},
-	}, 1718773640000000000)
-}
-
-func TestExtractTimestampFromFields_Now(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs < 1 {
-			t.Fatalf("expected generated timestamp, got error: %s", err)
-		}
-	}
-
-	// RFC5424 allows `-` for nil timestamp (log ingestion time)
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "-"},
-	})
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: ""},
-	})
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "0"},
-	})
-}
-
-func TestExtractTimestampFromFields_Error(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		fields := []logstorage.Field{
-			{Name: "time", Value: s},
-		}
-		nsecs, err := ExtractTimestampFromFields([]string{"time"}, fields)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if nsecs != 0 {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
-		}
-	}
-
-	// invalid time
-	f("foobar")
-
-	// incomplete time
-	f("2024-06-18")
-	f("2024-06-18T23:37")
-}

app/vlinsert/insertutils/common_params.go

@@ -0,0 +1,200 @@
+package insertutils
+
+import (
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+)
+
+// CommonParams contains common HTTP parameters used by log ingestion APIs.
+//
+// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
+type CommonParams struct {
+	TenantID     logstorage.TenantID
+	TimeField    string
+	MsgField     string
+	StreamFields []string
+	IgnoreFields []string
+
+	Debug           bool
+	DebugRequestURI string
+	DebugRemoteAddr string
+}
+
+// GetCommonParams returns CommonParams from r.
+func GetCommonParams(r *http.Request) (*CommonParams, error) {
+	// Extract tenantID
+	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// Extract time field name from _time_field query arg
+	var timeField = "_time"
+	if tf := r.FormValue("_time_field"); tf != "" {
+		timeField = tf
+	}
+
+	// Extract message field name from _msg_field query arg
+	var msgField = ""
+	if msgf := r.FormValue("_msg_field"); msgf != "" {
+		msgField = msgf
+	}
+
+	streamFields := httputils.GetArray(r, "_stream_fields")
+	ignoreFields := httputils.GetArray(r, "ignore_fields")
+
+	debug := httputils.GetBool(r, "debug")
+	debugRequestURI := ""
+	debugRemoteAddr := ""
+	if debug {
+		debugRequestURI = httpserver.GetRequestURI(r)
+		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
+	}
+
+	cp := &CommonParams{
+		TenantID:        tenantID,
+		TimeField:       timeField,
+		MsgField:        msgField,
+		StreamFields:    streamFields,
+		IgnoreFields:    ignoreFields,
+		Debug:           debug,
+		DebugRequestURI: debugRequestURI,
+		DebugRemoteAddr: debugRemoteAddr,
+	}
+	return cp, nil
+}
+
+// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
+	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
+	cp := &CommonParams{
+		TenantID:  tenantID,
+		TimeField: "timestamp",
+		MsgField:  "message",
+		StreamFields: []string{
+			"hostname",
+			"app_name",
+			"proc_id",
+		},
+	}
+
+	return cp
+}
+
+// LogMessageProcessor is an interface for log message processors.
+type LogMessageProcessor interface {
+	// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
+	//
+	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
+	AddRow(timestamp int64, fields []logstorage.Field)
+
+	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
+	MustClose()
+}
+
+type logMessageProcessor struct {
+	mu            sync.Mutex
+	wg            sync.WaitGroup
+	stopCh        chan struct{}
+	lastFlushTime time.Time
+
+	cp *CommonParams
+	lr *logstorage.LogRows
+}
+
+func (lmp *logMessageProcessor) initPeriodicFlush() {
+	lmp.lastFlushTime = time.Now()
+
+	lmp.wg.Add(1)
+	go func() {
+		defer lmp.wg.Done()
+
+		d := timeutil.AddJitterToDuration(time.Second)
+		ticker := time.NewTicker(d)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-lmp.stopCh:
+				return
+			case <-ticker.C:
+				lmp.mu.Lock()
+				if time.Since(lmp.lastFlushTime) >= d {
+					lmp.flushLocked()
+				}
+				lmp.mu.Unlock()
+			}
+		}
+	}()
+}
+
+// AddRow adds new log message to lmp with the given timestamp and fields.
+func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	lmp.mu.Lock()
+	defer lmp.mu.Unlock()
+
+	if len(fields) > *MaxFieldsPerLine {
+		rf := logstorage.RowFormatter(fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		rowsDroppedTotalTooManyFields.Inc()
+		return
+	}
+
+	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
+	if lmp.cp.Debug {
+		s := lmp.lr.GetRowString(0)
+		lmp.lr.ResetKeepSettings()
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		rowsDroppedTotalDebug.Inc()
+		return
+	}
+	if lmp.lr.NeedFlush() {
+		lmp.flushLocked()
+	}
+}
+
+// flushLocked must be called under locked lmp.mu.
+func (lmp *logMessageProcessor) flushLocked() {
+	lmp.lastFlushTime = time.Now()
+	vlstorage.MustAddRows(lmp.lr)
+	lmp.lr.ResetKeepSettings()
+}
+
+// MustClose flushes the remaining data to the underlying storage and closes lmp.
+func (lmp *logMessageProcessor) MustClose() {
+	close(lmp.stopCh)
+	lmp.wg.Wait()
+
+	lmp.flushLocked()
+	logstorage.PutLogRows(lmp.lr)
+	lmp.lr = nil
+}
+
+// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
+//
+// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
+func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	lmp := &logMessageProcessor{
+		cp: cp,
+		lr: lr,
+
+		stopCh: make(chan struct{}),
+	}
+	lmp.initPeriodicFlush()
+
+	return lmp
+}
+
+var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/flags.go

@@ -0,0 +1,15 @@
+package insertutils
+
+import (
+	"flag"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+)
+
+var (
+	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+
+	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
+)

app/vlinsert/insertutils/testutils.go

@@ -0,0 +1,53 @@
+package insertutils
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// TestLogMessageProcessor implements LogMessageProcessor for testing.
+type TestLogMessageProcessor struct {
+	timestamps []int64
+	rows       []string
+}
+
+// AddRow adds row with the given timestamp and fields to tlp
+func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
+	tlp.timestamps = append(tlp.timestamps, timestamp)
+	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
+}
+
+// MustClose closes tlp.
+func (tlp *TestLogMessageProcessor) MustClose() {
+}
+
+// Verify verifies the number of rows, timestamps and results after AddRow calls.
+func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error {
+	result := strings.Join(tlp.rows, "\n")
+	if len(tlp.rows) != rowsExpected {
+		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), rowsExpected, result, resultExpected)
+	}
+
+	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
+		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
+	}
+	if result != resultExpected {
+		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+	}
+
+	return nil
+}
+
+// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
+type BenchmarkLogMessageProcessor struct{}
+
+// AddRow implements LogMessageProcessor interface.
+func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field) {
+}
+
+// MustClose implements LogMessageProcessor interface.
+func (blp *BenchmarkLogMessageProcessor) MustClose() {
+}

app/vlinsert/insertutils/timestamp.go

@@ -0,0 +1,33 @@
+package insertutils
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+//
+// The value for the timeField is set to empty string after returning from the function,
+// so it could be ignored during data ingestion.
+//
+// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
+func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		if f.Value == "" || f.Value == "0" {
+			return time.Now().UnixNano(), nil
+		}
+		nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
+		if !ok {
+			return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
+		}
+		f.Value = ""
+		return nsecs, nil
+	}
+	return time.Now().UnixNano(), nil
+}

app/vlinsert/insertutils/timestamp_test.go

@@ -0,0 +1,75 @@
+package insertutils
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
+	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
+		t.Helper()
+
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if nsecs != nsecsExpected {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
+		}
+
+		for _, f := range fields {
+			if f.Name == timeField {
+				if f.Value != "" {
+					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
+				}
+			}
+		}
+	}
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20Z"},
+	}, 1718753840000000000)
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
+	}, 1718725040000000000)
+
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
+	}, 1718773640123000000)
+
+	f("time", []logstorage.Field{
+		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
+		{Name: "foo", Value: "bar"},
+	}, 1718773640123456789)
+}
+
+func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+
+		fields := []logstorage.Field{
+			{Name: "time", Value: s},
+		}
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if nsecs != 0 {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
+		}
+	}
+
+	f("foobar")
+
+	// no Z at the end
+	f("2024-06-18T23:37:20")
+
+	// incomplete time
+	f("2024-06-18")
+	f("2024-06-18T23:37")
+}

app/vlinsert/internalinsert/internalinsert.go

@@ -1,88 +0,0 @@
-package internalinsert
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
-)
-
-// RequestHandler processes /internal/insert requests.
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-	version := r.FormValue("version")
-	if version != netinsert.ProtocolVersion {
-		httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("internalinsert", false)
-		irp := lmp.(insertutil.InsertRowProcessor)
-		err := parseData(irp, data)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
-		return
-	}
-
-	requestDuration.UpdateDuration(startTime)
-}
-
-func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
-	r := logstorage.GetInsertRow()
-	src := data
-	i := 0
-	for len(src) > 0 {
-		tail, err := r.UnmarshalInplace(src)
-		if err != nil {
-			return fmt.Errorf("cannot parse row #%d: %s", i, err)
-		}
-		src = tail
-		i++
-
-		irp.AddInsertRow(r)
-	}
-	logstorage.PutInsertRow(r)
-
-	return nil
-}
-
-var (
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
-
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/internal/insert"}`)
-)

app/vlinsert/journald/journald.go

@@ -1,377 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"slices"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
-const maxFieldNameLen = 64
-
-var (
-	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
-	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Comma-separated list of fields to ignore for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#dropping-fields")
-	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Field to use as a log timestamp for logs ingested via journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
-	journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
-	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include Journald fields with double underscore prefixes")
-)
-
-func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		tenantID, err := logstorage.ParseTenantID(*journaldTenantID)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse -journald.tenantID=%q for journald: %w", *journaldTenantID, err)
-		}
-		cp.TenantID = tenantID
-	}
-
-	if !cp.IsTimeFieldSet {
-		cp.TimeFields = []string{*journaldTimeField}
-	}
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = getStreamFields()
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *journaldIgnoreFields
-	}
-	cp.MsgFields = []string{"MESSAGE"}
-	return cp, nil
-}
-
-func getStreamFields() []string {
-	if len(*journaldStreamFields) > 0 {
-		return *journaldStreamFields
-	}
-	return defaultStreamFields
-}
-
-var defaultStreamFields = []string{
-	"_MACHINE_ID",
-	"_HOSTNAME",
-	"_SYSTEMD_UNIT",
-}
-
-// RequestHandler processes Journald Export insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/journald/upload":
-		if r.Header.Get("Content-Type") != "application/vnd.fdo.journal" {
-			httpserver.Errorf(w, r, "only application/vnd.fdo.journal encoding is supported for Journald")
-			return true
-		}
-		handleJournald(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-// handleJournald parses Journal binary entries
-func handleJournald(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsTotal.Inc()
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-
-	if err := insertutil.CanWriteData(); err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
-	if err != nil {
-		errorsTotal.Inc()
-		logger.Errorf("cannot decode journald request: %s", err)
-		return
-	}
-
-	lmp := cp.NewLogMessageProcessor("journald", true)
-	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, lmp, cp)
-	protoparserutil.PutUncompressedReader(reader)
-	lmp.MustClose()
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
-		return
-	}
-
-	// systemd starting release v258 will support compression, which starts working after negotiation: it expects supported compression
-	// algorithms list in Accept-Encoding response header in a format "<algorithm_1>[:<priority_1>][;<algorithm_2>:<priority_2>]"
-	// See https://github.com/systemd/systemd/pull/34822
-	w.Header().Set("Accept-Encoding", "zstd")
-
-	// update requestDuration only for successfully parsed requests
-	// There is no need in updating requestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
-	errorsTotal     = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
-)
-
-func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lr := insertutil.NewLineReader("journald", wcr)
-
-	for {
-		err := readJournaldLogEntry(streamName, lr, lmp, cp)
-		wcr.DecConcurrency()
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				return nil
-			}
-			return fmt.Errorf("%s: %w", streamName, err)
-		}
-	}
-}
-
-type fieldsBuf struct {
-	fields []logstorage.Field
-
-	buf   []byte
-	name  []byte
-	value []byte
-}
-
-func (fb *fieldsBuf) reset() {
-	fb.fields = fb.fields[:0]
-	fb.buf = fb.buf[:0]
-	fb.name = fb.name[:0]
-	fb.value = fb.value[:0]
-}
-
-func (fb *fieldsBuf) addField(name, value string) {
-	bufLen := len(fb.buf)
-	fb.buf = append(fb.buf, name...)
-	nameCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
-
-	bufLen = len(fb.buf)
-	fb.buf = append(fb.buf, value...)
-	valueCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
-
-	fb.fields = append(fb.fields, logstorage.Field{
-		Name:  nameCopy,
-		Value: valueCopy,
-	})
-}
-
-func (fb *fieldsBuf) appendNextLineToValue(lr *insertutil.LineReader) error {
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return err
-		}
-		return fmt.Errorf("unexpected end of stream")
-	}
-	fb.value = append(fb.value, lr.Line...)
-	fb.value = append(fb.value, '\n')
-	return nil
-}
-
-func getFieldsBuf() *fieldsBuf {
-	fb := fieldsBufPool.Get()
-	if fb == nil {
-		return &fieldsBuf{}
-	}
-	return fb.(*fieldsBuf)
-}
-
-func putFieldsBuf(fb *fieldsBuf) {
-	fb.reset()
-	fieldsBufPool.Put(fb)
-}
-
-var fieldsBufPool sync.Pool
-
-// readJournaldLogEntry reads a single log entry in Journald format.
-//
-// See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
-	var ts int64
-	var name, value string
-
-	fb := getFieldsBuf()
-	defer putFieldsBuf(fb)
-
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return fmt.Errorf("cannot read the first field: %w", err)
-		}
-		return io.EOF
-	}
-
-	for {
-		line := lr.Line
-		if len(line) == 0 {
-			// The end of a single log entry. Write it to the storage
-			if len(fb.fields) > 0 {
-				if ts == 0 {
-					ts = time.Now().UnixNano()
-				}
-				lmp.AddRow(ts, fb.fields, nil)
-			}
-			return nil
-		}
-
-		// line could be either "key=value" or "key"
-		// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-		if n := bytes.IndexByte(line, '='); n >= 0 {
-			// line = "key=value"
-			fb.name = append(fb.name[:0], line[:n]...)
-			name = bytesutil.ToUnsafeString(fb.name)
-
-			fb.value = append(fb.value[:0], line[n+1:]...)
-			value = bytesutil.ToUnsafeString(fb.value)
-		} else {
-			// line = "key"
-			// Parse the binary-encoded value from the next line according to "key\n<little_endian_size_64>value\n" format
-			fb.name = append(fb.name[:0], line...)
-			name = bytesutil.ToUnsafeString(fb.name)
-
-			fb.value = fb.value[:0]
-			for len(fb.value) < 8 {
-				if err := fb.appendNextLineToValue(lr); err != nil {
-					return fmt.Errorf("cannot read value size: %w", err)
-				}
-			}
-			size := binary.LittleEndian.Uint64(fb.value[:8])
-
-			// Read the value until its lenth exceeds the given size - the last char in the read value will always be '\n'
-			// because it is appended by appendNextLineToValue().
-			for uint64(len(fb.value[8:])) <= size {
-				if err := fb.appendNextLineToValue(lr); err != nil {
-					return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
-				}
-			}
-			value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
-			if uint64(len(value)) != size {
-				return fmt.Errorf("unexpected %q value size; got %d bytes; want %d bytes; value: %q", fb.name, len(value), size, value)
-			}
-		}
-
-		if !lr.NextLine() {
-			if err := lr.Err(); err != nil {
-				return fmt.Errorf("cannot read the next log field: %w", err)
-			}
-
-			// add the last log field below before the return
-		}
-
-		if len(name) > maxFieldNameLen {
-			logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
-			continue
-		}
-		if !isValidFieldName(name) {
-			logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
-			continue
-		}
-
-		if slices.Contains(cp.TimeFields, name) {
-			t, err := strconv.ParseInt(value, 10, 64)
-			if err != nil {
-				logger.Errorf("%s: cannot parse timestamp from the field %q: %w; using the current timestamp", streamName, name, err)
-				ts = 0
-			} else {
-				// Convert journald microsecond timestamp to nanoseconds
-				ts = t * 1e3
-			}
-			continue
-		}
-
-		if slices.Contains(cp.MsgFields, name) {
-			name = "_msg"
-		}
-
-		if name == "PRIORITY" {
-			priority := journaldPriorityToLevel(value)
-			fb.addField("level", priority)
-		}
-
-		if !strings.HasPrefix(name, "__") || *journaldIncludeEntryMetadata {
-			fb.addField(name, value)
-		}
-	}
-}
-
-func journaldPriorityToLevel(priority string) string {
-	// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
-	// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
-	switch priority {
-	case "0":
-		return "emerg"
-	case "1":
-		return "alert"
-	case "2":
-		return "critical"
-	case "3":
-		return "error"
-	case "4":
-		return "warning"
-	case "5":
-		return "notice"
-	case "6":
-		return "info"
-	case "7":
-		return "debug"
-	default:
-		return priority
-	}
-}
-
-func isValidFieldName(s string) bool {
-	if len(s) == 0 {
-		return false
-	}
-	c := s[0]
-	if !(c >= 'A' && c <= 'Z' || c == '_') {
-		return false
-	}
-
-	for i := 1; i < len(s); i++ {
-		c := s[i]
-		if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
-			return false
-		}
-	}
-	return true
-}

app/vlinsert/journald/journald_test.go

@@ -1,166 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"net/http"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestIsValidFieldName(t *testing.T) {
-	f := func(name string, resultExpected bool) {
-		t.Helper()
-
-		result := isValidFieldName(name)
-		if result != resultExpected {
-			t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
-		}
-	}
-
-	f("", false)
-	f("a", false)
-	f("1", false)
-	f("_", true)
-	f("X", true)
-	f("Xa", false)
-	f("X_343", true)
-	f("X_0123456789_AZ", true)
-	f("SDDFD sdf", false)
-}
-
-func TestGetCommonParams_TimeField(t *testing.T) {
-	f := func(timeFieldHeader, expectedTimeField string) {
-		t.Helper()
-
-		req, err := http.NewRequest("POST", "/insert/journald/upload", nil)
-		if err != nil {
-			t.Fatalf("unexpected error creating request: %s", err)
-		}
-
-		if timeFieldHeader != "" {
-			req.Header.Set("VL-Time-Field", timeFieldHeader)
-		}
-
-		cp, err := getCommonParams(req)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if len(cp.TimeFields) != 1 || cp.TimeFields[0] != expectedTimeField {
-			t.Fatalf("unexpected TimeFields; got %v; want [%s]", cp.TimeFields, expectedTimeField)
-		}
-	}
-
-	// Test default behavior - when no custom time field is specified, journald uses __REALTIME_TIMESTAMP
-	f("", "__REALTIME_TIMESTAMP")
-
-	// Test custom time field - when a custom time field is specified via HTTP header, it's respected
-	f("custom_time", "custom_time")
-}
-
-func TestPushJournald_Success(t *testing.T) {
-	f := func(src string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		cp, err := getCommonParams(r)
-		if err != nil {
-			t.Fatalf("cannot create commonParams: %s", err)
-		}
-
-		buf := bytes.NewBufferString(src)
-		if err := processStreamInternal("test", buf, tlp, cp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Single event
-	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n",
-		[]int64{91723819283000},
-		"{\"_msg\":\"Test message\"}",
-	)
-
-	// Multiple events
-	f("__REALTIME_TIMESTAMP=91723819283\nPRIORITY=3\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
-		[]int64{91723819283000, 91723819284000},
-		"{\"level\":\"error\",\"PRIORITY\":\"3\",\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
-	)
-
-	// Parse binary data
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\nE=JobStateChanged\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
-		[]int64{1729698775704404000},
-		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
-	)
-
-	// Parse binary data with trailing newline
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x14\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"foo\nbar\n\n\nasda\nasda\n","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"123456789\n","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"1234567890","_PID":"2763"}`,
-	)
-
-	// Empty field name must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
-	f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
-
-	// field name starting with number must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
-
-	// field name exceeding 64 bytes limit must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n", nil, "")
-
-	// field name with invalid chars must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n", nil, "")
-}
-
-func TestPushJournald_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		cp, err := getCommonParams(r)
-		if err != nil {
-			t.Fatalf("cannot create commonParams: %s", err)
-		}
-
-		buf := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", buf, tlp, cp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// too short binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasd")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00_PID=2763\n\n")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890_PID=2763\n\n")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n_PID=2763\n\n")
-
-	// too long binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")
-}

app/vlinsert/journald/journald_timing_test.go

@@ -1,82 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func BenchmarkIsValidFieldName(b *testing.B) {
-	b.ReportAllocs()
-	b.SetBytes(int64(len(benchmarkFields)))
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			for _, field := range benchmarkFields {
-				if !isValidFieldName(field) {
-					panic(fmt.Errorf("cannot validate field %q", field))
-				}
-			}
-		}
-	})
-}
-
-var benchmarkFields = strings.Split(
-	"E,_BOOT_ID,_UID,_GID,_MACHINE_ID,_HOSTNAME,_RUNTIME_SCOPE,_TRANSPORT,_CAP_EFFECTIVE,_SYSTEMD_CGROUP,_SYSTEMD_UNIT,"+
-		"_SYSTEMD_SLICE,CODE_FILE,CODE_LINE,CODE_FUNC,SYSLOG_IDENTIFIER,_COMM,_EXE,_CMDLINE,MESSAGE,_PID,_SOURCE_REALTIME_TIMESTAMP,_REALTIME_TIMESTAMP",
-	",")
-
-func BenchmarkPushJournaldPerformance(b *testing.B) {
-	cp := &insertutil.CommonParams{
-		TimeFields: []string{"__REALTIME_TIMESTAMP"},
-		MsgFields:  []string{"MESSAGE"},
-	}
-	const dataChunkSize = 1024 * 1024
-
-	data := generateJournaldData(dataChunkSize)
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		blp := &insertutil.BenchmarkLogMessageProcessor{}
-		for pb.Next() {
-			r.Reset(data)
-			if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func generateJournaldData(size int) []byte {
-	var buf []byte
-	timestamp := time.Now().UnixMicro()
-	binaryMsg := []byte("binary message data for performance test")
-	var sizeBuf [8]byte
-
-	for len(buf) < size {
-		timestamp++
-
-		var entry string
-		// Generate a mix of simple and binary messages
-		if timestamp%10 == 0 {
-			// Generate binary message
-			binary.LittleEndian.PutUint64(sizeBuf[:], uint64(len(binaryMsg)))
-			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE\n%s%s\n\n",
-				timestamp,
-				sizeBuf[:],
-				binaryMsg,
-			)
-		} else {
-			// Generate simple message
-			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE=Performance test message %d\n\n", timestamp, timestamp)
-		}
-		buf = append(buf, entry...)
-	}
-	return buf
-}

app/vlinsert/jsonline/jsonline.go

@@ -1,16 +1,20 @@
 package jsonline
 
 import (
+	"bufio"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -27,97 +31,105 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 
 	requestsTotal.Inc()
 
-	cp, err := insertutil.GetCommonParams(r)
+	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	if err := insertutil.CanWriteData(); err != nil {
+	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
 
-	encoding := r.Header.Get("Content-Encoding")
-	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
-	if err != nil {
-		logger.Errorf("cannot decode jsonline request: %s", err)
-		return
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			logger.Errorf("cannot read gzipped jsonline request: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
 	}
-	defer protoparserutil.PutUncompressedReader(reader)
 
-	lmp := cp.NewLogMessageProcessor("jsonline", true)
-	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, cp.TimeFields, cp.MsgFields, lmp)
+	lmp := cp.NewLogMessageProcessor()
+	err = processStreamInternal(reader, cp.TimeField, cp.MsgField, lmp)
 	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot process jsonline request; error: %s", err)
-		return
-	}
 
-	requestDuration.UpdateDuration(startTime)
+	if err != nil {
+		logger.Errorf("jsonline: %s", err)
+	} else {
+		// update requestDuration only for successfully parsed requests.
+		// There is no need in updating requestDuration for request errors,
+		// since their timings are usually much smaller than the timing for successful request parsing.
+		requestDuration.UpdateDuration(startTime)
+	}
 }
 
-func processStreamInternal(streamName string, r io.Reader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) error {
+func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertutils.LogMessageProcessor) error {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
-	lr := insertutil.NewLineReader(streamName, wcr)
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
 
 	n := 0
-	errors := 0
-	var lastError error
 	for {
-		ok, err := readLine(lr, timeFields, msgFields, lmp)
+		ok, err := readLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil {
-			lastError = err
-			errors++
-			logger.Warnf("jsonline: cannot read line #%d in /jsonline request: %s", n, err)
+			errorsTotal.Inc()
+			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
 		}
 		if !ok {
-			break
+			return nil
 		}
 		n++
+		rowsIngestedTotal.Inc()
 	}
-	errorsTotal.Add(errors)
-
-	if errors > 0 && n == errors {
-		// Return an error if no logs were processed and there were errors
-		return lastError
-	}
-
-	return nil
 }
 
-func readLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
+func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
 		}
-		line = lr.Line
+		line = sc.Bytes()
 	}
 
 	p := logstorage.GetJSONParser()
-	defer logstorage.PutJSONParser(p)
-
 	if err := p.ParseLogMessage(line); err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
+		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
-	ts, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
+	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
 	if err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
+		return false, fmt.Errorf("cannot get timestamp: %w", err)
 	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
+	logstorage.PutJSONParser(p)
 
 	return true, nil
 }
 
+var lineBufferPool bytesutil.ByteBufferPool
+
 var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+
 	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
 	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
 
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
+	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
 )

app/vlinsert/jsonline/jsonline_test.go

@@ -2,96 +2,54 @@ package jsonline
 
 import (
 	"bytes"
-	"strings"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestProcessStreamInternalSuccess(t *testing.T) {
-	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
+func TestProcessStreamInternal_Success(t *testing.T) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		timeFields := []string{timeField}
-		msgFields := []string{msgField}
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, timeFields, msgFields, tlp); err != nil {
+		if err := processStreamInternal(r, timeField, msgField, tlp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}
 
 	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06 04:48:13.735Z","x":"y"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// Non-existing msgField
-	data = `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-`
-	timeField = "@timestamp"
-	msgField = "foobar"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
-{"message":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// invalid lines among valid lines
-	data = `
-dsfodmasd
-
-{"time":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-invalid line
-{"time":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-asbsdf
-
-`
-	timeField = "time"
-	msgField = "message"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 
-func TestProcessStreamInternalFailure(t *testing.T) {
+func TestProcessStreamInternal_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := strings.NewReader(data)
-		if err := processStreamInternal("test", r, []string{"time"}, nil, tlp); err == nil {
-			t.Fatalf("expected error, got nil")
-		}
-
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal(r, "time", "", tlp); err == nil {
+			t.Fatalf("expecting non-nil error")
 		}
 	}
 
 	// invalid json
 	f("foobar")
 
-	f(`foo
-bar`)
-
-	f(`
-foo
-
-`)
-
 	// invalid timestamp field
 	f(`{"time":"foobar"}`)
 }

app/vlinsert/loki/loki.go

@@ -1,25 +1,19 @@
 package loki
 
 import (
-	"flag"
-	"fmt"
 	"net/http"
-	"strconv"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whether to disable automatic parsing of JSON-encoded log fields inside Loki log message into distinct log fields")
-
 // RequestHandler processes Loki insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
-	case "/insert/loki/api/v1/push":
+	case "/api/v1/push":
 		handleInsert(r, w)
 		return true
-	case "/insert/loki/ready":
+	case "/ready":
 		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte("ready"))
@@ -41,17 +35,8 @@ func handleInsert(r *http.Request, w http.ResponseWriter) {
 	}
 }
 
-type commonParams struct {
-	cp *insertutil.CommonParams
-
-	// Whether to parse JSON inside plaintext log message.
-	//
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8486
-	parseMessage bool
-}
-
-func getCommonParams(r *http.Request) (*commonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
+func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
+	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		return nil, err
 	}
@@ -70,17 +55,5 @@ func getCommonParams(r *http.Request) (*commonParams, error) {
 
 	}
 
-	parseMessage := !*disableMessageParsing
-	if rv := httputil.GetRequestValue(r, "disable_message_parsing", "VL-Loki-Disable-Message-Parsing"); rv != "" {
-		bv, err := strconv.ParseBool(rv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse dusable_message_parsing=%q: %s", rv, err)
-		}
-		parseMessage = !bv
-	}
-
-	return &commonParams{
-		cp:           cp,
-		parseMessage: parseMessage,
-	}, nil
+	return cp, nil
 }

app/vlinsert/loki/loki_json.go

@@ -2,218 +2,177 @@ package loki
 
 import (
 	"fmt"
+	"io"
+	"math"
 	"net/http"
+	"strconv"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
 )
 
-var maxRequestSize = flagutil.NewBytes("loki.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single Loki request")
-
 var parserPool fastjson.ParserPool
 
 func handleJSON(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsJSONTotal.Inc()
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
 	}
-	if err := insertutil.CanWriteData(); err != nil {
+	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_json", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseJSONRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseJSONRequest(data, lmp)
+	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki json data: %s", err)
+		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
 		return
 	}
 
+	rowsIngestedJSONTotal.Add(n)
+
 	// update requestJSONDuration only for successfully parsed requests
 	// There is no need in updating requestJSONDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	requestJSONDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
 }
 
 var (
-	requestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
+	requestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
+	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
+	requestJSONDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
-func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
+func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
-
 	v, err := p.ParseBytes(data)
 	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
+		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
 	}
 
 	streamsV := v.Get("streams")
 	if streamsV == nil {
-		return fmt.Errorf("missing `streams` item in the parsed JSON")
+		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
 	}
 	streams, err := streamsV.Array()
 	if err != nil {
-		return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
-	}
-
-	fields := getFields()
-	defer putFields(fields)
-
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
+		return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
 	}
 
 	currentTimestamp := time.Now().UnixNano()
-
+	var commonFields []logstorage.Field
+	rowsIngested := 0
 	for _, stream := range streams {
 		// populate common labels from `stream` dict
-		fields.fields = fields.fields[:0]
+		commonFields = commonFields[:0]
 		labelsV := stream.Get("stream")
 		var labels *fastjson.Object
 		if labelsV != nil {
 			o, err := labelsV.Object()
 			if err != nil {
-				return fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
+				return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
 			}
 			labels = o
 		}
 		labels.Visit(func(k []byte, v *fastjson.Value) {
+			if err != nil {
+				return
+			}
 			vStr, errLocal := v.StringBytes()
 			if errLocal != nil {
 				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
 				return
 			}
-			fields.fields = append(fields.fields, logstorage.Field{
+			commonFields = append(commonFields, logstorage.Field{
 				Name:  bytesutil.ToUnsafeString(k),
 				Value: bytesutil.ToUnsafeString(vStr),
 			})
 		})
 		if err != nil {
-			return fmt.Errorf("error when parsing `stream` object: %w", err)
+			return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
 		}
 
 		// populate messages from `values` array
 		linesV := stream.Get("values")
 		if linesV == nil {
-			return fmt.Errorf("missing `values` item in the parsed `stream` object %q", stream)
+			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
 		}
 		lines, err := linesV.Array()
 		if err != nil {
-			return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
+			return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
 		}
 
-		commonFieldsLen := len(fields.fields)
+		fields := commonFields
 		for _, line := range lines {
-			fields.fields = fields.fields[:commonFieldsLen]
-
 			lineA, err := line.Array()
 			if err != nil {
-				return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
+				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
 			}
-			if len(lineA) < 2 || len(lineA) > 3 {
-				return fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2 or 3", line, len(lineA))
+			if len(lineA) != 2 {
+				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
 			}
 
 			// parse timestamp
 			timestamp, err := lineA[0].StringBytes()
 			if err != nil {
-				return fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
+				return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
 			}
 			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
 			if err != nil {
-				return fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
+				return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
 			}
 			if ts == 0 {
 				ts = currentTimestamp
 			}
 
-			// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-			if len(lineA) > 2 {
-				structuredMetadata, err := lineA[2].Object()
-				if err != nil {
-					return fmt.Errorf("unexpected structured metadata type for %q; want JSON object", lineA[2])
-				}
-
-				structuredMetadata.Visit(func(k []byte, v *fastjson.Value) {
-					vStr, errLocal := v.StringBytes()
-					if errLocal != nil {
-						err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-						return
-					}
-
-					fields.fields = append(fields.fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(vStr),
-					})
-				})
-				if err != nil {
-					return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
-				}
-			}
-
 			// parse log message
 			msg, err := lineA[1].StringBytes()
 			if err != nil {
-				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
+				return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
 			}
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, bytesutil.ToUnsafeString(msg))
 
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
-			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
+			fields = append(fields[:len(commonFields)], logstorage.Field{
+				Name:  "_msg",
+				Value: bytesutil.ToUnsafeString(msg),
+			})
+			lmp.AddRow(ts, fields)
 		}
+		rowsIngested += len(lines)
 	}
 
-	return nil
-}
-
-func addMsgField(dst []logstorage.Field, msgParser *logstorage.JSONParser, msg string) ([]logstorage.Field, bool) {
-	if msgParser == nil || len(msg) < 2 || msg[0] != '{' || msg[len(msg)-1] != '}' {
-		return append(dst, logstorage.Field{
-			Name:  "_msg",
-			Value: msg,
-		}), false
-	}
-	if msgParser != nil && len(msg) >= 2 && msg[0] == '{' && msg[len(msg)-1] == '}' {
-		if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msg)); err == nil {
-			return append(dst, msgParser.Fields...), true
-		}
-	}
-	return append(dst, logstorage.Field{
-		Name:  "_msg",
-		Value: msg,
-	}), false
+	return rowsIngested, nil
 }
 
 func parseLokiTimestamp(s string) (int64, error) {
@@ -221,5 +180,23 @@ func parseLokiTimestamp(s string) (int64, error) {
 		// Special case - an empty timestamp must be substituted with the current time by the caller.
 		return 0, nil
 	}
-	return insertutil.ParseUnixTimestamp(s)
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		// Fall back to parsing floating-point value
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return 0, err
+		}
+		if f > math.MaxInt64 {
+			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
+		}
+		if f < math.MinInt64 {
+			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
+		}
+		n = int64(f)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
+	}
+	return n, nil
 }

app/vlinsert/loki/loki_json_test.go

@@ -3,19 +3,20 @@ package loki
 import (
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestParseJSONRequest_Failure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err == nil {
+		tlp := &insertutils.TestLogMessageProcessor{}
+		n, err := parseJSONRequest([]byte(s), tlp)
+		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
+		if n != 0 {
+			t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
 		}
 	}
 	f(``)
@@ -44,31 +45,26 @@ func TestParseJSONRequest_Failure(t *testing.T) {
 	// Invalid length of `values` individual item
 	f(`{"streams":[{"values":[[]]}]}`)
 	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789","8123"]]}]}`)
+	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
 
 	// Invalid type for timestamp inside `values` individual item
 	f(`{"streams":[{"values":[[123,"456"]}]}`)
 
 	// Invalid type for log message
 	f(`{"streams":[{"values":[["123",1234]]}]}`)
-
-	// invalid structured metadata type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", ["metadata_1", "md_value"]]]}]}`)
-
-	// structured metadata with unexpected value type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": 1}]] }]}`)
 }
 
 func TestParseJSONRequest_Success(t *testing.T) {
 	f := func(s string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
+		n, err := parseJSONRequest([]byte(s), tlp)
+		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -89,9 +85,9 @@ func TestParseJSONRequest_Success(t *testing.T) {
 	"label2": "value2"
 },"values":[
 	["1577836800000000001", "foo bar"],
-	["1686026123.62", "abc"],
+	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1686026123620000000, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
 {"label1":"value1","label2":"value2","_msg":"abc"}
 {"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
@@ -120,50 +116,4 @@ func TestParseJSONRequest_Success(t *testing.T) {
 }`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
 {"foo":"bar","a":"b","_msg":"abc"}
 {"x":"y","_msg":"yx"}`)
-
-	// values with metadata
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"metadata_1":"md_value","_msg":"foo bar"}`)
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-}
-
-func TestParseJSONRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		if err := parseJSONRequest([]byte(s), tlp, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"111\",\"parent_id\":\"abc\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"111","parent_id":"abc"}`)
 }

app/vlinsert/loki/loki_json_timing_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func BenchmarkParseJSONRequest(b *testing.B) {
@@ -22,13 +22,14 @@ func BenchmarkParseJSONRequest(b *testing.B) {
 }
 
 func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseJSONRequest(data, blp, nil, false, true); err != nil {
+			_, err := parseJSONRequest(data, blp)
+			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}

app/vlinsert/loki/loki_protobuf.go

@@ -2,97 +2,101 @@ package loki
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
 )
 
 var (
+	bytesBufPool bytesutil.ByteBufferPool
 	pushReqsPool sync.Pool
 )
 
 func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsProtobufTotal.Inc()
+	wcr := writeconcurrencylimiter.GetReader(r.Body)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
 	}
-	if err := insertutil.CanWriteData(); err != nil {
+	if err := vlstorage.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	if encoding == "" {
-		// Loki protocol uses snappy compression by default.
-		// See https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-		encoding = "snappy"
-	}
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_protobuf", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseProtobufRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseProtobufRequest(data, lmp)
+	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki protobuf data: %s", err)
+		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
 		return
 	}
 
+	rowsIngestedProtobufTotal.Add(n)
+
 	// update requestProtobufDuration only for successfully parsed requests
 	// There is no need in updating requestProtobufDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	requestProtobufDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
 }
 
 var (
-	requestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	requestsProtobufTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
+	requestProtobufDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
-func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
+func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
+	bb := bytesBufPool.Get()
+	defer bytesBufPool.Put(bb)
+
+	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
+	if err != nil {
+		return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
+	}
+	bb.B = buf
+
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err := req.UnmarshalProtobuf(data)
+	err = req.UnmarshalProtobuf(bb.B)
 	if err != nil {
-		return fmt.Errorf("cannot parse request body: %w", err)
+		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
 	fields := getFields()
 	defer putFields(fields)
 
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
-	}
-
+	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
-
 	for i := range streams {
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
 		// Labels are same for all entries in the stream.
 		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
 		if err != nil {
-			return fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
+			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
 		commonFieldsLen := len(fields.fields)
 
@@ -108,25 +112,21 @@ func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFi
 				})
 			}
 
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, e.Line)
+			fields.fields = append(fields.fields, logstorage.Field{
+				Name:  "_msg",
+				Value: e.Line,
+			})
 
 			ts := e.Timestamp.UnixNano()
 			if ts == 0 {
 				ts = currentTimestamp
 			}
 
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
-			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
+			lmp.AddRow(ts, fields.fields)
 		}
+		rowsIngested += len(stream.Entries)
 	}
-	return nil
+	return rowsIngested, nil
 }
 
 func getFields() *fields {

app/vlinsert/loki/loki_protobuf_test.go

@@ -6,18 +6,16 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/golang/snappy"
 )
 
 type testLogMessageProcessor struct {
 	pr PushRequest
 }
 
-func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("unexpected non-nil streamFields: %v", streamFields))
-	}
+func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
 	msg := ""
 	for _, f := range fields {
 		if f.Name == "_msg" {
@@ -52,20 +50,23 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 		t.Helper()
 
 		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
+		n, err := parseJSONRequest([]byte(s), tlp)
+		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
+		if n != len(tlp.pr.Streams) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
 		}
 
 		data := tlp.pr.MarshalProtobuf(nil)
+		encodedData := snappy.Encode(nil, data)
 
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, nil, false, false); err != nil {
+		tlp2 := &insertutils.TestLogMessageProcessor{}
+		n, err = parseProtobufRequest(encodedData, tlp2)
+		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp2.Verify(n, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -88,7 +89,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
 {"label1":"value1","label2":"value2","_msg":"abc"}
 {"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
@@ -119,57 +120,6 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 {"x":"y","_msg":"yx"}`)
 }
 
-func TestParseProtobufRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
-		}
-
-		data := tlp.pr.MarshalProtobuf(nil)
-
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"432\",\"parent_id\":\"qwerty\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"432","parent_id":"qwerty"}`)
-}
-
 func TestParsePromLabels_Success(t *testing.T) {
 	f := func(s string) {
 		t.Helper()

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -6,7 +6,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
@@ -23,13 +25,14 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
 }
 
 func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseProtobufRequest(body, blp, nil, false, true); err != nil {
+			_, err := parseProtobufRequest(body, blp)
+			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}
@@ -76,5 +79,8 @@ func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
 		Streams: streams,
 	}
 
-	return pr.MarshalProtobuf(nil)
+	body := pr.MarshalProtobuf(nil)
+	encodedBody := snappy.Encode(nil, body)
+
+	return encodedBody
 }

app/vlinsert/loki/pb.go

@@ -17,7 +17,7 @@ var mp easyproto.MarshalerPool
 
 // PushRequest represents Loki PushRequest
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L14
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
 type PushRequest struct {
 	Streams []Stream
 
@@ -87,7 +87,7 @@ func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []Labe
 
 // Stream represents Loki stream.
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L23
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
 type Stream struct {
 	Labels  string
 	Entries []Entry
@@ -139,7 +139,7 @@ func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair,
 
 // Entry represents Loki entry.
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L38
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
 type Entry struct {
 	Timestamp          time.Time
 	Line               string
@@ -203,7 +203,7 @@ func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]Label
 
 // LabelPair represents Loki label pair.
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L33
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
 type LabelPair struct {
 	Name  string
 	Value string

app/vlinsert/main.go

@@ -1,25 +1,13 @@
 package vlinsert
 
 import (
-	"flag"
-	"fmt"
 	"net/http"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-)
-
-var (
-	disableInsert   = flag.Bool("insert.disable", false, "Whether to disable /insert/* HTTP endpoints")
-	disableInternal = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
 )
 
 // Init initializes vlinsert
@@ -34,55 +22,26 @@ func Stop() {
 
 // RequestHandler handles insert requests for VictoriaLogs
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
-
-	if strings.HasPrefix(path, "/insert/") {
-		if *disableInsert {
-			httpserver.Errorf(w, r, "requests to /insert/* are disabled with -insert.disable command-line flag")
-			return true
-		}
-
-		return insertHandler(w, r, path)
+	path := r.URL.Path
+	if !strings.HasPrefix(path, "/insert/") {
+		// Skip requests, which do not start with /insert/, since these aren't our requests.
+		return false
 	}
+	path = strings.TrimPrefix(path, "/insert")
+	path = strings.ReplaceAll(path, "//", "/")
 
-	if path == "/internal/insert" {
-		if *disableInternal || *disableInsert {
-			httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable or -insert.disable command-line flag")
-			return true
-		}
-		internalinsert.RequestHandler(w, r)
-		return true
-	}
-
-	return false
-}
-
-func insertHandler(w http.ResponseWriter, r *http.Request, path string) bool {
-	switch path {
-	case "/insert/jsonline":
+	if path == "/jsonline" {
 		jsonline.RequestHandler(w, r)
 		return true
-	case "/insert/ready":
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(200)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	}
 	switch {
-	// some clients may omit trailing slash at elasticsearch protocol.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
-	case strings.HasPrefix(path, "/insert/elasticsearch"):
+	case strings.HasPrefix(path, "/elasticsearch/"):
+		path = strings.TrimPrefix(path, "/elasticsearch")
 		return elasticsearch.RequestHandler(path, w, r)
-
-	case strings.HasPrefix(path, "/insert/loki/"):
+	case strings.HasPrefix(path, "/loki/"):
+		path = strings.TrimPrefix(path, "/loki")
 		return loki.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/opentelemetry/"):
-		return opentelemetry.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/journald/"):
-		return journald.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/datadog/"):
-		return datadog.RequestHandler(path, w, r)
+	default:
+		return false
 	}
-
-	return false
 }

app/vlinsert/opentelemetry/opentelemetry.go

@@ -1,153 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var maxRequestSize = flagutil.NewBytes("opentelemetry.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single OpenTelemetry request")
-
-// RequestHandler processes Opentelemetry insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	// use the same path as opentelemetry collector
-	// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
-	case "/insert/opentelemetry/v1/logs":
-		if r.Header.Get("Content-Type") == "application/json" {
-			httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-			return true
-		}
-		handleProtobuf(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf", false)
-		useDefaultStreamFields := len(cp.StreamFields) == 0
-		err := pushProtobufRequest(data, lmp, cp.MsgFields, useDefaultStreamFields)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read OpenTelemetry protocol data: %s", err)
-		return
-	}
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-
-	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-)
-
-func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {
-	var req pb.ExportLogsServiceRequest
-	if err := req.UnmarshalProtobuf(data); err != nil {
-		errorsTotal.Inc()
-		return fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
-	}
-
-	var commonFields []logstorage.Field
-	for _, rl := range req.ResourceLogs {
-		commonFields = commonFields[:0]
-		commonFields = appendKeyValues(commonFields, rl.Resource.Attributes, "")
-		commonFieldsLen := len(commonFields)
-		for _, sc := range rl.ScopeLogs {
-			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, msgFields, useDefaultStreamFields)
-		}
-	}
-
-	return nil
-}
-
-func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) []logstorage.Field {
-	fields := commonFields
-	for _, lr := range sc.LogRecords {
-		fields = fields[:len(commonFields)]
-		if lr.Body.KeyValueList != nil {
-			fields = appendKeyValues(fields, lr.Body.KeyValueList.Values, "")
-			logstorage.RenameField(fields[len(commonFields):], msgFields, "_msg")
-		} else {
-			fields = append(fields, logstorage.Field{
-				Name:  "_msg",
-				Value: lr.Body.FormatString(true),
-			})
-		}
-		fields = appendKeyValues(fields, lr.Attributes, "")
-		if len(lr.TraceID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "trace_id",
-				Value: lr.TraceID,
-			})
-		}
-		if len(lr.SpanID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "span_id",
-				Value: lr.SpanID,
-			})
-		}
-		fields = append(fields, logstorage.Field{
-			Name:  "severity",
-			Value: lr.FormatSeverity(),
-		})
-
-		var streamFields []logstorage.Field
-		if useDefaultStreamFields {
-			streamFields = commonFields
-		}
-		lmp.AddRow(lr.ExtractTimestampNano(), fields, streamFields)
-	}
-	return fields
-}
-
-func appendKeyValues(fields []logstorage.Field, kvs []*pb.KeyValue, parentField string) []logstorage.Field {
-	for _, attr := range kvs {
-		fieldName := attr.Key
-		if parentField != "" {
-			fieldName = parentField + "." + fieldName
-		}
-
-		if attr.Value.KeyValueList != nil {
-			fields = appendKeyValues(fields, attr.Value.KeyValueList.Values, fieldName)
-		} else {
-			fields = append(fields, logstorage.Field{
-				Name:  fieldName,
-				Value: attr.Value.FormatString(true),
-			})
-		}
-	}
-	return fields
-}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -1,208 +0,0 @@
-package opentelemetry
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func TestPushProtoOk(t *testing.T) {
-	f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-		lr := pb.ExportLogsServiceRequest{
-			ResourceLogs: src,
-		}
-
-		pData := lr.MarshalProtobuf(nil)
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := pushProtobufRequest(pData, tlp, nil, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// single line without resource attributes
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234},
-		`{"_msg":"log-line-message","severity":"Trace"}`,
-	)
-
-	// severities mapping
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 13, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 24, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1234, 1234},
-		`{"_msg":"log-line-message","severity":"Trace"}
-{"_msg":"log-line-message","severity":"Warn"}
-{"_msg":"log-line-message","severity":"Fatal4"}`,
-	)
-
-	// multi-line with resource attributes
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 25, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
-	)
-
-	// multi-scope with resource attributes and multi-line
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
-						{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
-					},
-				},
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
-						{TraceID: "1234", SpanID: "45", ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
-						{TraceID: "4bf92f3577b34da6a3ce929d0e0e4736", SpanID: "00f067aa0ba902b7", ObservedTimeUnixNano: 3333, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 2345, 2346, 2347, 2348, 3333},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Debug"}
-{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-1","trace_id":"1234","span_id":"45","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-2","trace_id":"4bf92f3577b34da6a3ce929d0e0e4736","span_id":"00f067aa0ba902b7","severity":"Unspecified"}`,
-	)
-
-	// nested fields
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{
-							TimeUnixNano: 1234,
-							Body:         pb.AnyValue{StringValue: ptrTo("nested fields")},
-							Attributes: []*pb.KeyValue{
-								{Key: "error", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-									{
-										Key:   "type",
-										Value: &pb.AnyValue{StringValue: ptrTo("document_parsing_exception")},
-									},
-									{
-										Key:   "reason",
-										Value: &pb.AnyValue{StringValue: ptrTo("failed to parse field [_msg] of type [text]")},
-									},
-									{
-										Key: "caused_by",
-										Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-											{
-												Key:   "type",
-												Value: &pb.AnyValue{StringValue: ptrTo("x_content_parse_exception")},
-											},
-											{
-												Key:   "reason",
-												Value: &pb.AnyValue{StringValue: ptrTo("unexpected end-of-input in VALUE_STRING")},
-											},
-											{
-												Key: "caused_by",
-												Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-													{
-														Key:   "type",
-														Value: &pb.AnyValue{StringValue: ptrTo("json_e_o_f_exception")},
-													},
-													{
-														Key:   "reason",
-														Value: &pb.AnyValue{StringValue: ptrTo("eof")},
-													},
-												}}},
-											},
-										}}},
-									},
-								}}}},
-							},
-						},
-					},
-				},
-			},
-		},
-	}, []int64{1234},
-		`{"_msg":"nested fields","error.type":"document_parsing_exception","error.reason":"failed to parse field [_msg] of type [text]",`+
-			`"error.caused_by.type":"x_content_parse_exception","error.caused_by.reason":"unexpected end-of-input in VALUE_STRING",`+
-			`"error.caused_by.caused_by.type":"json_e_o_f_exception","error.caused_by.caused_by.reason":"eof","severity":"Unspecified"}`)
-}
-
-func ptrTo[T any](s T) *T {
-	return &s
-}

app/vlinsert/opentelemetry/opentelemetry_timing_test.go

@@ -1,78 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, scopes := range []int{1, 2} {
-		for _, rows := range []int{100, 1000} {
-			for _, attributes := range []int{5, 10} {
-				b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, scopes, rows, attributes)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			if err := pushProtobufRequest(body, blp, nil, false); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
-	msg := "12345678910"
-
-	attrValues := []*pb.AnyValue{
-		{StringValue: ptrTo("string-attribute")},
-		{IntValue: ptrTo[int64](12345)},
-		{DoubleValue: ptrTo(3.14)},
-	}
-	attrs := make([]*pb.KeyValue, attributesCount)
-	for j := 0; j < attributesCount; j++ {
-		attrs[j] = &pb.KeyValue{
-			Key:   fmt.Sprintf("key-%d", j),
-			Value: attrValues[j%3],
-		}
-	}
-	entries := make([]pb.LogRecord, rowsCount)
-	for j := 0; j < rowsCount; j++ {
-		entries[j] = pb.LogRecord{
-			TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
-		}
-	}
-	scopes := make([]pb.ScopeLogs, scopesCount)
-
-	for j := 0; j < scopesCount; j++ {
-		scopes[j] = pb.ScopeLogs{
-			LogRecords: entries,
-		}
-	}
-
-	pr := pb.ExportLogsServiceRequest{
-		ResourceLogs: []pb.ResourceLogs{
-			{
-				Resource: pb.Resource{
-					Attributes: attrs,
-				},
-				ScopeLogs: scopes,
-			},
-		},
-	}
-
-	return pr.MarshalProtobuf(nil)
-}

app/vlinsert/syslog/syslog.go

@@ -3,20 +3,21 @@ package syslog
 import (
 	"bufio"
 	"crypto/tls"
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"net"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/klauspost/compress/gzip"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -24,7 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -34,30 +35,10 @@ var (
 	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
 		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
-	streamFieldsTCP = flagutil.NewArrayString("syslog.streamFields.tcp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-	streamFieldsUDP = flagutil.NewArrayString("syslog.streamFields.udp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-
-	ignoreFieldsTCP = flagutil.NewArrayString("syslog.ignoreFields.tcp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-	ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-
-	decolorizeFieldsTCP = flagutil.NewArrayString("syslog.decolorizeFields.tcp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-	decolorizeFieldsUDP = flagutil.NewArrayString("syslog.decolorizeFields.udp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-
-	extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-	extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-
-	tenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
-	tenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
+	syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
 	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
 		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
@@ -169,7 +150,7 @@ func runUDPListener(addr string, argIdx int) {
 		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
 	}
 
-	tenantIDStr := tenantIDUDP.GetOptionalArg(argIdx)
+	tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
 	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
 	if err != nil {
 		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
@@ -180,33 +161,9 @@ func runUDPListener(addr string, argIdx int) {
 
 	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
 
-	streamFieldsStr := streamFieldsUDP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.udp=%q for -syslog.listenAddr.udp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsUDP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	decolorizeFieldsStr := decolorizeFieldsUDP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.udp=%q for -syslog.listenAddr.udp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.udp=%q for -syslog.listenAddr.udp=%q: %s", extraFieldsStr, addr, err)
-	}
-
 	doneCh := make(chan struct{})
 	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
+		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
 		close(doneCh)
 	}()
 
@@ -236,7 +193,7 @@ func runTCPListener(addr string, argIdx int) {
 		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
 	}
 
-	tenantIDStr := tenantIDTCP.GetOptionalArg(argIdx)
+	tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
 	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
 	if err != nil {
 		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
@@ -247,33 +204,9 @@ func runTCPListener(addr string, argIdx int) {
 
 	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
 
-	streamFieldsStr := streamFieldsTCP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsTCP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	decolorizeFieldsStr := decolorizeFieldsTCP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", extraFieldsStr, addr, err)
-	}
-
 	doneCh := make(chan struct{})
 	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
+		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
 		close(doneCh)
 	}()
 
@@ -288,14 +221,14 @@ func runTCPListener(addr string, argIdx int) {
 
 func checkCompressMethod(compressMethod, addr, protocol string) {
 	switch compressMethod {
-	case "", "none", "zstd", "gzip", "deflate":
+	case "", "none", "gzip", "deflate":
 		return
 	default:
-		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
+		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
 	}
 }
 
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
 	gomaxprocs := cgroup.AvailableCPUs()
 	var wg sync.WaitGroup
 	localAddr := ln.LocalAddr()
@@ -303,7 +236,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
 			var bb bytesutil.ByteBuffer
 			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
 			for {
@@ -328,7 +261,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 				}
 				bb.B = bb.B[:n]
 				udpRequestsTotal.Inc()
-				if err := processStream("udp", bb.NewReader(), encoding, useLocalTimestamp, cp); err != nil {
+				if err := processStream(bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
 					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
 				}
 			}
@@ -337,7 +270,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 	wg.Wait()
 }
 
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
 	var cm ingestserver.ConnsMap
 	cm.Init("syslog")
 
@@ -367,8 +300,8 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us
 
 		wg.Add(1)
 		go func() {
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
-			if err := processStream("tcp", c, encoding, useLocalTimestamp, cp); err != nil {
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
 				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
 			}
 
@@ -383,29 +316,52 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us
 }
 
 // processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutil.CommonParams) error {
-	if err := insertutil.CanWriteData(); err != nil {
+func processStream(r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
+	if err := vlstorage.CanWriteData(); err != nil {
 		return err
 	}
 
-	lmp := cp.NewLogMessageProcessor("syslog_"+protocol, true)
-	err := processStreamInternal(r, encoding, useLocalTimestamp, lmp)
+	lmp := cp.NewLogMessageProcessor()
+	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
 	lmp.MustClose()
 
 	return err
 }
 
-func processStreamInternal(r io.Reader, encoding string, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return fmt.Errorf("cannot decode syslog data: %w", err)
+func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	switch compressMethod {
+	case "", "none":
+	case "gzip":
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read gzipped data: %w", err)
+		}
+		r = zr
+	case "deflate":
+		zr, err := common.GetZlibReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read deflated data: %w", err)
+		}
+		r = zr
+	default:
+		logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
 	}
-	defer protoparserutil.PutUncompressedReader(reader)
 
-	return processUncompressedStream(reader, useLocalTimestamp, lmp)
+	err := processUncompressedStream(r, useLocalTimestamp, lmp)
+
+	switch compressMethod {
+	case "gzip":
+		zr := r.(*gzip.Reader)
+		common.PutGzipReader(zr)
+	case "deflate":
+		zr := r.(io.ReadCloser)
+		common.PutZlibReader(zr)
+	}
+
+	return err
 }
 
-func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
+func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
@@ -427,6 +383,7 @@ func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertut
 			return fmt.Errorf("cannot read line #%d: %s", n, err)
 		}
 		n++
+		rowsIngestedTotal.Inc()
 	}
 	return slr.Error()
 }
@@ -490,7 +447,7 @@ again:
 			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
 			return false
 		}
-		if maxMsgLen := insertutil.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
+		if maxMsgLen := insertutils.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
 			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
 			return false
 		}
@@ -542,7 +499,7 @@ func putSyslogLineReader(slr *syslogLineReader) {
 
 var syslogLineReaderPool sync.Pool
 
-func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
+func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
 	p := logstorage.GetSyslogParser(currentYear, timezone)
 	lineStr := bytesutil.ToUnsafeString(line)
 	p.Parse(lineStr)
@@ -551,57 +508,24 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	if useLocalTimestamp {
 		ts = time.Now().UnixNano()
 	} else {
-		nsecs, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
+		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
 		if err != nil {
 			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
 		}
 		ts = nsecs
 	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
+	logstorage.RenameField(p.Fields, "message", "_msg")
+	lmp.AddRow(ts, p.Fields)
 	logstorage.PutSyslogParser(p)
 
 	return nil
 }
 
-var timeFields = []string{"timestamp"}
-var msgFields = []string{"message"}
-
 var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
+
 	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
 
 	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
 	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
 )
-
-func parseFieldsList(s string) ([]string, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var a []string
-	err := json.Unmarshal([]byte(s), &a)
-	return a, err
-}
-
-func parseExtraFields(s string) ([]logstorage.Field, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var m map[string]string
-	if err := json.Unmarshal([]byte(s), &m); err != nil {
-		return nil, err
-	}
-	fields := make([]logstorage.Field, 0, len(m))
-	for k, v := range m {
-		fields = append(fields, logstorage.Field{
-			Name:  k,
-			Value: v,
-		})
-	}
-	sort.Slice(fields, func(i, j int) bool {
-		return fields[i].Name < fields[j].Name
-	})
-	return fields, nil
-}

app/vlinsert/syslog/syslog_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestSyslogLineReader_Success(t *testing.T) {
@@ -75,7 +75,7 @@ func TestSyslogLineReader_Failure(t *testing.T) {
 }
 
 func TestProcessStreamInternal_Success(t *testing.T) {
-	f := func(data string, currentYear int, timestampsExpected []int64, resultExpected string) {
+	f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
 		MustInit()
@@ -84,12 +84,12 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 		globalTimezone = time.UTC
 		globalCurrentYear.Store(int64(currentYear))
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
 		if err := processStreamInternal(r, "", false, tlp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -99,11 +99,12 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
 `
 	currentYear := 2023
+	rowsExpected := 3
 	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
-	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
-	f(data, currentYear, timestampsExpected, resultExpected)
+	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
 }
 
 func TestProcessStreamInternal_Failure(t *testing.T) {
@@ -113,7 +114,7 @@ func TestProcessStreamInternal_Failure(t *testing.T) {
 		MustInit()
 		defer MustStop()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
 		if err := processStreamInternal(r, "", false, tlp); err == nil {
 			t.Fatalf("expecting non-nil error")

app/vlogscli/Makefile

@@ -1,109 +0,0 @@
-# All these commands must run from repository root.
-
-vlogscli:
-	APP_NAME=vlogscli $(MAKE) app-local
-
-vlogscli-race:
-	APP_NAME=vlogscli RACE=-race $(MAKE) app-local
-
-vlogscli-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker
-
-vlogscli-pure-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-pure
-
-vlogscli-linux-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-amd64
-
-vlogscli-linux-arm-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm
-
-vlogscli-linux-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm64
-
-vlogscli-linux-ppc64le-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-ppc64le
-
-vlogscli-linux-386-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-386
-
-vlogscli-darwin-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-amd64
-
-vlogscli-darwin-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-arm64
-
-vlogscli-freebsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-freebsd-amd64
-
-vlogscli-openbsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-openbsd-amd64
-
-vlogscli-windows-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-windows-amd64
-
-package-vlogscli:
-	APP_NAME=vlogscli $(MAKE) package-via-docker
-
-package-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-pure
-
-package-vlogscli-amd64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-amd64
-
-package-vlogscli-arm:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm
-
-package-vlogscli-arm64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm64
-
-package-vlogscli-ppc64le:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-ppc64le
-
-package-vlogscli-386:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-386
-
-publish-vlogscli:
-	APP_NAME=vlogscli $(MAKE) publish-via-docker
-
-vlogscli-linux-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-ppc64le:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-s390x:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-loong64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-386:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-freebsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-openbsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-windows-amd64:
-	GOARCH=amd64 APP_NAME=vlogscli $(MAKE) app-local-windows-goarch
-
-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) app-local-pure
-
-run-vlogscli:
-	APP_NAME=vlogscli $(MAKE) run-via-docker

app/vlogscli/README.md

@@ -1,5 +0,0 @@
-# vlogscli
-
-Command-line utility for querying [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).

app/vlogscli/deployment/Dockerfile

@@ -1,6 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-ENTRYPOINT ["/vlogscli-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vlogscli-prod

app/vlogscli/json_prettifier.go

@@ -1,245 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-type outputMode int
-
-const (
-	outputModeJSONMultiline  = outputMode(0)
-	outputModeJSONSingleline = outputMode(1)
-	outputModeLogfmt         = outputMode(2)
-	outputModeCompact        = outputMode(3)
-)
-
-func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
-	switch outputMode {
-	case outputModeJSONMultiline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, true)
-		}
-	case outputModeJSONSingleline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, false)
-		}
-	case outputModeLogfmt:
-		return writeLogfmtObject
-	case outputModeCompact:
-		return writeCompactObject
-	default:
-		panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
-	}
-}
-
-type jsonPrettifier struct {
-	r         io.ReadCloser
-	formatter func(w io.Writer, fields []logstorage.Field) error
-
-	d *json.Decoder
-
-	pr *io.PipeReader
-	pw *io.PipeWriter
-	bw *bufio.Writer
-
-	wg sync.WaitGroup
-}
-
-func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
-	d := json.NewDecoder(r)
-	pr, pw := io.Pipe()
-	bw := bufio.NewWriter(pw)
-
-	formatter := getOutputFormatter(outputMode)
-
-	jp := &jsonPrettifier{
-		r:         r,
-		formatter: formatter,
-
-		d: d,
-
-		pr: pr,
-		pw: pw,
-		bw: bw,
-	}
-
-	jp.wg.Add(1)
-	go func() {
-		defer jp.wg.Done()
-		err := jp.prettifyJSONLines()
-		jp.closePipesWithError(err)
-	}()
-
-	return jp
-}
-
-func (jp *jsonPrettifier) closePipesWithError(err error) {
-	_ = jp.pr.CloseWithError(err)
-	_ = jp.pw.CloseWithError(err)
-}
-
-func (jp *jsonPrettifier) prettifyJSONLines() error {
-	for jp.d.More() {
-		fields, err := readNextJSONObject(jp.d)
-		if err != nil {
-			return err
-		}
-		sort.Slice(fields, func(i, j int) bool {
-			return fields[i].Name < fields[j].Name
-		})
-		if err := jp.formatter(jp.bw, fields); err != nil {
-			return err
-		}
-
-		// Flush bw after every output line in order to show results as soon as they appear.
-		if err := jp.bw.Flush(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (jp *jsonPrettifier) Close() error {
-	jp.closePipesWithError(io.ErrUnexpectedEOF)
-	err := jp.r.Close()
-	jp.wg.Wait()
-	return err
-}
-
-func (jp *jsonPrettifier) Read(p []byte) (int, error) {
-	return jp.pr.Read(p)
-}
-
-func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
-	t, err := d.Token()
-	if err != nil {
-		return nil, fmt.Errorf("cannot read '{': %w", err)
-	}
-	delim, ok := t.(json.Delim)
-	if !ok || delim.String() != "{" {
-		return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
-	}
-
-	var fields []logstorage.Field
-	for {
-		// Read object key
-		t, err := d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object key or closing brace: %w", err)
-		}
-		delim, ok := t.(json.Delim)
-		if ok {
-			if delim.String() == "}" {
-				return fields, nil
-			}
-			return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
-		}
-		key, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object key: %v; want string or '}'", t)
-		}
-
-		// read object value
-		t, err = d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object value: %w", err)
-		}
-		value, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object value: %v; want string", t)
-		}
-
-		fields = append(fields, logstorage.Field{
-			Name:  key,
-			Value: value,
-		})
-	}
-}
-
-func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
-	data := logstorage.MarshalFieldsToLogfmt(nil, fields)
-	_, err := fmt.Fprintf(w, "%s\n", data)
-	return err
-}
-
-func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
-	if len(fields) == 1 {
-		// Just write field value as is without name
-		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
-		return err
-	}
-	if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
-		// Write _time\tfieldValue as is
-		if fields[0].Name == "_time" {
-			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)
-			return err
-		}
-		_, err := fmt.Fprintf(w, "%s\t%s\n", fields[1].Value, fields[0].Value)
-		return err
-	}
-
-	// Fall back to logfmt
-	return writeLogfmtObject(w, fields)
-}
-
-func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
-	if len(fields) == 0 {
-		fmt.Fprintf(w, "{}\n")
-		return nil
-	}
-
-	fmt.Fprintf(w, "{")
-	writeNewlineIfNeeded(w, isMultiline)
-	if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
-		return err
-	}
-	for _, f := range fields[1:] {
-		fmt.Fprintf(w, ",")
-		writeNewlineIfNeeded(w, isMultiline)
-		if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
-			return err
-		}
-	}
-	writeNewlineIfNeeded(w, isMultiline)
-	fmt.Fprintf(w, "}\n")
-	return nil
-}
-
-func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
-	if isMultiline {
-		fmt.Fprintf(w, "\n")
-	}
-}
-
-func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
-	key := getJSONString(f.Name)
-	value := getJSONString(f.Value)
-	if isMultiline {
-		_, err := fmt.Fprintf(w, "  %s: %s", key, value)
-		return err
-	}
-	_, err := fmt.Fprintf(w, "%s:%s", key, value)
-	return err
-}
-
-func getJSONString(s string) string {
-	data, err := json.Marshal(s)
-	if err != nil {
-		panic(fmt.Errorf("unexpected error when marshaling string to JSON: %w", err))
-	}
-	return jsonHTMLReplacer.Replace(string(data))
-}
-
-var jsonHTMLReplacer = strings.NewReplacer(
-	`\u003c`, "\u003c",
-	`\u003e`, "\u003e",
-	`\u0026`, "\u0026",
-)

app/vlogscli/less_wrapper.go

@@ -1,123 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-
-	"github.com/mattn/go-isatty"
-)
-
-func isTerminal() bool {
-	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
-}
-
-func readWithLess(r io.Reader, disableColors, wrapLongLines bool) error {
-	if !isTerminal() {
-		// Just write everything to stdout if no terminal is available.
-		_, err := io.Copy(os.Stdout, r)
-		if err != nil && !isErrPipe(err) {
-			return fmt.Errorf("error when forwarding data to stdout: %w", err)
-		}
-		if err := os.Stdout.Sync(); err != nil {
-			return fmt.Errorf("cannot sync data to stdout: %w", err)
-		}
-		return nil
-	}
-
-	pr, pw, err := os.Pipe()
-	if err != nil {
-		return fmt.Errorf("cannot create pipe: %w", err)
-	}
-	defer func() {
-		_ = pr.Close()
-		_ = pw.Close()
-	}()
-
-	// Ignore Ctrl+C in the current process, so 'less' could handle it properly
-	cancel := ignoreSignals(os.Interrupt)
-	defer cancel()
-
-	// Start 'less' process
-	path, err := exec.LookPath("less")
-	if err != nil {
-		return fmt.Errorf("cannot find 'less' command: %w", err)
-	}
-	opts := []string{"less", "-F", "-X"}
-	if !disableColors {
-		opts = append(opts, "-R")
-	}
-	if !wrapLongLines {
-		opts = append(opts, "-S")
-	}
-	p, err := os.StartProcess(path, opts, &os.ProcAttr{
-		Env:   append(os.Environ(), "LESSCHARSET=utf-8"),
-		Files: []*os.File{pr, os.Stdout, os.Stderr},
-	})
-	if err != nil {
-		return fmt.Errorf("cannot start 'less' process: %w", err)
-	}
-
-	// Close pr after 'less' finishes in a parallel goroutine
-	// in order to unblock forwarding data to stopped 'less' below.
-	waitch := make(chan *os.ProcessState)
-	go func() {
-		// Wait for 'less' process to finish.
-		ps, err := p.Wait()
-		if err != nil {
-			fatalf("unexpected error when waiting for 'less' process: %w", err)
-		}
-		_ = pr.Close()
-		waitch <- ps
-	}()
-
-	// Forward data from r to 'less'
-	_, err = io.Copy(pw, r)
-	_ = pw.Sync()
-	_ = pw.Close()
-
-	// Wait until 'less' finished
-	ps := <-waitch
-
-	// Verify 'less' status.
-	if !ps.Success() {
-		return fmt.Errorf("'less' finished with unexpected code %d", ps.ExitCode())
-	}
-
-	if err != nil && !isErrPipe(err) {
-		return fmt.Errorf("error when forwarding data to 'less': %w", err)
-	}
-
-	return nil
-}
-
-func isErrPipe(err error) bool {
-	return errors.Is(err, syscall.EPIPE) || errors.Is(err, io.ErrClosedPipe)
-}
-
-func ignoreSignals(sigs ...os.Signal) func() {
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, sigs...)
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for {
-			_, ok := <-ch
-			if !ok {
-				return
-			}
-		}
-	}()
-	return func() {
-		signal.Stop(ch)
-		close(ch)
-		wg.Wait()
-	}
-}

app/vlogscli/main.go

@@ -1,457 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strconv"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/ergochat/readline"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
-		"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs . See also -tail.url")
-	tailURL = flag.String("tail.url", "", "URL for live tailing queries to VictoriaLogs; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing ."+
-		"The url is automatically detected from -datasource.url by replacing /query with /tail at the end if -tail.url is empty")
-	historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
-	header      = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
-	accountID   = flag.Int("accountID", 0, "Account ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-	projectID   = flag.Int("projectID", 0, "Project ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-)
-
-const (
-	firstLinePrompt = ";> "
-	nextLinePrompt  = ""
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.InitNoLogFlags()
-
-	hes, err := parseHeaders(*header)
-	if err != nil {
-		fatalf("cannot parse -header command-line flag: %s", err)
-	}
-	headers = hes
-
-	incompleteLine := ""
-	cfg := &readline.Config{
-		Prompt:                 firstLinePrompt,
-		DisableAutoSaveHistory: true,
-		Listener: func(line []rune, pos int, _ rune) ([]rune, int, bool) {
-			incompleteLine = string(line)
-			return line, pos, false
-		},
-	}
-	rl, err := readline.NewFromConfig(cfg)
-	if err != nil {
-		fatalf("cannot initialize readline: %s", err)
-	}
-
-	fmt.Fprintf(rl, "sending queries to -datasource.url=%s\n", *datasourceURL)
-	fmt.Fprintf(rl, `type ? and press enter to see available commands`+"\n")
-	runReadlineLoop(rl, &incompleteLine)
-
-	if err := rl.Close(); err != nil {
-		fatalf("cannot close readline: %s", err)
-	}
-
-}
-
-func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
-	historyLines, err := loadFromHistory(*historyFile)
-	if err != nil {
-		fatalf("cannot load query history: %s", err)
-	}
-	for _, line := range historyLines {
-		if err := rl.SaveToHistory(line); err != nil {
-			fatalf("cannot initialize query history: %s", err)
-		}
-	}
-
-	outputMode := outputModeJSONMultiline
-	disableColors := true
-	wrapLongLines := false
-	s := ""
-	for {
-		line, err := rl.ReadLine()
-		if err != nil {
-			switch err {
-			case io.EOF:
-				if s != "" {
-					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode, disableColors, wrapLongLines)
-				}
-				return
-			case readline.ErrInterrupt:
-				if s == "" && *incompleteLine == "" {
-					fmt.Fprintf(rl, "interrupted\n")
-					os.Exit(128 + int(syscall.SIGINT))
-				}
-				// Default value for Ctrl+C - clear the prompt and store the incompletely entered line into history
-				s += *incompleteLine
-				historyLines = pushToHistory(rl, historyLines, s)
-				s = ""
-				rl.SetPrompt(firstLinePrompt)
-				continue
-			default:
-				fatalf("unexpected error in readline: %s", err)
-			}
-		}
-
-		s += line
-		if s == "" {
-			// Skip empty lines
-			continue
-		}
-
-		if isQuitCommand(s) {
-			fmt.Fprintf(rl, "bye!\n")
-			_ = pushToHistory(rl, historyLines, s)
-			return
-		}
-		if isHelpCommand(s) {
-			printCommandsHelp(rl)
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\s` {
-			fmt.Fprintf(rl, "singleline json output mode\n")
-			outputMode = outputModeJSONSingleline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\m` {
-			fmt.Fprintf(rl, "multiline json output mode\n")
-			outputMode = outputModeJSONMultiline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\c` {
-			fmt.Fprintf(rl, "compact output mode\n")
-			outputMode = outputModeCompact
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\logfmt` {
-			fmt.Fprintf(rl, "logfmt output mode\n")
-			outputMode = outputModeLogfmt
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\wrap_long_lines` {
-			if wrapLongLines {
-				wrapLongLines = false
-				fmt.Fprintf(rl, "wrapping of long lines is disabled\n")
-			} else {
-				wrapLongLines = true
-				fmt.Fprintf(rl, "wrapping of long lines is enabled\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\disable_colors` {
-			if !disableColors {
-				disableColors = true
-				fmt.Fprintf(rl, `disabled colors in compact output mode; enter \enable_colors for enabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\enable_colors` {
-			if disableColors {
-				disableColors = false
-				fmt.Fprintf(rl, `enabled colors in compact output mode; type \disable_colors for disabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if line != "" && !strings.HasSuffix(line, ";") {
-			// Assume the query is incomplete and allow the user finishing the query on the next line
-			s += "\n"
-			rl.SetPrompt(nextLinePrompt)
-			continue
-		}
-
-		// Execute the query
-		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode, disableColors, wrapLongLines)
-		cancel()
-
-		historyLines = pushToHistory(rl, historyLines, s)
-		s = ""
-		rl.SetPrompt(firstLinePrompt)
-	}
-}
-
-func pushToHistory(rl *readline.Instance, historyLines []string, s string) []string {
-	s = strings.TrimSpace(s)
-	if len(historyLines) == 0 || historyLines[len(historyLines)-1] != s {
-		historyLines = append(historyLines, s)
-		if len(historyLines) > 500 {
-			historyLines = historyLines[len(historyLines)-500:]
-		}
-		if err := saveToHistory(*historyFile, historyLines); err != nil {
-			fatalf("cannot save query history: %s", err)
-		}
-	}
-	if err := rl.SaveToHistory(s); err != nil {
-		fatalf("cannot update query history: %s", err)
-	}
-	return historyLines
-}
-
-func loadFromHistory(filePath string) ([]string, error) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	linesQuoted := strings.Split(string(data), "\n")
-	lines := make([]string, 0, len(linesQuoted))
-	i := 0
-	for _, lineQuoted := range linesQuoted {
-		i++
-		if lineQuoted == "" {
-			continue
-		}
-		line, err := strconv.Unquote(lineQuoted)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse line #%d at %s: %w; line: [%s]", i, filePath, err, line)
-		}
-		lines = append(lines, line)
-	}
-	return lines, nil
-}
-
-func saveToHistory(filePath string, lines []string) error {
-	linesQuoted := make([]string, len(lines))
-	for i, line := range lines {
-		lineQuoted := strconv.Quote(line)
-		linesQuoted[i] = lineQuoted
-	}
-	data := strings.Join(linesQuoted, "\n")
-	return os.WriteFile(filePath, []byte(data), 0600)
-}
-
-func isQuitCommand(s string) bool {
-	switch s {
-	case `\q`, "q", "quit", "exit":
-		return true
-	default:
-		return false
-	}
-}
-
-func isHelpCommand(s string) bool {
-	switch s {
-	case `\h`, "h", "help", "?":
-		return true
-	default:
-		return false
-	}
-}
-
-func printCommandsHelp(w io.Writer) {
-	fmt.Fprintf(w, "%s", `Available commands:
-\q - quit
-\h - show this help
-\s - singleline json output mode
-\m - multiline json output mode
-\c - compact output mode
-\logfmt - logfmt output mode
-\wrap_long_lines - toggles wrapping long lines
-\enable_colors - enable ANSI colors in compact output mode
-\disable_colors - disable ANSI colors in compact output mode
-\tail <query> - live tail <query> results
-
-See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
-`)
-}
-
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, disableColors, wrapLongLines bool) {
-	if strings.HasPrefix(qStr, `\tail `) {
-		tailQuery(ctx, output, qStr, outputMode)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, *datasourceURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if err := readWithLess(respBody, disableColors, wrapLongLines); err != nil {
-		fmt.Fprintf(output, "error when reading query response: %s\n", err)
-		return
-	}
-}
-
-func tailQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
-	qStr = strings.TrimPrefix(qStr, `\tail `)
-	qURL, err := getTailURL()
-	if err != nil {
-		fmt.Fprintf(output, "%s\n", err)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, qURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if _, err := io.Copy(output, respBody); err != nil {
-		if !errors.Is(err, context.Canceled) && !isErrPipe(err) {
-			fmt.Fprintf(output, "error when live tailing query response: %s\n", err)
-		}
-		fmt.Fprintf(output, "\n")
-		return
-	}
-}
-
-func getTailURL() (string, error) {
-	if *tailURL != "" {
-		return *tailURL, nil
-	}
-
-	u, err := url.Parse(*datasourceURL)
-	if err != nil {
-		return "", fmt.Errorf("cannot parse -datasource.url=%q: %w", *datasourceURL, err)
-	}
-	if !strings.HasSuffix(u.Path, "/query") {
-		return "", fmt.Errorf("cannot find /query suffix in -datasource.url=%q", *datasourceURL)
-	}
-	u.Path = u.Path[:len(u.Path)-len("/query")] + "/tail"
-	return u.String(), nil
-}
-
-func getQueryResponse(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, qURL string) io.ReadCloser {
-	// Parse the query and convert it to canonical view.
-	qStr = strings.TrimSuffix(qStr, ";")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		fmt.Fprintf(output, "cannot parse query: %s\n", err)
-		return nil
-	}
-	qStr = q.String()
-	fmt.Fprintf(output, "executing [%s]...", qStr)
-
-	// Prepare HTTP request for qURL
-	args := make(url.Values)
-	args.Set("query", qStr)
-	data := strings.NewReader(args.Encode())
-
-	req, err := http.NewRequestWithContext(ctx, "POST", qURL, data)
-	if err != nil {
-		panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	for _, h := range headers {
-		req.Header.Set(h.Name, h.Value)
-	}
-	req.Header.Set("AccountID", strconv.Itoa(*accountID))
-	req.Header.Set("ProjectID", strconv.Itoa(*projectID))
-
-	// Execute HTTP request at qURL
-	startTime := time.Now()
-	resp, err := httpClient.Do(req)
-	queryDuration := time.Since(startTime)
-	fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
-	if err != nil {
-		if errors.Is(err, context.Canceled) {
-			fmt.Fprintf(output, "\n")
-		} else {
-			fmt.Fprintf(output, "cannot execute query: %s\n", err)
-		}
-		return nil
-	}
-
-	// Verify response code
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			body = []byte(fmt.Sprintf("cannot read response body: %s", err))
-		}
-		fmt.Fprintf(output, "unexpected status code: %d; response body:\n%s\n", resp.StatusCode, body)
-		return nil
-	}
-
-	// Prettify the response body
-	jp := newJSONPrettifier(resp.Body, outputMode)
-
-	return jp
-}
-
-var httpClient = &http.Client{}
-
-var headers []headerEntry
-
-type headerEntry struct {
-	Name  string
-	Value string
-}
-
-func parseHeaders(a []string) ([]headerEntry, error) {
-	hes := make([]headerEntry, len(a))
-	for i, s := range a {
-		a := strings.SplitN(s, ":", 2)
-		if len(a) != 2 {
-			return nil, fmt.Errorf("cannot parse header=%q; it must contain at least one ':'; for example, 'Cookie: foo'", s)
-		}
-		hes[i] = headerEntry{
-			Name:  strings.TrimSpace(a[0]),
-			Value: strings.TrimSpace(a[1]),
-		}
-	}
-	return hes, nil
-}
-
-func fatalf(format string, args ...any) {
-	fmt.Fprintf(os.Stderr, format+"\n", args...)
-	os.Exit(1)
-}
-
-func usage() {
-	const s = `
-vlogscli is a command-line tool for querying VictoriaLogs.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/querying/vlogscli/
-`
-	flagutil.Usage(s)
-}

app/vlogscli/multiarch/Dockerfile

@@ -1,11 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-ENTRYPOINT ["/vlogscli-prod"]
-ARG TARGETARCH
-COPY vlogscli-linux-${TARGETARCH}-prod ./vlogscli-prod

app/vlogsgenerator/main.go

@@ -18,20 +18,20 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 var (
 	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
 	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
 
-	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
-	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
+	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
+	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
 	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
 	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
 		"during data generation")
 	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
-	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constant values to generate per each log entry; "+
+	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
@@ -45,8 +45,6 @@ var (
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	i64FieldsPerLog = flag.Int("i64FieldsPerLog", 1, "The number of fields with int64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
@@ -177,10 +175,7 @@ func generateAndPushLogs(cfg *workerConfig, workerID int) {
 	sw := &statWriter{
 		w: pw,
 	}
-
-	// The 1MB write buffer increases data ingestion performance by reducing the number of send() syscalls
-	bw := bufio.NewWriterSize(sw, 1024*1024)
-
+	bw := bufio.NewWriter(sw)
 	doneCh := make(chan struct{})
 	go func() {
 		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
@@ -206,8 +201,6 @@ func generateAndPushLogs(cfg *workerConfig, workerID int) {
 	if err != nil {
 		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
 	}
-	defer resp.Body.Close()
-
 	if resp.StatusCode/100 != 2 {
 		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
 	}
@@ -237,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 	for i := 0; i < activeStreams; i++ {
 		ip := toIPv4(rand.Uint32())
 		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
+		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
 			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
 		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
 		for j := 0; j < *constFieldsPerLog; j++ {
@@ -261,9 +254,6 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 		for j := 0; j < *u64FieldsPerLog; j++ {
 			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
 		}
-		for j := 0; j < *i64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"i64_%d":"%d"`, j, int64(rand.Uint64()))
-		}
 		for j := 0; j < *floatFieldsPerLog; j++ {
 			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
 		}
@@ -311,7 +301,7 @@ type timeFlag struct {
 }
 
 func (tf *timeFlag) Set(s string) error {
-	msec, err := timeutil.ParseTimeMsec(s)
+	msec, err := promutils.ParseTimeMsec(s)
 	if err != nil {
 		return fmt.Errorf("cannot parse time from %q: %w", s, err)
 	}

app/vlselect/internalselect/internalselect.go

@@ -1,316 +0,0 @@
-package internalselect
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-)
-
-// RequestHandler processes requests to /internal/select/*
-func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-
-	path := r.URL.Path
-	rh := requestHandlers[path]
-	if rh == nil {
-		httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
-		return
-	}
-
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
-	if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		// The return is skipped intentionally in order to track the duration of failed queries.
-	}
-	metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
-}
-
-var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
-	"/internal/select/query":               processQueryRequest,
-	"/internal/select/field_names":         processFieldNamesRequest,
-	"/internal/select/field_values":        processFieldValuesRequest,
-	"/internal/select/stream_field_names":  processStreamFieldNamesRequest,
-	"/internal/select/stream_field_values": processStreamFieldValuesRequest,
-	"/internal/select/streams":             processStreamsRequest,
-	"/internal/select/stream_ids":          processStreamIDsRequest,
-}
-
-func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	var wLock sync.Mutex
-	var dataLenBuf []byte
-
-	sendBuf := func(bb *bytesutil.ByteBuffer) error {
-		if len(bb.B) == 0 {
-			return nil
-		}
-
-		data := bb.B
-		if !cp.DisableCompression {
-			bufLen := len(bb.B)
-			bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
-			data = bb.B[bufLen:]
-		}
-
-		wLock.Lock()
-		dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
-		_, err := w.Write(dataLenBuf)
-		if err == nil {
-			_, err = w.Write(data)
-		}
-		wLock.Unlock()
-
-		// Reset the sent buf
-		bb.Reset()
-
-		return err
-	}
-
-	var bufs atomicutil.Slice[bytesutil.ByteBuffer]
-
-	var errGlobalLock sync.Mutex
-	var errGlobal error
-
-	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
-		if errGlobal != nil {
-			return
-		}
-
-		bb := bufs.Get(workerID)
-
-		bb.B = db.Marshal(bb.B)
-
-		if len(bb.B) < 1024*1024 {
-			// Fast path - the bb is too small to be sent to the client yet.
-			return
-		}
-
-		// Slow path - the bb must be sent to the client.
-		if err := sendBuf(bb); err != nil {
-			errGlobalLock.Lock()
-			if errGlobal != nil {
-				errGlobal = err
-			}
-			errGlobalLock.Unlock()
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
-		return err
-	}
-	if errGlobal != nil {
-		return errGlobal
-	}
-
-	// Send the remaining data
-	for _, bb := range bufs.All() {
-		if err := sendBuf(bb); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streams, cp.DisableCompression)
-}
-
-func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
-}
-
-type commonParams struct {
-	TenantIDs []logstorage.TenantID
-	Query     *logstorage.Query
-
-	DisableCompression bool
-}
-
-func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
-	version := r.FormValue("version")
-	if version != expectedProtocolVersion {
-		return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
-	}
-
-	tenantIDsStr := r.FormValue("tenant_ids")
-	tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
-	}
-
-	timestamp, err := getInt64FromRequest(r, "timestamp")
-	if err != nil {
-		return nil, err
-	}
-
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
-	}
-
-	s := r.FormValue("disable_compression")
-	disableCompression, err := strconv.ParseBool(s)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
-	}
-
-	cp := &commonParams{
-		TenantIDs: tenantIDs,
-		Query:     q,
-
-		DisableCompression: disableCompression,
-	}
-	return cp, nil
-}
-
-func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
-	var b []byte
-	for i := range vhs {
-		b = vhs[i].Marshal(b)
-	}
-
-	if !disableCompression {
-		b = zstd.CompressLevel(nil, b, 1)
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	if _, err := w.Write(b); err != nil {
-		return fmt.Errorf("cannot send response to the client: %w", err)
-	}
-
-	return nil
-}
-
-func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
-	s := r.FormValue(argName)
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
-	}
-	return n, nil
-}

app/vlselect/logsql/buffered_writer.go

@@ -0,0 +1,47 @@
+package logsql
+
+import (
+	"bufio"
+	"io"
+	"sync"
+)
+
+func getBufferedWriter(w io.Writer) *bufferedWriter {
+	v := bufferedWriterPool.Get()
+	if v == nil {
+		return &bufferedWriter{
+			bw: bufio.NewWriter(w),
+		}
+	}
+	bw := v.(*bufferedWriter)
+	bw.bw.Reset(w)
+	return bw
+}
+
+func putBufferedWriter(bw *bufferedWriter) {
+	bw.reset()
+	bufferedWriterPool.Put(bw)
+}
+
+var bufferedWriterPool sync.Pool
+
+type bufferedWriter struct {
+	mu sync.Mutex
+	bw *bufio.Writer
+}
+
+func (bw *bufferedWriter) reset() {
+	// nothing to do
+}
+
+func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
+	bw.mu.Lock()
+	_, _ = bw.bw.Write(p)
+	bw.mu.Unlock()
+}
+
+func (bw *bufferedWriter) FlushIgnoreErrors() {
+	bw.mu.Lock()
+	_ = bw.bw.Flush()
+	bw.mu.Unlock()
+}

app/vlselect/logsql/facets_response.qtpl

@@ -1,49 +0,0 @@
-{% import (
-	"slices"
-) %}
-
-{% stripspace %}
-
-{% func FacetsResponse(m map[string][]facetEntry) %}
-{
-	{% code
-		sortedKeys := make([]string, 0, len(m))
-		for k := range m {
-			sortedKeys = append(sortedKeys, k)
-		}
-		slices.Sort(sortedKeys)
-	%}
-	"facets":[
-		{% if len(sortedKeys) > 0 %}
-			{%= facetsLine(m, sortedKeys[0]) %}
-			{% for _, k := range sortedKeys[1:] %}
-				,{%= facetsLine(m, k) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetsLine(m map[string][]facetEntry, k string) %}
-{
-	"field_name":{%q= k %},
-	"values":[
-		{% code	fes := m[k] %}
-		{% if len(fes) > 0 %}
-			{%= facetLine(fes[0]) %}
-			{% for _, fe := range fes[1:] %}
-				,{%= facetLine(fe) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetLine(fe facetEntry) %}
-{
-	"field_value":{%q= fe.value %},
-	"hits":{%s= fe.hits %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/facets_response.qtpl.go

@@ -1,178 +0,0 @@
-// Code generated by qtc from "facets_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-import (
-	"slices"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-func StreamFacetsResponse(qw422016 *qt422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:7
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/facets_response.qtpl:10
-	sortedKeys := make([]string, 0, len(m))
-	for k := range m {
-		sortedKeys = append(sortedKeys, k)
-	}
-	slices.Sort(sortedKeys)
-
-//line app/vlselect/logsql/facets_response.qtpl:15
-	qw422016.N().S(`"facets":[`)
-//line app/vlselect/logsql/facets_response.qtpl:17
-	if len(sortedKeys) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:18
-		streamfacetsLine(qw422016, m, sortedKeys[0])
-//line app/vlselect/logsql/facets_response.qtpl:19
-		for _, k := range sortedKeys[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:19
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:20
-			streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:21
-		}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func WriteFacetsResponse(qq422016 qtio422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	StreamFacetsResponse(qw422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func FacetsResponse(m map[string][]facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:25
-	WriteFacetsResponse(qb422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:27
-func streamfacetsLine(qw422016 *qt422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:27
-	qw422016.N().S(`{"field_name":`)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().Q(k)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().S(`,"values":[`)
-//line app/vlselect/logsql/facets_response.qtpl:31
-	fes := m[k]
-
-//line app/vlselect/logsql/facets_response.qtpl:32
-	if len(fes) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:33
-		streamfacetLine(qw422016, fes[0])
-//line app/vlselect/logsql/facets_response.qtpl:34
-		for _, fe := range fes[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:34
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:35
-			streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:36
-		}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func writefacetsLine(qq422016 qtio422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func facetsLine(m map[string][]facetEntry, k string) string {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:40
-	writefacetsLine(qb422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:42
-func streamfacetLine(qw422016 *qt422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:42
-	qw422016.N().S(`{"field_value":`)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().Q(fe.value)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().S(`,"hits":`)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(fe.hits)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func writefacetLine(qq422016 qtio422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func facetLine(fe facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:47
-	writefacetLine(qb422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:47
-}