lib/timerpool: remove check of Reset output value

This reverts commit ccf97a4143.

reason for revert: this change may break tests, which expect that ServesMetrics.GetMetric() fails
when the given metric doesn't exist in the output.

It is better to add 'TryGetMetric() (float64, bool)' function, which would return '(0, false)'
when the given metric doesn't exist, so the caller could decide what to do next.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9773

.github/copilot-instructions.md

@@ -0,0 +1,23 @@
+# Project Overview
+
+VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
+
+## Folder Structure
+
+- `/app`: Contains the compilable binaries.
+- `/lib`: Contains the golang reusable libraries
+- `/docs/victoriametrics`: Contains documentation for the project.
+- `/apptest/tests`: Contains integration tests.
+
+## Libraries and Frameworks
+
+- Backend: Golang, no framework. Use third-party libraries sparingly.
+- Frontend: React.
+
+## Code review guidelines
+
+Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
+Verify the entry is under the ## tip section and matches the structure and style of existing entries.
+Chore-only changes may be omitted from the changelog.
+
+

.github/pull_request_template.md

@@ -7,3 +7,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 The following checks are **mandatory**:
 
 - [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
+- [ ] My change adheres to [VictoriaMetrics development goals](https://docs.victoriametrics.com/victoriametrics/goals/).

.github/workflows/build.yml

@@ -7,16 +7,20 @@ on:
       - master
     paths:
       - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Dockerfile'
       - '**/Makefile'
+      - '!app/vmui/**'
+      - '.github/workflows/build.yml'
   pull_request:
     branches:
       - cluster
       - master
     paths:
       - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Dockerfile'
       - '**/Makefile'
+      - '!app/vmui/**'
+      - '.github/workflows/build.yml'
 
 permissions:
   contents: read
@@ -27,28 +31,48 @@ concurrency:
 
 jobs:
   build:
-    name: Build
+    name: ${{ matrix.os }}-${{ matrix.arch }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: linux
+            arch: 386
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+          - os: linux
+            arch: arm
+          - os: linux
+            arch: ppc64le
+          - os: darwin
+            arch: amd64
+          - os: darwin
+            arch: arm64
+          - os: freebsd
+            arch: amd64
+          - os: openbsd
+            arch: amd64
+          - os: windows
+            arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
           go-version: stable
-          cache: false
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
+      - name: Build victoria-metrics for ${{ matrix.os }}-${{ matrix.arch }}
+        run: make victoria-metrics-${{ matrix.os }}-${{ matrix.arch }}
 
-      - name: Run crossbuild
-        run: make crossbuild
+      - name: Build vmutils for ${{ matrix.os }}-${{ matrix.arch }}
+        run: make vmutils-${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/check-commit-signed.yml

@@ -0,0 +1,37 @@
+name: check-commit-signed
+
+on:
+  pull_request:
+
+jobs:
+  check-commit-signed:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0  # we need full history for commit verification
+
+      - name: Check commit signatures
+        run: |
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            echo "Not a PR event, skipping signature check"
+            exit 0
+          fi
+          
+          RANGE="${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
+          echo "Checking commits in PR range: $RANGE"
+          
+          if [ -z "$(git rev-list $RANGE)" ]; then
+            echo "No new commits in this PR, skipping signature check"
+            exit 0
+          fi
+          
+          unsigned=$(git log --pretty="%H %G?" $RANGE | grep -vE " (G|E)$" || true)
+          if [ -n "$unsigned" ]; then
+            echo "Found unsigned commits:"
+            echo "$unsigned"
+            exit 1
+          fi
+          
+          echo "All commits in PR are signed (G or E)"

.github/workflows/check-licenses.yml

@@ -19,7 +19,7 @@ jobs:
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: stable
           cache: false

.github/workflows/codeql-analysis-go.yml

@@ -29,11 +29,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Go
         id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           cache: false
           go-version: stable

.github/workflows/codeql-analysis-js-ts.yml

@@ -1,46 +0,0 @@
-name: 'CodeQL JS/TS'
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: javascript-typescript
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: 'language:js/ts'

.github/workflows/docs.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/main.yml

@@ -1,122 +0,0 @@
-name: main
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-all-
-
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
-
-  test:
-    name: test
-    needs: lint
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        scenario:
-          - 'test-full'
-          - 'test-full-386'
-          - 'test-pure'
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v5
-        with:
-          files: ./coverage.txt
-
-  integration-test:
-    name: integration-test
-    needs: [lint, test]
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: Run integration tests
-        run: make integration-test

.github/workflows/test.yml

@@ -0,0 +1,113 @@
+name: test
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - 'go.*'
+      - '.github/workflows/main.yml'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - 'go.*'
+      - '.github/workflows/main.yml'
+
+permissions:
+  contents: read
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
+          go-version: stable
+
+
+      - name: Cache golangci-lint
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/golangci-lint
+            ~/go/bin
+          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
+
+      - name: Run check-all
+        run: |
+          make check-all
+          git diff --exit-code
+
+  unit:
+    name: unit
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        scenario:
+          - 'test-full'
+          - 'test-full-386'
+          - 'test-pure'
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
+          go-version: stable
+
+      - name: Run tests
+        run: GOGC=10 make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.txt
+
+  integration:
+    name: integration
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
+          go-version: stable
+
+      - name: Run integration tests
+        run: make integration-test

.github/workflows/vmui.yml

@@ -0,0 +1,82 @@
+name: vmui
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - 'app/vmui/packages/vmui/**'
+      - '.github/workflows/vmui.yml'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - 'app/vmui/packages/vmui/**'
+      - '.github/workflows/vmui.yml'
+
+permissions:
+  contents: read
+  packages: read
+  pull-requests: read
+  checks: write
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  vmui-checks:
+    name: VMUI Checks (lint, test, typecheck)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24.x'
+
+      - name: Cache node-modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            app/vmui/packages/vmui/node_modules
+          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
+          restore-keys: vmui-artifacts-${{ runner.os }}-
+
+      - name: Run lint
+        id: lint
+        run: make vmui-lint
+        continue-on-error: true
+
+      - name: Run tests
+        id: test
+        run: make vmui-test
+        continue-on-error: true
+
+      - name: Run typecheck
+        id: typecheck
+        run: make vmui-typecheck
+        continue-on-error: true
+
+      - name: Annotate Code Linting Results
+        uses: ataylorme/eslint-annotate-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          report-json: app/vmui/packages/vmui/vmui-lint-report.json
+
+      - name: Check overall status
+        run: |
+          echo "Lint status: ${{ steps.lint.outcome }}"
+          echo "Test status: ${{ steps.test.outcome }}"
+          echo "Typecheck status: ${{ steps.typecheck.outcome }}"
+
+          if [[ "${{ steps.lint.outcome }}" == "failure" || "${{ steps.test.outcome }}" == "failure" || "${{ steps.typecheck.outcome }}" == "failure" ]]; then
+            echo "One or more checks failed"
+            exit 1
+          else
+            echo "All checks passed"
+          fi

.gitignore

@@ -12,6 +12,7 @@
 /victoria-logs-data
 /victoria-metrics-data
 /vmagent-remotewrite-data
+/vlagent-remotewritewrite
 /vmstorage-data
 /vmselect-cache
 /package/temp-deb-*

.golangci.yml

@@ -1,22 +1,29 @@
-run:
-  timeout: 2m
-
+version: "2"
 linters:
-  enable:
-  - revive
-
-issues:
-  exclude-rules:
-  - linters:
-    - staticcheck
-    text: "SA(4003|1019|5011):"
-  include:
-  - EXC0012
-  - EXC0014
-
-linters-settings:
-  errcheck:
-    exclude-functions:
-      - "fmt.Fprintf"
-      - "fmt.Fprint"
-      - "(net/http.ResponseWriter).Write"
+  settings:
+    errcheck:
+      exclude-functions:
+        - fmt.Fprintf
+        - fmt.Fprint
+        - (net/http.ResponseWriter).Write
+  exclusions:
+    generated: lax
+    presets:
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - linters:
+          - staticcheck
+        text: 'SA(4003|1019|5011):'
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

Makefile

@@ -11,9 +11,14 @@ ifeq ($(PKG_TAG),)
 PKG_TAG := $(BUILDINFO_TAG)
 endif
 
+EXTRA_DOCKER_TAG_SUFFIX ?=
+EXTRA_GO_BUILD_TAGS ?=
+
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
+GOLANGCI_LINT_VERSION := 2.4.0
+
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
@@ -22,11 +27,10 @@ include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
 include package/release/Makefile
+include benchmarks/Makefile
 
 all: \
 	victoria-metrics-prod \
-	victoria-logs-prod \
-	vlogscli-prod \
 	vmagent-prod \
 	vmalert-prod \
 	vmalert-tool-prod \
@@ -50,8 +54,6 @@ publish: \
 
 package: \
 	package-victoria-metrics \
-	package-victoria-logs \
-	package-vlogscli \
 	package-vmagent \
 	package-vmalert \
 	package-vmalert-tool \
@@ -168,9 +170,11 @@ vmutils-windows-amd64: \
 	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 crossbuild:
 	$(MAKE_PARALLEL) victoria-metrics-crossbuild vmutils-crossbuild
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 victoria-metrics-crossbuild: \
 	victoria-metrics-linux-386 \
 	victoria-metrics-linux-amd64 \
@@ -183,6 +187,7 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-openbsd-amd64 \
 	victoria-metrics-windows-amd64
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 vmutils-crossbuild: \
 	vmutils-linux-386 \
 	vmutils-linux-amd64 \
@@ -195,6 +200,31 @@ vmutils-crossbuild: \
 	vmutils-openbsd-amd64 \
 	vmutils-windows-amd64
 
+publish-final-images:
+	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=victoria-metrics $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmagent $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmalert $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmauth $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackup $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmrestore $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-from-rc && \
+	PKG_TAG=$(TAG) $(MAKE) publish-latest
+
 publish-latest:
 	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-latest && \
 	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-latest && \
@@ -210,10 +240,6 @@ publish-latest:
 	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
 	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
 
-publish-victoria-logs-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-logs $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG) APP_NAME=vlogscli $(MAKE) publish-via-docker-latest
-
 publish-release:
 	rm -rf bin/*
 	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
@@ -283,128 +309,6 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-metrics-windows-$(GOARCH)-prod.exe
 
-release-victoria-logs-bundle: \
-	release-victoria-logs \
-	release-vlogscli
-
-publish-victoria-logs-bundle: \
-	publish-victoria-logs \
-	publish-vlogscli
-
-release-victoria-logs:
-	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
-		release-victoria-logs-linux-amd64 \
-		release-victoria-logs-linux-arm \
-		release-victoria-logs-linux-arm64 \
-		release-victoria-logs-darwin-amd64 \
-		release-victoria-logs-darwin-arm64 \
-		release-victoria-logs-freebsd-amd64 \
-		release-victoria-logs-openbsd-amd64 \
-		release-victoria-logs-windows-amd64
-
-release-victoria-logs-linux-386:
-	GOOS=linux GOARCH=386 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-linux-amd64:
-	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-linux-arm:
-	GOOS=linux GOARCH=arm $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-linux-arm64:
-	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-darwin-arm64:
-	GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-freebsd-amd64:
-	GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-openbsd-amd64:
-	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-logs-windows-goarch
-
-release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-logs-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-logs-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf victoria-logs-$(GOOS)-$(GOARCH)-prod
-
-release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-logs-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-logs-windows-$(GOARCH)-prod.exe \
-			> victoria-logs-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		victoria-logs-windows-$(GOARCH)-prod.exe
-
-release-vlogscli:
-	$(MAKE_PARALLEL) release-vlogscli-linux-386 \
-		release-vlogscli-linux-amd64 \
-		release-vlogscli-linux-arm \
-		release-vlogscli-linux-arm64 \
-		release-vlogscli-darwin-amd64 \
-		release-vlogscli-darwin-arm64 \
-		release-vlogscli-freebsd-amd64 \
-		release-vlogscli-openbsd-amd64 \
-		release-vlogscli-windows-amd64
-
-release-vlogscli-linux-386:
-	GOOS=linux GOARCH=386 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-amd64:
-	GOOS=linux GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-arm:
-	GOOS=linux GOARCH=arm $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-linux-arm64:
-	GOOS=linux GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-darwin-arm64:
-	GOOS=darwin GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-freebsd-amd64:
-	GOOS=freebsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-openbsd-amd64:
-	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
-
-release-vlogscli-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vlogscli-windows-goarch
-
-release-vlogscli-goos-goarch: vlogscli-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vlogscli-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vlogscli-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf vlogscli-$(GOOS)-$(GOARCH)-prod
-
-release-vlogscli-windows-goarch: vlogscli-windows-$(GOARCH)-prod
-	cd bin && \
-		zip vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vlogscli-windows-$(GOARCH)-prod.exe \
-		&& sha256sum vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vlogscli-windows-$(GOARCH)-prod.exe \
-			> vlogscli-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vlogscli-windows-$(GOARCH)-prod.exe
-
 release-vmutils: \
 	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
@@ -545,7 +449,11 @@ test-full:
 test-full-386:
 	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
-integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore victoria-logs
+integration-test:
+	$(MAKE) apptest
+
+apptest:
+	$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
 	go test ./apptest/... -skip="^TestCluster.*"
 
 benchmark:
@@ -563,16 +471,16 @@ vendor-update:
 	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-goos-goarch:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-windows-goarch:
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
@@ -585,7 +493,7 @@ golangci-lint: install-golangci-lint
 	GOEXPERIMENT=synctest golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7
+	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

README.md

@@ -1,14 +1,14 @@
 # VictoriaMetrics
 
-![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)
+[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 ![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)
-![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)
-![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)
-![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)
-![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)
+[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
+[![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml)
+[![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
+[![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
 ![Slack](https://img.shields.io/badge/Join-4A154B?logo=slack&link=https%3A%2F%2Fslack.victoriametrics.com)
-![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)
-![Reddit](https://img.shields.io/reddit/subreddit-subscribers/VictoriaMetrics?style=flat&label=Join&labelColor=red&logoColor=white&logo=reddit&link=https%3A%2F%2Fwww.reddit.com%2Fr%2FVictoriaMetrics)
+[![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)](https://x.com/VictoriaMetrics/)
+[![Reddit](https://img.shields.io/reddit/subreddit-subscribers/VictoriaMetrics?style=flat&label=Join&labelColor=red&logoColor=white&logo=reddit&link=https%3A%2F%2Fwww.reddit.com%2Fr%2FVictoriaMetrics)](https://www.reddit.com/r/VictoriaMetrics/)
 
 <picture>
   <source srcset="docs/victoriametrics/logo_white.webp" media="(prefers-color-scheme: dark)">

SECURITY.md

@@ -4,12 +4,11 @@
 
 The following versions of VictoriaMetrics receive regular security fixes:
 
-| Version | Supported          |
-|---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| other releases  | :x:                |
+| Version                                                                        | Supported          |
+|--------------------------------------------------------------------------------|--------------------|
+| [Latest release](https://docs.victoriametrics.com/victoriametrics/changelog/)  | :white_check_mark: |
+| [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| other releases                                                                 | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 

app/victoria-logs/Makefile

@@ -1,113 +0,0 @@
-# All these commands must run from repository root.
-
-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) app-local
-
-victoria-logs-race:
-	APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
-
-victoria-logs-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker
-
-victoria-logs-pure-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
-
-victoria-logs-linux-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
-
-victoria-logs-linux-arm-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
-
-victoria-logs-linux-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
-
-victoria-logs-linux-ppc64le-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
-
-victoria-logs-linux-386-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
-
-victoria-logs-darwin-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
-
-victoria-logs-darwin-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
-
-victoria-logs-freebsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
-
-victoria-logs-openbsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
-
-victoria-logs-windows-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
-
-package-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker
-
-package-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
-
-package-victoria-logs-amd64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
-
-package-victoria-logs-arm:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
-
-package-victoria-logs-arm64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
-
-package-victoria-logs-ppc64le:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
-
-package-victoria-logs-386:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-386
-
-publish-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) publish-via-docker
-
-victoria-logs-linux-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-ppc64le:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-s390x:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-loong64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-386:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-freebsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-openbsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-windows-amd64:
-	GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
-
-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) app-local-pure
-
-run-victoria-logs:
-	mkdir -p victoria-logs-data
-	DOCKER_OPTS='-v $(shell pwd)/victoria-logs-data:/victoria-logs-data' \
-	APP_NAME=victoria-logs \
-	ARGS='' \
-	$(MAKE) run-via-docker

app/victoria-logs/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-EXPOSE 9428
-
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -1,113 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-)
-
-var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9428"}
-	}
-	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
-	startTime := time.Now()
-
-	vlstorage.Init()
-	vlselect.Init()
-
-	insertutil.SetLogRowsStorage(&vlstorage.Storage{})
-	vlinsert.Init()
-
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
-	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
-
-	pushmetrics.Init()
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
-
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
-
-	vlinsert.Stop()
-	vlselect.Stop()
-	vlstorage.Stop()
-
-	fs.MustStopDirRemover()
-
-	logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
-}
-
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
-			return false
-		}
-		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/'>https://docs.victoriametrics.com/victorialogs/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints:</br>")
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"select/vmui", "Web UI for VictoriaLogs"},
-			{"metrics", "available service metrics"},
-			{"flags", "command-line flags"},
-		})
-		return true
-	}
-	if vlinsert.RequestHandler(w, r) {
-		return true
-	}
-	if vlselect.RequestHandler(w, r) {
-		return true
-	}
-	if vlstorage.RequestHandler(w, r) {
-		return true
-	}
-	return false
-}
-
-func usage() {
-	const s = `
-victoria-logs is a log management and analytics service.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/
-`
-	flagutil.Usage(s)
-}

app/victoria-logs/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 9428
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG TARGETARCH
-COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/main.go

@@ -17,7 +17,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -125,8 +124,6 @@ func main() {
 	vmstorage.Stop()
 	vmselect.Stop()
 
-	fs.MustStopDirRemover()
-
 	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
 }
 
@@ -172,7 +169,7 @@ func usage() {
 	const s = `
 victoria-metrics is a time series database and monitoring solution.
 
-See the docs at https://docs.victoriametrics.com/
+See the docs at https://docs.victoriametrics.com/victoriametrics/
 `
 	flagutil.Usage(s)
 }

app/victoria-metrics/self_scraper.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
@@ -49,7 +49,7 @@ func selfScraper(scrapeInterval time.Duration) {
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
 	var mrs []storage.MetricRow
-	var labels []prompbmarshal.Label
+	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
 	f := func(currentTime time.Time, sendStaleMarkers bool) {
 		currentTimestamp := currentTime.UnixNano() / 1e6
@@ -57,7 +57,8 @@ func selfScraper(scrapeInterval time.Duration) {
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
-		rows.Unmarshal(s)
+		// VictoriaMetrics components don't expose metadata yet, only need to parse samples
+		rows.UnmarshalWithErrLogger(s, nil)
 		mrs = mrs[:0]
 		for i := range rows.Rows {
 			r := &rows.Rows[i]
@@ -104,11 +105,11 @@ func selfScraper(scrapeInterval time.Duration) {
 	}
 }
 
-func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label {
+func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
 	if len(dst) < cap(dst) {
 		dst = dst[:len(dst)+1]
 	} else {
-		dst = append(dst, prompbmarshal.Label{})
+		dst = append(dst, prompb.Label{})
 	}
 	lb := &dst[len(dst)-1]
 	lb.Name = key

app/vlagent/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlinsert/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlinsert/datadog/datadog.go

@@ -1,260 +0,0 @@
-package datadog
-
-import (
-	"bytes"
-	"fmt"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#stream-fields")
-	datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Comma-separated list of fields to ignore for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#dropping-fields")
-
-	maxRequestSize = flagutil.NewBytes("datadog.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single DataDog request")
-)
-
-var parserPool fastjson.ParserPool
-
-// RequestHandler processes Datadog insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/datadog/api/v1/validate":
-		fmt.Fprintf(w, `{}`)
-		return true
-	case "/insert/datadog/api/v2/logs":
-		return datadogLogsIngestion(w, r)
-	default:
-		return false
-	}
-}
-
-func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	startTime := time.Now()
-	v2LogsRequestsTotal.Inc()
-
-	var ts int64
-	if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
-		var err error
-		ts, err = strconv.ParseInt(tsValue, 10, 64)
-		if err != nil {
-			httpserver.Errorf(w, r, "could not parse dd-message-timestamp header value: %s", err)
-			return true
-		}
-		ts *= 1e6
-	} else {
-		ts = startTime.UnixNano()
-	}
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = *datadogStreamFields
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *datadogIgnoreFields
-	}
-
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("datadog", false)
-		err := readLogsRequest(ts, data, lmp)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read DataDog protocol data: %s", err)
-		return true
-	}
-
-	// update v2LogsRequestDuration only for successfully parsed requests
-	// There is no need in updating v2LogsRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	v2LogsRequestDuration.UpdateDuration(startTime)
-	w.WriteHeader(http.StatusAccepted)
-	fmt.Fprintf(w, `{}`)
-	return true
-}
-
-var (
-	v2LogsRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
-	v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
-)
-
-// datadog message field has two formats:
-//   - regular log message with string text
-//   - nested json format for serverless plugins
-//     which has the following format:
-//     {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
-//
-// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
-func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage.Field, error) {
-	switch v.Type() {
-	case fastjson.TypeString:
-		val := v.GetStringBytes()
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: bytesutil.ToUnsafeString(val),
-		})
-	case fastjson.TypeObject:
-		var firstErr error
-		v.GetObject().Visit(func(k []byte, v *fastjson.Value) {
-			if firstErr != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "_msg",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "status":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "status",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "lamdba":
-				obj, err := v.Object()
-				if err != nil {
-					firstErr = err
-					firstErr = fmt.Errorf("unexpected lambda value type for %q:%q; want object", k, v)
-					return
-				}
-				obj.Visit(func(k []byte, v *fastjson.Value) {
-					if firstErr != nil {
-						return
-					}
-					val, err := v.StringBytes()
-					if err != nil {
-						firstErr = fmt.Errorf("unexpected lambda label value type for %q:%q; want string", k, v)
-						return
-					}
-					fields = append(fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(val),
-					})
-				})
-
-			}
-		})
-	default:
-		return fields, fmt.Errorf("unsupported message type %q", v.Type().String())
-	}
-	return fields, nil
-}
-
-// readLogsRequest parses data according to DataDog logs format
-// https://docs.datadoghq.com/api/latest/logs/#send-logs
-func readLogsRequest(ts int64, data []byte, lmp insertutil.LogMessageProcessor) error {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-	records, err := v.Array()
-	if err != nil {
-		return fmt.Errorf("cannot extract array from parsed JSON: %w", err)
-	}
-
-	var fields []logstorage.Field
-	for _, r := range records {
-		o, err := r.Object()
-		if err != nil {
-			return fmt.Errorf("could not extract log record: %w", err)
-		}
-		o.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				fields, err = appendMsgFields(fields, v)
-				if err != nil {
-					return
-				}
-			case "timestamp":
-				val, e := v.Int64()
-				if e != nil {
-					err = fmt.Errorf("failed to parse timestamp for %q:%q", k, v)
-				}
-				if val > 0 {
-					ts = val * 1e6
-				}
-			case "ddtags":
-				// https://docs.datadoghq.com/getting_started/tagging/
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				var pair []byte
-				idx := 0
-				for idx >= 0 {
-					idx = bytes.IndexByte(val, ',')
-					if idx < 0 {
-						pair = val
-					} else {
-						pair = val[:idx]
-						val = val[idx+1:]
-					}
-					if len(pair) > 0 {
-						n := bytes.IndexByte(pair, ':')
-						if n < 0 {
-							// No tag value.
-							fields = append(fields, logstorage.Field{
-								Name:  bytesutil.ToUnsafeString(pair),
-								Value: "no_label_value",
-							})
-						}
-						fields = append(fields, logstorage.Field{
-							Name:  bytesutil.ToUnsafeString(pair[:n]),
-							Value: bytesutil.ToUnsafeString(pair[n+1:]),
-						})
-					}
-				}
-			default:
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				fields = append(fields, logstorage.Field{
-					Name:  bytesutil.ToUnsafeString(k),
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			}
-		})
-		if err != nil {
-			return err
-		}
-		lmp.AddRow(ts, fields, nil)
-		fields = fields[:0]
-	}
-	return nil
-}

app/vlinsert/datadog/datadog_test.go

@@ -1,104 +0,0 @@
-package datadog
-
-import (
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestReadLogsRequestFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-
-		lmp := &insertutil.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if err := lmp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`["create":{}]`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadLogsRequestSuccess(t *testing.T) {
-	f := func(data string, rowsExpected int, resultExpected string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-		var timestampsExpected []int64
-		for i := 0; i < rowsExpected; i++ {
-			timestampsExpected = append(timestampsExpected, ts)
-		}
-		lmp := &insertutil.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := lmp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	// Verify non-empty data
-	data := `[
-		{
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"bar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-                        "ddtags":"tag1:value1,tag2:value2",
-                        "hostname":"127.0.0.1",
-                        "message":{"message": "nested"},
-                        "service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"foobar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"baz",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"xyz",
-			"service":"test"
-		}, {
-			"ddsource": "nginx",
-			"ddtags":"tag1:value1,tag2:value2,",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-                }, {
-			"ddsource":"nginx",
-			"ddtags":",tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-		}
-	]`
-	rowsExpected := 7
-	resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"nested","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}`
-	f(data, rowsExpected, resultExpected)
-}

app/vlinsert/elasticsearch/bulk_response.qtpl

@@ -1,20 +0,0 @@
-{% stripspace %}
-
-{% func BulkResponse(n int, tookMs int64) %}
-{
-	"took":{%dl tookMs %},
-	"errors":false,
-	"items":[
-		{% for i := 0; i < n; i++ %}
-		{
-			"create":{
-				"status":201
-			}
-		}
-		{% if i+1 < n %},{% endif %}
-		{% endfor %}
-	]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlinsert/elasticsearch/bulk_response.qtpl.go

@@ -1,69 +0,0 @@
-// Code generated by qtc from "bulk_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-package elasticsearch
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-func StreamBulkResponse(qw422016 *qt422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-	qw422016.N().S(`{"took":`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().DL(tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().S(`,"errors":false,"items":[`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-	for i := 0; i < n; i++ {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-		qw422016.N().S(`{"create":{"status":201}}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		if i+1 < n {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	qw422016.N().S(`]}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func WriteBulkResponse(qq422016 qtio422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	StreamBulkResponse(qw422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func BulkResponse(n int, tookMs int64) string {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	WriteBulkResponse(qb422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	return qs422016
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -1,248 +0,0 @@
-package elasticsearch
-
-import (
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-)
-
-var (
-	elasticsearchVersion = flag.String("elasticsearch.version", "8.9.0", "Elasticsearch version to report to client")
-)
-
-// RequestHandler processes Elasticsearch insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	// This header is needed for Logstash
-	w.Header().Set("X-Elastic-Product", "Elasticsearch")
-
-	if strings.HasPrefix(path, "/insert/elasticsearch/_ilm/policy") {
-		// Return fake response for Elasticsearch ilm request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_index_template") {
-		// Return fake response for Elasticsearch index template request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_ingest") {
-		// Return fake response for Elasticsearch ingest pipeline request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_nodes") {
-		// Return fake response for Elasticsearch nodes discovery request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/logstash") || strings.HasPrefix(path, "/insert/elasticsearch/_logstash") {
-		// Return fake response for Logstash APIs requests.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	switch path {
-	// some clients may omit trailing slash
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
-	case "/insert/elasticsearch/", "/insert/elasticsearch":
-		switch r.Method {
-		case http.MethodGet:
-			// Return fake response for Elasticsearch ping request.
-			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
-			fmt.Fprintf(w, `{
-			"version": {
-				"number": %q
-			}
-		}`, *elasticsearchVersion)
-		case http.MethodHead:
-			// Return empty response for Logstash ping request.
-		}
-
-		return true
-	case "/insert/elasticsearch/_license":
-		// Return fake response for Elasticsearch license request.
-		fmt.Fprintf(w, `{
-			"license": {
-				"uid": "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
-				"type": "oss",
-				"status": "active",
-				"expiry_date_in_millis" : 4000000000000
-			}
-		}`)
-		return true
-	case "/insert/elasticsearch/_bulk":
-		startTime := time.Now()
-		bulkRequestsTotal.Inc()
-
-		cp, err := insertutil.GetCommonParams(r)
-		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		if err := insertutil.CanWriteData(); err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk", true)
-		encoding := r.Header.Get("Content-Encoding")
-		streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-		n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeFields, cp.MsgFields, lmp)
-		lmp.MustClose()
-		if err != nil {
-			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
-			return true
-		}
-
-		tookMs := time.Since(startTime).Milliseconds()
-		bw := bufferedwriter.Get(w)
-		defer bufferedwriter.Put(bw)
-		WriteBulkResponse(bw, n, tookMs)
-		_ = bw.Flush()
-
-		// update bulkRequestDuration only for successfully parsed requests
-		// There is no need in updating bulkRequestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		bulkRequestDuration.UpdateDuration(startTime)
-
-		return true
-	default:
-		return false
-	}
-}
-
-var (
-	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
-	bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
-)
-
-func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
-	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
-
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return 0, fmt.Errorf("cannot decode Elasticsearch protocol data: %w", err)
-	}
-	defer protoparserutil.PutUncompressedReader(reader)
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lr := insertutil.NewLineReader(streamName, wcr)
-
-	n := 0
-	for {
-		ok, err := readBulkLine(lr, timeFields, msgFields, lmp)
-		wcr.DecConcurrency()
-		if err != nil || !ok {
-			return n, err
-		}
-		n++
-	}
-}
-
-func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
-	var line []byte
-
-	// Read the command, must be "create" or "index"
-	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
-		}
-		line = lr.Line
-	}
-	lineStr := bytesutil.ToUnsafeString(line)
-	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
-		return false, fmt.Errorf(`unexpected command %q; expecting "create" or "index"`, line)
-	}
-
-	// Decode log message
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return false, err
-		}
-		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
-	}
-	line = lr.Line
-	if len(line) == 0 {
-		// Special case - the line could be too long, so it was skipped.
-		// Continue parsing next lines.
-		return true, nil
-	}
-	p := logstorage.GetJSONParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-
-	ts, err := extractTimestampFromFields(timeFields, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
-	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
-	logstorage.PutJSONParser(p)
-
-	return true, nil
-}
-
-func extractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			timestamp, err := parseElasticsearchTimestamp(f.Value)
-			if err != nil {
-				return 0, err
-			}
-			f.Value = ""
-			return timestamp, nil
-		}
-	}
-	return 0, nil
-}
-
-func parseElasticsearchTimestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case - zero or empty timestamp must be substituted
-		// with the current time by the caller.
-		return 0, nil
-	}
-	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in seconds or milliseconds
-		return insertutil.ParseUnixTimestamp(s)
-	}
-	if len(s) == len("YYYY-MM-DD") {
-		t, err := time.Parse("2006-01-02", s)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse date %q: %w", s, err)
-		}
-		return t.UnixNano(), nil
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot parse timestamp %q", s)
-	}
-	return nsecs, nil
-}

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -1,128 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/golang/snappy"
-	"github.com/klauspost/compress/gzip"
-	"github.com/klauspost/compress/zlib"
-	"github.com/klauspost/compress/zstd"
-	"io"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestReadBulkRequest_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", []string{"_time"}, []string{"_msg"}, tlp)
-		if err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if rows != 0 {
-			t.Fatalf("unexpected non-zero rows=%d", rows)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`{"create":{}}`)
-	f(`{"creat":{}}
-{}`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadBulkRequest_Success(t *testing.T) {
-	f := func(data, encoding, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		timeFields := []string{"non_existing_foo", timeField, "non_existing_bar"}
-		msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		// Read the request without compression
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", timeFields, msgFields, tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-
-		// Read the request with compression
-		tlp = &insertutil.TestLogMessageProcessor{}
-		if encoding != "" {
-			data = compressData(data, encoding)
-		}
-		r = bytes.NewBufferString(data)
-		rows, err = readBulkRequest("test", r, encoding, timeFields, msgFields, tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("verification failure after compression: %s", err)
-		}
-	}
-
-	// Verify an empty data
-	f("", "gzip", "_time", "_msg", nil, "")
-	f("\n", "gzip", "_time", "_msg", nil, "")
-	f("\n\n", "gzip", "_time", "_msg", nil, "")
-
-	// Verify non-empty data
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06 04:48:12.735+01:00","message":"baz"}
-{"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"1686026893735","x":"y"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty","@timestamp":"1686026893"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty float","@timestamp":"1686026123.62"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000, 1686026123620000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}
-{"_msg":"qwe rty"}
-{"_msg":"qwe rty float"}`
-	f(data, "zstd", timeField, msgField, timestampsExpected, resultExpected)
-}
-
-func compressData(s string, encoding string) string {
-	var bb bytes.Buffer
-	var zw io.WriteCloser
-	switch encoding {
-	case "gzip":
-		zw = gzip.NewWriter(&bb)
-	case "zstd":
-		zw, _ = zstd.NewWriter(&bb)
-	case "snappy":
-		return string(snappy.Encode(nil, []byte(s)))
-	case "deflate":
-		zw = zlib.NewWriter(&bb)
-	default:
-		panic(fmt.Errorf("%q encoding is not supported", encoding))
-	}
-	if _, err := zw.Write([]byte(s)); err != nil {
-		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
-	}
-	if err := zw.Close(); err != nil {
-		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
-	}
-	return bb.String()
-}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -1,59 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-)
-
-func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("encoding:none", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "")
-	})
-	b.Run("encoding:gzip", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "gzip")
-	})
-	b.Run("encoding:zstd", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "zstd")
-	})
-	b.Run("encoding:deflate", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "deflate")
-	})
-	b.Run("encoding:snappy", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "snappy")
-	})
-}
-
-func benchmarkReadBulkRequest(b *testing.B, encoding string) {
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	if encoding != "" {
-		data = compressData(data, encoding)
-	}
-	dataBytes := bytesutil.ToUnsafeBytes(data)
-
-	timeFields := []string{"@timestamp"}
-	msgFields := []string{"message"}
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		for pb.Next() {
-			r.Reset(dataBytes)
-			_, err := readBulkRequest("test", r, encoding, timeFields, msgFields, blp)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}

app/vlinsert/insertutil/common_params.go

@@ -1,335 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	defaultMsgValue = flag.String("defaultMsgValue", "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field",
-		"Default value for _msg field if the ingested log entry doesn't contain it; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field")
-)
-
-// CommonParams contains common HTTP parameters used by log ingestion APIs.
-//
-// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
-type CommonParams struct {
-	TenantID         logstorage.TenantID
-	TimeFields       []string
-	MsgFields        []string
-	StreamFields     []string
-	IgnoreFields     []string
-	DecolorizeFields []string
-	ExtraFields      []logstorage.Field
-
-	IsTimeFieldSet  bool
-	Debug           bool
-	DebugRequestURI string
-	DebugRemoteAddr string
-}
-
-// GetCommonParams returns CommonParams from r.
-func GetCommonParams(r *http.Request) (*CommonParams, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, err
-	}
-
-	var isTimeFieldSet bool
-	timeFields := []string{"_time"}
-	if tfs := httputil.GetArray(r, "_time_field", "VL-Time-Field"); len(tfs) > 0 {
-		isTimeFieldSet = true
-		timeFields = tfs
-	}
-
-	msgFields := httputil.GetArray(r, "_msg_field", "VL-Msg-Field")
-	streamFields := httputil.GetArray(r, "_stream_fields", "VL-Stream-Fields")
-	ignoreFields := httputil.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
-	decolorizeFields := httputil.GetArray(r, "decolorize_fields", "VL-Decolorize-Fields")
-
-	extraFields, err := getExtraFields(r)
-	if err != nil {
-		return nil, err
-	}
-
-	debug := false
-	if dv := httputil.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
-		debug, err = strconv.ParseBool(dv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
-		}
-	}
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-	if debug {
-		debugRequestURI = httpserver.GetRequestURI(r)
-		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
-	}
-
-	cp := &CommonParams{
-		TenantID:         tenantID,
-		TimeFields:       timeFields,
-		MsgFields:        msgFields,
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-
-		IsTimeFieldSet:  isTimeFieldSet,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
-	}
-
-	return cp, nil
-}
-
-func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
-	efs := httputil.GetArray(r, "extra_fields", "VL-Extra-Fields")
-	if len(efs) == 0 {
-		return nil, nil
-	}
-
-	extraFields := make([]logstorage.Field, len(efs))
-	for i, ef := range efs {
-		n := strings.Index(ef, "=")
-		if n <= 0 || n == len(ef)-1 {
-			return nil, fmt.Errorf(`invalid extra_field format: %q; must be in the form "field=value"`, ef)
-		}
-		extraFields[i] = logstorage.Field{
-			Name:  ef[:n],
-			Value: ef[n+1:],
-		}
-	}
-	return extraFields, nil
-}
-
-// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) *CommonParams {
-	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
-	if streamFields == nil {
-		streamFields = []string{
-			"hostname",
-			"app_name",
-			"proc_id",
-		}
-	}
-	cp := &CommonParams{
-		TenantID: tenantID,
-		TimeFields: []string{
-			"timestamp",
-		},
-		MsgFields: []string{
-			"message",
-		},
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-	}
-
-	return cp
-}
-
-// LogRowsStorage is an interface for ingesting logs into the storage.
-type LogRowsStorage interface {
-	// MustAddRows must add lr to the underlying storage.
-	MustAddRows(lr *logstorage.LogRows)
-
-	// CanWriteData must returns non-nil error if logs cannot be added to the underlying storage.
-	CanWriteData() error
-}
-
-var logRowsStorage LogRowsStorage
-
-// SetLogRowsStorage sets the storage for writing data to via LogMessageProcessor.
-//
-// This function must be called before using LogMessageProcessor and CanWriteData from this package.
-func SetLogRowsStorage(storage LogRowsStorage) {
-	logRowsStorage = storage
-}
-
-// CanWriteData returns non-nil error if data cannot be written to the underlying storage.
-func CanWriteData() error {
-	return logRowsStorage.CanWriteData()
-}
-
-// LogMessageProcessor is an interface for log message processors.
-type LogMessageProcessor interface {
-	// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
-	//
-	// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
-	//
-	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can reuse them.
-	AddRow(timestamp int64, fields, streamFields []logstorage.Field)
-
-	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
-	MustClose()
-}
-
-type logMessageProcessor struct {
-	mu            sync.Mutex
-	wg            sync.WaitGroup
-	stopCh        chan struct{}
-	lastFlushTime time.Time
-
-	cp *CommonParams
-	lr *logstorage.LogRows
-
-	rowsIngestedTotal  *metrics.Counter
-	bytesIngestedTotal *metrics.Counter
-}
-
-func (lmp *logMessageProcessor) initPeriodicFlush() {
-	lmp.lastFlushTime = time.Now()
-
-	lmp.wg.Add(1)
-	go func() {
-		defer lmp.wg.Done()
-
-		d := timeutil.AddJitterToDuration(time.Second)
-		ticker := time.NewTicker(d)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-lmp.stopCh:
-				return
-			case <-ticker.C:
-				lmp.mu.Lock()
-				if time.Since(lmp.lastFlushTime) >= d {
-					lmp.flushLocked()
-				}
-				lmp.mu.Unlock()
-			}
-		}
-	}()
-}
-
-// AddRow adds new log message to lmp with the given timestamp and fields.
-//
-// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
-func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// InsertRowProcessor is used by native data ingestion protocol parser.
-type InsertRowProcessor interface {
-	// AddInsertRow must add r to the underlying storage.
-	AddInsertRow(r *logstorage.InsertRow)
-}
-
-// AddInsertRow adds r to lmp.
-func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(r.Fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(r.Fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAddInsertRow(r)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// flushLocked must be called under locked lmp.mu.
-func (lmp *logMessageProcessor) flushLocked() {
-	lmp.lastFlushTime = time.Now()
-	logRowsStorage.MustAddRows(lmp.lr)
-	lmp.lr.ResetKeepSettings()
-}
-
-// MustClose flushes the remaining data to the underlying storage and closes lmp.
-func (lmp *logMessageProcessor) MustClose() {
-	close(lmp.stopCh)
-	lmp.wg.Wait()
-
-	lmp.flushLocked()
-	logstorage.PutLogRows(lmp.lr)
-	lmp.lr = nil
-}
-
-// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
-//
-// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
-func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode bool) LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.DecolorizeFields, cp.ExtraFields, *defaultMsgValue)
-	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
-	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
-	lmp := &logMessageProcessor{
-		cp: cp,
-		lr: lr,
-
-		rowsIngestedTotal:  rowsIngestedTotal,
-		bytesIngestedTotal: bytesIngestedTotal,
-
-		stopCh: make(chan struct{}),
-	}
-
-	if isStreamMode {
-		lmp.initPeriodicFlush()
-	}
-
-	return lmp
-}
-
-var (
-	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
-)

app/vlinsert/insertutil/flags.go

@@ -1,17 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-)
-
-var (
-	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
-
-	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
-)

app/vlinsert/insertutil/line_reader.go

@@ -1,158 +0,0 @@
-package insertutil
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-)
-
-// LineReader reads newline-delimited lines from the underlying reader
-type LineReader struct {
-	// Line contains the next line read after the call to NextLine
-	//
-	// The Line contents is valid until the next call to NextLine.
-	Line []byte
-
-	// name is the LineReader name
-	name string
-
-	// r is the underlying reader to read data from
-	r io.Reader
-
-	// buf is a buffer for reading the next line
-	buf []byte
-
-	// bufOffset is the offset at buf to read the next line from
-	bufOffset int
-
-	// err is the last error when reading data from r
-	err error
-
-	// eofReached is set to true when all the data is read from r
-	eofReached bool
-}
-
-// NewLineReader returns LineReader for r.
-func NewLineReader(name string, r io.Reader) *LineReader {
-	return &LineReader{
-		name: name,
-		r:    r,
-	}
-}
-
-// NextLine reads the next line from the underlying reader.
-//
-// It returns true if the next line is successfully read into Line.
-// If the line length exceeds MaxLineSizeBytes, then this line is skipped
-// and an empty line is returned instead.
-//
-// If false is returned, then no more lines left to read from r.
-// Check for Err in this case.
-func (lr *LineReader) NextLine() bool {
-	for {
-		lr.Line = nil
-		if lr.bufOffset >= len(lr.buf) {
-			if lr.err != nil || lr.eofReached {
-				return false
-			}
-			if !lr.readMoreData() {
-				return false
-			}
-			if lr.bufOffset >= len(lr.buf) && lr.eofReached {
-				return false
-			}
-		}
-
-		buf := lr.buf[lr.bufOffset:]
-		if n := bytes.IndexByte(buf, '\n'); n >= 0 {
-			lr.Line = buf[:n]
-			lr.bufOffset += n + 1
-			return true
-		}
-		if lr.eofReached {
-			lr.Line = buf
-			lr.bufOffset += len(buf)
-			return true
-		}
-		if !lr.readMoreData() {
-			return false
-		}
-	}
-}
-
-// Err returns the last error after NextLine call.
-func (lr *LineReader) Err() error {
-	if lr.err == nil {
-		return nil
-	}
-	return fmt.Errorf("%s: %s", lr.name, lr.err)
-}
-
-func (lr *LineReader) readMoreData() bool {
-	if lr.bufOffset > 0 {
-		lr.buf = append(lr.buf[:0], lr.buf[lr.bufOffset:]...)
-		lr.bufOffset = 0
-	}
-
-	bufLen := len(lr.buf)
-	if bufLen >= MaxLineSizeBytes.IntN() {
-		ok, skippedBytes := lr.skipUntilNextLine()
-		logger.Warnf("%s: the line length exceeds -insert.maxLineSizeBytes=%d; skipping it; total skipped bytes=%d",
-			lr.name, MaxLineSizeBytes.IntN(), skippedBytes)
-		tooLongLinesSkipped.Inc()
-		return ok
-	}
-
-	lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-	n, err := lr.r.Read(lr.buf[bufLen:])
-	lr.buf = lr.buf[:bufLen+n]
-	if err != nil {
-		if errors.Is(err, io.EOF) {
-			lr.eofReached = true
-			return true
-		}
-		lr.err = fmt.Errorf("cannot read the next line: %s", err)
-	}
-	return n > 0
-}
-
-var tooLongLinesSkipped = metrics.NewCounter("vl_too_long_lines_skipped_total")
-
-func (lr *LineReader) skipUntilNextLine() (bool, int) {
-
-	// Initialize skipped bytes count with MaxLineSizeBytes because
-	// we've already read that many bytes without encountering a newline,
-	// indicating the line size exceeds the maximum allowed limit.
-	skipSizeBytes := MaxLineSizeBytes.IntN()
-
-	for {
-		lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-		n, err := lr.r.Read(lr.buf)
-		skipSizeBytes += n
-		lr.buf = lr.buf[:n]
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				lr.eofReached = true
-				lr.buf = lr.buf[:0]
-				return true, skipSizeBytes
-			}
-			lr.err = fmt.Errorf("cannot skip the current line: %s", err)
-			return false, skipSizeBytes
-		}
-		if n := bytes.IndexByte(lr.buf, '\n'); n >= 0 {
-			// Include skipped bytes before \n, including the newline itself.
-			skipSizeBytes += n + 1 - len(lr.buf)
-			// Include \n in the buf, so too long line is replaced with an empty line.
-			// This is needed for maintaining synchorinzation consistency between lines
-			// in protocols such as Elasticsearch bulk import.
-			lr.buf = append(lr.buf[:0], lr.buf[n:]...)
-			return true, skipSizeBytes
-		}
-	}
-}

app/vlinsert/insertutil/line_reader_test.go

@@ -1,164 +0,0 @@
-package insertutil
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"reflect"
-	"testing"
-)
-
-func TestLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if len(lr.Line) > 0 {
-			t.Fatalf("unexpected non-empty line after failed NextLine(): %q", lr.Line)
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", []string{""})
-	f("\n\n", []string{"", ""})
-	f("foo", []string{"foo"})
-	f("foo\n", []string{"foo"})
-	f("\nfoo", []string{"", "foo"})
-	f("foo\n\n", []string{"foo", ""})
-	f("foo\nbar", []string{"foo", "bar"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\n\nbar\n\n", []string{"", "foo", "", "bar", ""})
-}
-
-func TestLineReader_SkipUntilNextLine(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error for data=%q: %s", data, err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines for data=%q\ngot\n%q\nwant\n%q", data, lines, linesExpected)
-		}
-	}
-
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLineLen := MaxLineSizeBytes.IntN() + overflow
-		longLine := string(make([]byte, longLineLen))
-
-		// Single long line
-		data := longLine
-		f(data, nil)
-
-		// Multiple long lines
-		data = longLine + "\n" + longLine
-		f(data, []string{""})
-
-		data = longLine + "\n" + longLine + "\n"
-		f(data, []string{"", ""})
-
-		// Long line in the middle
-		data = "foo\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "bar"})
-
-		// Multiple long lines in the middle
-		data = "foo\n" + longLine + "\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "", "bar"})
-
-		// Long line in the end
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		// Long line in the end
-		data = "foo\n" + longLine + "\n"
-		f(data, []string{"foo", ""})
-	}
-}
-
-func TestLineReader_Failure(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		fr := &failureReader{
-			r: bytes.NewBufferString(data),
-		}
-		lr := NewLineReader("foo", fr)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error on the second call")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("foo", nil)
-	f("foo\n", []string{"foo"})
-	f("\n", []string{""})
-	f("foo\nbar", []string{"foo"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\nbar\n\n", []string{"", "foo", "bar", ""})
-
-	// long line
-	longLineLen := MaxLineSizeBytes.IntN()
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLine := string(make([]byte, longLineLen+overflow))
-
-		data := longLine
-		f(data, nil)
-
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		data = longLine + "\nfoo"
-		f(data, []string{""})
-
-		data = longLine + "\nfoo\n"
-		f(data, []string{"", "foo"})
-	}
-}
-
-type failureReader struct {
-	r io.Reader
-}
-
-func (r *failureReader) Read(p []byte) (int, error) {
-	n, _ := r.r.Read(p)
-	if n > 0 {
-		return n, nil
-	}
-	return 0, fmt.Errorf("some error")
-}

app/vlinsert/insertutil/testutils.go

@@ -1,56 +0,0 @@
-package insertutil
-
-import (
-	"fmt"
-	"reflect"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// TestLogMessageProcessor implements LogMessageProcessor for testing.
-type TestLogMessageProcessor struct {
-	timestamps []int64
-	rows       []string
-}
-
-// AddRow adds row with the given timestamp and fields to tlp
-func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("BUG: streamFields must be nil; got %v", streamFields))
-	}
-	tlp.timestamps = append(tlp.timestamps, timestamp)
-	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
-}
-
-// MustClose closes tlp.
-func (tlp *TestLogMessageProcessor) MustClose() {
-}
-
-// Verify verifies the number of rows, timestamps and results after AddRow calls.
-func (tlp *TestLogMessageProcessor) Verify(timestampsExpected []int64, resultExpected string) error {
-	result := strings.Join(tlp.rows, "\n")
-	if len(tlp.rows) != len(timestampsExpected) {
-		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), len(timestampsExpected), result, resultExpected)
-	}
-
-	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
-		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
-	}
-	if result != resultExpected {
-		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-	}
-
-	return nil
-}
-
-// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
-type BenchmarkLogMessageProcessor struct{}
-
-// AddRow implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _, _ []logstorage.Field) {
-}
-
-// MustClose implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) MustClose() {
-}

app/vlinsert/insertutil/timestamp.go

@@ -1,106 +0,0 @@
-package insertutil
-
-import (
-	"fmt"
-	"math"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ExtractTimestampFromFields extracts timestamp in nanoseconds from the first field the name from timeFields at fields.
-//
-// The value for the corresponding timeFields is set to empty string after returning from the function,
-// so it could be ignored during data ingestion.
-//
-// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			nsecs, err := parseTimestamp(f.Value)
-			if err != nil {
-				return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", f.Name, err)
-			}
-			f.Value = ""
-			if nsecs == 0 {
-				nsecs = time.Now().UnixNano()
-			}
-			return nsecs, nil
-		}
-	}
-	return time.Now().UnixNano(), nil
-}
-
-func parseTimestamp(s string) (int64, error) {
-	// "-" is a nil timestamp value, if the syslog
-	// application is incapable of obtaining system time
-	// https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
-	if s == "" || s == "0" || s == "-" {
-		return time.Now().UnixNano(), nil
-	}
-	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
-		return ParseUnixTimestamp(s)
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
-	}
-	return nsecs, nil
-}
-
-// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
-func ParseUnixTimestamp(s string) (int64, error) {
-	if strings.IndexByte(s, '.') >= 0 {
-		// Parse timestamp as floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-		}
-		if f < (1<<31) && f >= (-1<<31) {
-			// The timestamp is in seconds.
-			return int64(f * 1e9), nil
-		}
-		if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
-			// The timestamp is in milliseconds.
-			return int64(f * 1e6), nil
-		}
-		if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
-			// The timestamp is in microseconds.
-			return int64(f * 1e3), nil
-		}
-		// The timestamp is in nanoseconds
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		return int64(f), nil
-	}
-
-	// Parse timestamp as integer
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-	}
-	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds.
-		return n * 1e9, nil
-	}
-	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
-		// The timestamp is in milliseconds.
-		return n * 1e6, nil
-	}
-	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
-		// The timestamp is in microseconds.
-		return n * 1e3, nil
-	}
-	// The timestamp is in nanoseconds
-	return n, nil
-}

app/vlinsert/insertutil/timestamp_test.go

@@ -1,185 +0,0 @@
-package insertutil
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestParseUnixTimestamp_Success(t *testing.T) {
-	f := func(s string, timestampExpected int64) {
-		t.Helper()
-
-		timestamp, err := ParseUnixTimestamp(s)
-		if err != nil {
-			t.Fatalf("unexpected error in ParseUnixTimestamp(%q): %s", s, err)
-		}
-		if timestamp != timestampExpected {
-			t.Fatalf("unexpected timestamp returned from ParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
-		}
-	}
-
-	f("0", 0)
-
-	// nanoseconds
-	f("-1234567890123456789", -1234567890123456789)
-	f("1234567890123456789", 1234567890123456789)
-
-	// microseconds
-	f("-1234567890123456", -1234567890123456000)
-	f("1234567890123456", 1234567890123456000)
-	f("1234567890123456.789", 1234567890123456768)
-
-	// milliseconds
-	f("-1234567890123", -1234567890123000000)
-	f("1234567890123", 1234567890123000000)
-	f("1234567890123.456", 1234567890123456000)
-
-	// seconds
-	f("-1234567890", -1234567890000000000)
-	f("1234567890", 1234567890000000000)
-	f("-1234567890.123456", -1234567890123456000)
-}
-
-func TestParseUnixTimestamp_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := ParseUnixTimestamp(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error in ParseUnixTimestamp(%q)", s)
-		}
-	}
-
-	// non-numeric timestamp
-	f("")
-	f("foobar")
-	f("foo.bar")
-
-	// too big timestamp
-	f("12345678901234567890")
-	f("-12345678901234567890")
-	f("12345678901234567890.235424")
-	f("-12345678901234567890.235424")
-}
-
-func TestExtractTimestampFromFields_Success(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs != nsecsExpected {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
-		}
-
-		for _, f := range fields {
-			if f.Name == timeField {
-				if f.Value != "" {
-					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
-				}
-			}
-		}
-	}
-
-	// UTC time
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20Z"},
-	}, 1718753840000000000)
-
-	// Time with timezone
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
-	}, 1718725040000000000)
-
-	// SQL datetime format
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
-	}, 1718773640123000000)
-
-	// Time with nanosecond precision
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
-		{Name: "foo", Value: "bar"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in nanoseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456789"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in microseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456"},
-	}, 1718773640123456000)
-
-	// Unix timestamp in milliseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123"},
-	}, 1718773640123000000)
-
-	// Unix timestamp in seconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640"},
-	}, 1718773640000000000)
-}
-
-func TestExtractTimestampFromFields_Now(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs < 1 {
-			t.Fatalf("expected generated timestamp, got error: %s", err)
-		}
-	}
-
-	// RFC5424 allows `-` for nil timestamp (log ingestion time)
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "-"},
-	})
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: ""},
-	})
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "0"},
-	})
-}
-
-func TestExtractTimestampFromFields_Error(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		fields := []logstorage.Field{
-			{Name: "time", Value: s},
-		}
-		nsecs, err := ExtractTimestampFromFields([]string{"time"}, fields)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if nsecs != 0 {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
-		}
-	}
-
-	// invalid time
-	f("foobar")
-
-	// incomplete time
-	f("2024-06-18")
-	f("2024-06-18T23:37")
-}

app/vlinsert/internalinsert/internalinsert.go

@@ -1,88 +0,0 @@
-package internalinsert
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
-)
-
-// RequestHandler processes /internal/insert requests.
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-	version := r.FormValue("version")
-	if version != netinsert.ProtocolVersion {
-		httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("internalinsert", false)
-		irp := lmp.(insertutil.InsertRowProcessor)
-		err := parseData(irp, data)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
-		return
-	}
-
-	requestDuration.UpdateDuration(startTime)
-}
-
-func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
-	r := logstorage.GetInsertRow()
-	src := data
-	i := 0
-	for len(src) > 0 {
-		tail, err := r.UnmarshalInplace(src)
-		if err != nil {
-			return fmt.Errorf("cannot parse row #%d: %s", i, err)
-		}
-		src = tail
-		i++
-
-		irp.AddInsertRow(r)
-	}
-	logstorage.PutInsertRow(r)
-
-	return nil
-}
-
-var (
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
-
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/internal/insert"}`)
-)

app/vlinsert/journald/journald.go

@@ -1,377 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"slices"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
-const maxFieldNameLen = 64
-
-var (
-	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
-	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Comma-separated list of fields to ignore for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#dropping-fields")
-	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Field to use as a log timestamp for logs ingested via journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
-	journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
-	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include Journald fields with double underscore prefixes")
-)
-
-func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		tenantID, err := logstorage.ParseTenantID(*journaldTenantID)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse -journald.tenantID=%q for journald: %w", *journaldTenantID, err)
-		}
-		cp.TenantID = tenantID
-	}
-
-	if !cp.IsTimeFieldSet {
-		cp.TimeFields = []string{*journaldTimeField}
-	}
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = getStreamFields()
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *journaldIgnoreFields
-	}
-	cp.MsgFields = []string{"MESSAGE"}
-	return cp, nil
-}
-
-func getStreamFields() []string {
-	if len(*journaldStreamFields) > 0 {
-		return *journaldStreamFields
-	}
-	return defaultStreamFields
-}
-
-var defaultStreamFields = []string{
-	"_MACHINE_ID",
-	"_HOSTNAME",
-	"_SYSTEMD_UNIT",
-}
-
-// RequestHandler processes Journald Export insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/journald/upload":
-		if r.Header.Get("Content-Type") != "application/vnd.fdo.journal" {
-			httpserver.Errorf(w, r, "only application/vnd.fdo.journal encoding is supported for Journald")
-			return true
-		}
-		handleJournald(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-// handleJournald parses Journal binary entries
-func handleJournald(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsTotal.Inc()
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-
-	if err := insertutil.CanWriteData(); err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
-	if err != nil {
-		errorsTotal.Inc()
-		logger.Errorf("cannot decode journald request: %s", err)
-		return
-	}
-
-	lmp := cp.NewLogMessageProcessor("journald", true)
-	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, lmp, cp)
-	protoparserutil.PutUncompressedReader(reader)
-	lmp.MustClose()
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
-		return
-	}
-
-	// systemd starting release v258 will support compression, which starts working after negotiation: it expects supported compression
-	// algorithms list in Accept-Encoding response header in a format "<algorithm_1>[:<priority_1>][;<algorithm_2>:<priority_2>]"
-	// See https://github.com/systemd/systemd/pull/34822
-	w.Header().Set("Accept-Encoding", "zstd")
-
-	// update requestDuration only for successfully parsed requests
-	// There is no need in updating requestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
-	errorsTotal     = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
-)
-
-func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lr := insertutil.NewLineReader("journald", wcr)
-
-	for {
-		err := readJournaldLogEntry(streamName, lr, lmp, cp)
-		wcr.DecConcurrency()
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				return nil
-			}
-			return fmt.Errorf("%s: %w", streamName, err)
-		}
-	}
-}
-
-type fieldsBuf struct {
-	fields []logstorage.Field
-
-	buf   []byte
-	name  []byte
-	value []byte
-}
-
-func (fb *fieldsBuf) reset() {
-	fb.fields = fb.fields[:0]
-	fb.buf = fb.buf[:0]
-	fb.name = fb.name[:0]
-	fb.value = fb.value[:0]
-}
-
-func (fb *fieldsBuf) addField(name, value string) {
-	bufLen := len(fb.buf)
-	fb.buf = append(fb.buf, name...)
-	nameCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
-
-	bufLen = len(fb.buf)
-	fb.buf = append(fb.buf, value...)
-	valueCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
-
-	fb.fields = append(fb.fields, logstorage.Field{
-		Name:  nameCopy,
-		Value: valueCopy,
-	})
-}
-
-func (fb *fieldsBuf) appendNextLineToValue(lr *insertutil.LineReader) error {
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return err
-		}
-		return fmt.Errorf("unexpected end of stream")
-	}
-	fb.value = append(fb.value, lr.Line...)
-	fb.value = append(fb.value, '\n')
-	return nil
-}
-
-func getFieldsBuf() *fieldsBuf {
-	fb := fieldsBufPool.Get()
-	if fb == nil {
-		return &fieldsBuf{}
-	}
-	return fb.(*fieldsBuf)
-}
-
-func putFieldsBuf(fb *fieldsBuf) {
-	fb.reset()
-	fieldsBufPool.Put(fb)
-}
-
-var fieldsBufPool sync.Pool
-
-// readJournaldLogEntry reads a single log entry in Journald format.
-//
-// See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
-	var ts int64
-	var name, value string
-
-	fb := getFieldsBuf()
-	defer putFieldsBuf(fb)
-
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return fmt.Errorf("cannot read the first field: %w", err)
-		}
-		return io.EOF
-	}
-
-	for {
-		line := lr.Line
-		if len(line) == 0 {
-			// The end of a single log entry. Write it to the storage
-			if len(fb.fields) > 0 {
-				if ts == 0 {
-					ts = time.Now().UnixNano()
-				}
-				lmp.AddRow(ts, fb.fields, nil)
-			}
-			return nil
-		}
-
-		// line could be either "key=value" or "key"
-		// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-		if n := bytes.IndexByte(line, '='); n >= 0 {
-			// line = "key=value"
-			fb.name = append(fb.name[:0], line[:n]...)
-			name = bytesutil.ToUnsafeString(fb.name)
-
-			fb.value = append(fb.value[:0], line[n+1:]...)
-			value = bytesutil.ToUnsafeString(fb.value)
-		} else {
-			// line = "key"
-			// Parse the binary-encoded value from the next line according to "key\n<little_endian_size_64>value\n" format
-			fb.name = append(fb.name[:0], line...)
-			name = bytesutil.ToUnsafeString(fb.name)
-
-			fb.value = fb.value[:0]
-			for len(fb.value) < 8 {
-				if err := fb.appendNextLineToValue(lr); err != nil {
-					return fmt.Errorf("cannot read value size: %w", err)
-				}
-			}
-			size := binary.LittleEndian.Uint64(fb.value[:8])
-
-			// Read the value until its lenth exceeds the given size - the last char in the read value will always be '\n'
-			// because it is appended by appendNextLineToValue().
-			for uint64(len(fb.value[8:])) <= size {
-				if err := fb.appendNextLineToValue(lr); err != nil {
-					return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
-				}
-			}
-			value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
-			if uint64(len(value)) != size {
-				return fmt.Errorf("unexpected %q value size; got %d bytes; want %d bytes; value: %q", fb.name, len(value), size, value)
-			}
-		}
-
-		if !lr.NextLine() {
-			if err := lr.Err(); err != nil {
-				return fmt.Errorf("cannot read the next log field: %w", err)
-			}
-
-			// add the last log field below before the return
-		}
-
-		if len(name) > maxFieldNameLen {
-			logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
-			continue
-		}
-		if !isValidFieldName(name) {
-			logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
-			continue
-		}
-
-		if slices.Contains(cp.TimeFields, name) {
-			t, err := strconv.ParseInt(value, 10, 64)
-			if err != nil {
-				logger.Errorf("%s: cannot parse timestamp from the field %q: %w; using the current timestamp", streamName, name, err)
-				ts = 0
-			} else {
-				// Convert journald microsecond timestamp to nanoseconds
-				ts = t * 1e3
-			}
-			continue
-		}
-
-		if slices.Contains(cp.MsgFields, name) {
-			name = "_msg"
-		}
-
-		if name == "PRIORITY" {
-			priority := journaldPriorityToLevel(value)
-			fb.addField("level", priority)
-		}
-
-		if !strings.HasPrefix(name, "__") || *journaldIncludeEntryMetadata {
-			fb.addField(name, value)
-		}
-	}
-}
-
-func journaldPriorityToLevel(priority string) string {
-	// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
-	// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
-	switch priority {
-	case "0":
-		return "emerg"
-	case "1":
-		return "alert"
-	case "2":
-		return "critical"
-	case "3":
-		return "error"
-	case "4":
-		return "warning"
-	case "5":
-		return "notice"
-	case "6":
-		return "info"
-	case "7":
-		return "debug"
-	default:
-		return priority
-	}
-}
-
-func isValidFieldName(s string) bool {
-	if len(s) == 0 {
-		return false
-	}
-	c := s[0]
-	if !(c >= 'A' && c <= 'Z' || c == '_') {
-		return false
-	}
-
-	for i := 1; i < len(s); i++ {
-		c := s[i]
-		if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
-			return false
-		}
-	}
-	return true
-}

app/vlinsert/journald/journald_test.go

@@ -1,166 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"net/http"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestIsValidFieldName(t *testing.T) {
-	f := func(name string, resultExpected bool) {
-		t.Helper()
-
-		result := isValidFieldName(name)
-		if result != resultExpected {
-			t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
-		}
-	}
-
-	f("", false)
-	f("a", false)
-	f("1", false)
-	f("_", true)
-	f("X", true)
-	f("Xa", false)
-	f("X_343", true)
-	f("X_0123456789_AZ", true)
-	f("SDDFD sdf", false)
-}
-
-func TestGetCommonParams_TimeField(t *testing.T) {
-	f := func(timeFieldHeader, expectedTimeField string) {
-		t.Helper()
-
-		req, err := http.NewRequest("POST", "/insert/journald/upload", nil)
-		if err != nil {
-			t.Fatalf("unexpected error creating request: %s", err)
-		}
-
-		if timeFieldHeader != "" {
-			req.Header.Set("VL-Time-Field", timeFieldHeader)
-		}
-
-		cp, err := getCommonParams(req)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if len(cp.TimeFields) != 1 || cp.TimeFields[0] != expectedTimeField {
-			t.Fatalf("unexpected TimeFields; got %v; want [%s]", cp.TimeFields, expectedTimeField)
-		}
-	}
-
-	// Test default behavior - when no custom time field is specified, journald uses __REALTIME_TIMESTAMP
-	f("", "__REALTIME_TIMESTAMP")
-
-	// Test custom time field - when a custom time field is specified via HTTP header, it's respected
-	f("custom_time", "custom_time")
-}
-
-func TestPushJournald_Success(t *testing.T) {
-	f := func(src string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		cp, err := getCommonParams(r)
-		if err != nil {
-			t.Fatalf("cannot create commonParams: %s", err)
-		}
-
-		buf := bytes.NewBufferString(src)
-		if err := processStreamInternal("test", buf, tlp, cp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Single event
-	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n",
-		[]int64{91723819283000},
-		"{\"_msg\":\"Test message\"}",
-	)
-
-	// Multiple events
-	f("__REALTIME_TIMESTAMP=91723819283\nPRIORITY=3\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
-		[]int64{91723819283000, 91723819284000},
-		"{\"level\":\"error\",\"PRIORITY\":\"3\",\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
-	)
-
-	// Parse binary data
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\nE=JobStateChanged\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
-		[]int64{1729698775704404000},
-		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
-	)
-
-	// Parse binary data with trailing newline
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x14\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"foo\nbar\n\n\nasda\nasda\n","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"123456789\n","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"1234567890","_PID":"2763"}`,
-	)
-
-	// Empty field name must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
-	f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
-
-	// field name starting with number must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
-
-	// field name exceeding 64 bytes limit must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n", nil, "")
-
-	// field name with invalid chars must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n", nil, "")
-}
-
-func TestPushJournald_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		cp, err := getCommonParams(r)
-		if err != nil {
-			t.Fatalf("cannot create commonParams: %s", err)
-		}
-
-		buf := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", buf, tlp, cp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// too short binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasd")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00_PID=2763\n\n")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890_PID=2763\n\n")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n_PID=2763\n\n")
-
-	// too long binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")
-}

app/vlinsert/journald/journald_timing_test.go

@@ -1,82 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func BenchmarkIsValidFieldName(b *testing.B) {
-	b.ReportAllocs()
-	b.SetBytes(int64(len(benchmarkFields)))
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			for _, field := range benchmarkFields {
-				if !isValidFieldName(field) {
-					panic(fmt.Errorf("cannot validate field %q", field))
-				}
-			}
-		}
-	})
-}
-
-var benchmarkFields = strings.Split(
-	"E,_BOOT_ID,_UID,_GID,_MACHINE_ID,_HOSTNAME,_RUNTIME_SCOPE,_TRANSPORT,_CAP_EFFECTIVE,_SYSTEMD_CGROUP,_SYSTEMD_UNIT,"+
-		"_SYSTEMD_SLICE,CODE_FILE,CODE_LINE,CODE_FUNC,SYSLOG_IDENTIFIER,_COMM,_EXE,_CMDLINE,MESSAGE,_PID,_SOURCE_REALTIME_TIMESTAMP,_REALTIME_TIMESTAMP",
-	",")
-
-func BenchmarkPushJournaldPerformance(b *testing.B) {
-	cp := &insertutil.CommonParams{
-		TimeFields: []string{"__REALTIME_TIMESTAMP"},
-		MsgFields:  []string{"MESSAGE"},
-	}
-	const dataChunkSize = 1024 * 1024
-
-	data := generateJournaldData(dataChunkSize)
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		blp := &insertutil.BenchmarkLogMessageProcessor{}
-		for pb.Next() {
-			r.Reset(data)
-			if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func generateJournaldData(size int) []byte {
-	var buf []byte
-	timestamp := time.Now().UnixMicro()
-	binaryMsg := []byte("binary message data for performance test")
-	var sizeBuf [8]byte
-
-	for len(buf) < size {
-		timestamp++
-
-		var entry string
-		// Generate a mix of simple and binary messages
-		if timestamp%10 == 0 {
-			// Generate binary message
-			binary.LittleEndian.PutUint64(sizeBuf[:], uint64(len(binaryMsg)))
-			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE\n%s%s\n\n",
-				timestamp,
-				sizeBuf[:],
-				binaryMsg,
-			)
-		} else {
-			// Generate simple message
-			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE=Performance test message %d\n\n", timestamp, timestamp)
-		}
-		buf = append(buf, entry...)
-	}
-	return buf
-}

app/vlinsert/jsonline/jsonline.go

@@ -1,123 +0,0 @@
-package jsonline
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// RequestHandler processes jsonline insert requests
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-	w.Header().Add("Content-Type", "application/json")
-
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
-	if err != nil {
-		logger.Errorf("cannot decode jsonline request: %s", err)
-		return
-	}
-	defer protoparserutil.PutUncompressedReader(reader)
-
-	lmp := cp.NewLogMessageProcessor("jsonline", true)
-	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, cp.TimeFields, cp.MsgFields, lmp)
-	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot process jsonline request; error: %s", err)
-		return
-	}
-
-	requestDuration.UpdateDuration(startTime)
-}
-
-func processStreamInternal(streamName string, r io.Reader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lr := insertutil.NewLineReader(streamName, wcr)
-
-	n := 0
-	errors := 0
-	var lastError error
-	for {
-		ok, err := readLine(lr, timeFields, msgFields, lmp)
-		wcr.DecConcurrency()
-		if err != nil {
-			lastError = err
-			errors++
-			logger.Warnf("jsonline: cannot read line #%d in /jsonline request: %s", n, err)
-		}
-		if !ok {
-			break
-		}
-		n++
-	}
-	errorsTotal.Add(errors)
-
-	if errors > 0 && n == errors {
-		// Return an error if no logs were processed and there were errors
-		return lastError
-	}
-
-	return nil
-}
-
-func readLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
-	var line []byte
-	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
-		}
-		line = lr.Line
-	}
-
-	p := logstorage.GetJSONParser()
-	defer logstorage.PutJSONParser(p)
-
-	if err := p.ParseLogMessage(line); err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
-	}
-	ts, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
-	if err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
-	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
-
-	return true, nil
-}
-
-var (
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
-
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
-)

app/vlinsert/jsonline/jsonline_test.go

@@ -1,97 +0,0 @@
-package jsonline
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestProcessStreamInternalSuccess(t *testing.T) {
-	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		timeFields := []string{timeField}
-		msgFields := []string{msgField}
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, timeFields, msgFields, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06 04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// Non-existing msgField
-	data = `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-`
-	timeField = "@timestamp"
-	msgField = "foobar"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
-{"message":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// invalid lines among valid lines
-	data = `
-dsfodmasd
-
-{"time":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-invalid line
-{"time":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-asbsdf
-
-`
-	timeField = "time"
-	msgField = "message"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternalFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := strings.NewReader(data)
-		if err := processStreamInternal("test", r, []string{"time"}, nil, tlp); err == nil {
-			t.Fatalf("expected error, got nil")
-		}
-
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	// invalid json
-	f("foobar")
-
-	f(`foo
-bar`)
-
-	f(`
-foo
-
-`)
-
-	// invalid timestamp field
-	f(`{"time":"foobar"}`)
-}

app/vlinsert/loki/loki.go

@@ -1,86 +0,0 @@
-package loki
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whether to disable automatic parsing of JSON-encoded log fields inside Loki log message into distinct log fields")
-
-// RequestHandler processes Loki insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/loki/api/v1/push":
-		handleInsert(r, w)
-		return true
-	case "/insert/loki/ready":
-		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("ready"))
-		return true
-	default:
-		return false
-	}
-}
-
-// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-func handleInsert(r *http.Request, w http.ResponseWriter) {
-	contentType := r.Header.Get("Content-Type")
-	switch contentType {
-	case "application/json":
-		handleJSON(r, w)
-	default:
-		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		handleProtobuf(r, w)
-	}
-}
-
-type commonParams struct {
-	cp *insertutil.CommonParams
-
-	// Whether to parse JSON inside plaintext log message.
-	//
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8486
-	parseMessage bool
-}
-
-func getCommonParams(r *http.Request) (*commonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// If parsed tenant is (0,0) it is likely to be default tenant
-	// Try parsing tenant from Loki headers
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		org := r.Header.Get("X-Scope-OrgID")
-		if org != "" {
-			tenantID, err := logstorage.ParseTenantID(org)
-			if err != nil {
-				return nil, err
-			}
-			cp.TenantID = tenantID
-		}
-
-	}
-
-	parseMessage := !*disableMessageParsing
-	if rv := httputil.GetRequestValue(r, "disable_message_parsing", "VL-Loki-Disable-Message-Parsing"); rv != "" {
-		bv, err := strconv.ParseBool(rv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse dusable_message_parsing=%q: %s", rv, err)
-		}
-		parseMessage = !bv
-	}
-
-	return &commonParams{
-		cp:           cp,
-		parseMessage: parseMessage,
-	}, nil
-}

app/vlinsert/loki/loki_json.go

@@ -1,225 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var maxRequestSize = flagutil.NewBytes("loki.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single Loki request")
-
-var parserPool fastjson.ParserPool
-
-func handleJSON(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsJSONTotal.Inc()
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_json", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseJSONRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki json data: %s", err)
-		return
-	}
-
-	// update requestJSONDuration only for successfully parsed requests
-	// There is no need in updating requestJSONDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestJSONDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
-}
-
-var (
-	requestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
-)
-
-func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-
-	streamsV := v.Get("streams")
-	if streamsV == nil {
-		return fmt.Errorf("missing `streams` item in the parsed JSON")
-	}
-	streams, err := streamsV.Array()
-	if err != nil {
-		return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
-	}
-
-	fields := getFields()
-	defer putFields(fields)
-
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
-	}
-
-	currentTimestamp := time.Now().UnixNano()
-
-	for _, stream := range streams {
-		// populate common labels from `stream` dict
-		fields.fields = fields.fields[:0]
-		labelsV := stream.Get("stream")
-		var labels *fastjson.Object
-		if labelsV != nil {
-			o, err := labelsV.Object()
-			if err != nil {
-				return fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
-			}
-			labels = o
-		}
-		labels.Visit(func(k []byte, v *fastjson.Value) {
-			vStr, errLocal := v.StringBytes()
-			if errLocal != nil {
-				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-				return
-			}
-			fields.fields = append(fields.fields, logstorage.Field{
-				Name:  bytesutil.ToUnsafeString(k),
-				Value: bytesutil.ToUnsafeString(vStr),
-			})
-		})
-		if err != nil {
-			return fmt.Errorf("error when parsing `stream` object: %w", err)
-		}
-
-		// populate messages from `values` array
-		linesV := stream.Get("values")
-		if linesV == nil {
-			return fmt.Errorf("missing `values` item in the parsed `stream` object %q", stream)
-		}
-		lines, err := linesV.Array()
-		if err != nil {
-			return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
-		}
-
-		commonFieldsLen := len(fields.fields)
-		for _, line := range lines {
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			lineA, err := line.Array()
-			if err != nil {
-				return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
-			}
-			if len(lineA) < 2 || len(lineA) > 3 {
-				return fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2 or 3", line, len(lineA))
-			}
-
-			// parse timestamp
-			timestamp, err := lineA[0].StringBytes()
-			if err != nil {
-				return fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
-			}
-			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
-			if err != nil {
-				return fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
-			}
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-			if len(lineA) > 2 {
-				structuredMetadata, err := lineA[2].Object()
-				if err != nil {
-					return fmt.Errorf("unexpected structured metadata type for %q; want JSON object", lineA[2])
-				}
-
-				structuredMetadata.Visit(func(k []byte, v *fastjson.Value) {
-					vStr, errLocal := v.StringBytes()
-					if errLocal != nil {
-						err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-						return
-					}
-
-					fields.fields = append(fields.fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(vStr),
-					})
-				})
-				if err != nil {
-					return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
-				}
-			}
-
-			// parse log message
-			msg, err := lineA[1].StringBytes()
-			if err != nil {
-				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
-			}
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, bytesutil.ToUnsafeString(msg))
-
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
-			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
-		}
-	}
-
-	return nil
-}
-
-func addMsgField(dst []logstorage.Field, msgParser *logstorage.JSONParser, msg string) ([]logstorage.Field, bool) {
-	if msgParser == nil || len(msg) < 2 || msg[0] != '{' || msg[len(msg)-1] != '}' {
-		return append(dst, logstorage.Field{
-			Name:  "_msg",
-			Value: msg,
-		}), false
-	}
-	if msgParser != nil && len(msg) >= 2 && msg[0] == '{' && msg[len(msg)-1] == '}' {
-		if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msg)); err == nil {
-			return append(dst, msgParser.Fields...), true
-		}
-	}
-	return append(dst, logstorage.Field{
-		Name:  "_msg",
-		Value: msg,
-	}), false
-}
-
-func parseLokiTimestamp(s string) (int64, error) {
-	if s == "" {
-		// Special case - an empty timestamp must be substituted with the current time by the caller.
-		return 0, nil
-	}
-	return insertutil.ParseUnixTimestamp(s)
-}

app/vlinsert/loki/loki_json_test.go

@@ -1,169 +0,0 @@
-package loki
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestParseJSONRequest_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-	f(``)
-
-	// Invalid json
-	f(`{}`)
-	f(`[]`)
-	f(`"foo"`)
-	f(`123`)
-
-	// invalid type for `streams` item
-	f(`{"streams":123}`)
-
-	// Missing `values` item
-	f(`{"streams":[{}]}`)
-
-	// Invalid type for `values` item
-	f(`{"streams":[{"values":"foobar"}]}`)
-
-	// Invalid type for `stream` item
-	f(`{"streams":[{"stream":[],"values":[]}]}`)
-
-	// Invalid type for `values` individual item
-	f(`{"streams":[{"values":[123]}]}`)
-
-	// Invalid length of `values` individual item
-	f(`{"streams":[{"values":[[]]}]}`)
-	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789","8123"]]}]}`)
-
-	// Invalid type for timestamp inside `values` individual item
-	f(`{"streams":[{"values":[[123,"456"]}]}`)
-
-	// Invalid type for log message
-	f(`{"streams":[{"values":[["123",1234]]}]}`)
-
-	// invalid structured metadata type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", ["metadata_1", "md_value"]]]}]}`)
-
-	// structured metadata with unexpected value type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": 1}]] }]}`)
-}
-
-func TestParseJSONRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1686026123.62", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1686026123620000000, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
-
-	// values with metadata
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"metadata_1":"md_value","_msg":"foo bar"}`)
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-}
-
-func TestParseJSONRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		if err := parseJSONRequest([]byte(s), tlp, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"111\",\"parent_id\":\"abc\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"111","parent_id":"abc"}`)
-}

app/vlinsert/loki/loki_json_timing_test.go

@@ -1,78 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func BenchmarkParseJSONRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseJSONRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		data := getJSONBody(streams, rows, labels)
-		for pb.Next() {
-			if err := parseJSONRequest(data, blp, nil, false, true); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getJSONBody(streams, rows, labels int) []byte {
-	body := append([]byte{}, `{"streams":[`...)
-	now := time.Now().UnixNano()
-	valuePrefix := fmt.Sprintf(`["%d","value_`, now)
-
-	for i := 0; i < streams; i++ {
-		body = append(body, `{"stream":{`...)
-
-		for j := 0; j < labels; j++ {
-			body = append(body, `"label_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `":"value_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, '"')
-			if j < labels-1 {
-				body = append(body, ',')
-			}
-
-		}
-		body = append(body, `}, "values":[`...)
-
-		for j := 0; j < rows; j++ {
-			body = append(body, valuePrefix...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `"]`...)
-			if j < rows-1 {
-				body = append(body, ',')
-			}
-		}
-
-		body = append(body, `]}`...)
-		if i < streams-1 {
-			body = append(body, ',')
-		}
-
-	}
-
-	body = append(body, `]}`...)
-
-	return body
-}

app/vlinsert/loki/loki_protobuf.go

@@ -1,218 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	pushReqsPool sync.Pool
-)
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	if encoding == "" {
-		// Loki protocol uses snappy compression by default.
-		// See https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-		encoding = "snappy"
-	}
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_protobuf", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseProtobufRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki protobuf data: %s", err)
-		return
-	}
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
-}
-
-var (
-	requestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
-)
-
-func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
-	req := getPushRequest()
-	defer putPushRequest(req)
-
-	err := req.UnmarshalProtobuf(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse request body: %w", err)
-	}
-
-	fields := getFields()
-	defer putFields(fields)
-
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
-	}
-
-	streams := req.Streams
-	currentTimestamp := time.Now().UnixNano()
-
-	for i := range streams {
-		stream := &streams[i]
-		// st.Labels contains labels for the stream.
-		// Labels are same for all entries in the stream.
-		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
-		if err != nil {
-			return fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
-		}
-		commonFieldsLen := len(fields.fields)
-
-		entries := stream.Entries
-		for j := range entries {
-			e := &entries[j]
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			for _, lp := range e.StructuredMetadata {
-				fields.fields = append(fields.fields, logstorage.Field{
-					Name:  lp.Name,
-					Value: lp.Value,
-				})
-			}
-
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, e.Line)
-
-			ts := e.Timestamp.UnixNano()
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
-			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
-		}
-	}
-	return nil
-}
-
-func getFields() *fields {
-	v := fieldsPool.Get()
-	if v == nil {
-		return &fields{}
-	}
-	return v.(*fields)
-}
-
-func putFields(f *fields) {
-	f.fields = f.fields[:0]
-	fieldsPool.Put(f)
-}
-
-var fieldsPool sync.Pool
-
-type fields struct {
-	fields []logstorage.Field
-}
-
-// parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
-//
-// See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
-func parsePromLabels(dst []logstorage.Field, s string) ([]logstorage.Field, error) {
-	// Make sure s is wrapped into `{...}`
-	s = strings.TrimSpace(s)
-	if len(s) < 2 {
-		return nil, fmt.Errorf("too short string to parse: %q", s)
-	}
-	if s[0] != '{' {
-		return nil, fmt.Errorf("missing `{` at the beginning of %q", s)
-	}
-	if s[len(s)-1] != '}' {
-		return nil, fmt.Errorf("missing `}` at the end of %q", s)
-	}
-	s = s[1 : len(s)-1]
-
-	for len(s) > 0 {
-		// Parse label name
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("cannot find `=` char for label value at %s", s)
-		}
-		name := s[:n]
-		s = s[n+1:]
-
-		// Parse label value
-		qs, err := strconv.QuotedPrefix(s)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse value for label %q at %s: %w", name, s, err)
-		}
-		s = s[len(qs):]
-		value, err := strconv.Unquote(qs)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unquote value %q for label %q: %w", qs, name, err)
-		}
-
-		// Append the found field to dst.
-		dst = append(dst, logstorage.Field{
-			Name:  name,
-			Value: value,
-		})
-
-		// Check whether there are other labels remaining
-		if len(s) == 0 {
-			break
-		}
-		if !strings.HasPrefix(s, ",") {
-			return nil, fmt.Errorf("missing `,` char at %s", s)
-		}
-		s = s[1:]
-		s = strings.TrimPrefix(s, " ")
-	}
-	return dst, nil
-}
-
-func getPushRequest() *PushRequest {
-	v := pushReqsPool.Get()
-	if v == nil {
-		return &PushRequest{}
-	}
-	return v.(*PushRequest)
-}
-
-func putPushRequest(req *PushRequest) {
-	req.reset()
-	pushReqsPool.Put(req)
-}

app/vlinsert/loki/loki_protobuf_test.go

@@ -1,218 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-type testLogMessageProcessor struct {
-	pr PushRequest
-}
-
-func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("unexpected non-nil streamFields: %v", streamFields))
-	}
-	msg := ""
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			msg = f.Value
-		}
-	}
-	var a []string
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			continue
-		}
-		item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-		a = append(a, item)
-	}
-	labels := "{" + strings.Join(a, ", ") + "}"
-	tlp.pr.Streams = append(tlp.pr.Streams, Stream{
-		Labels: labels,
-		Entries: []Entry{
-			{
-				Timestamp: time.Unix(0, timestamp),
-				Line:      strings.Clone(msg),
-			},
-		},
-	})
-}
-
-func (tlp *testLogMessageProcessor) MustClose() {
-}
-
-func TestParseProtobufRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
-		}
-
-		data := tlp.pr.MarshalProtobuf(nil)
-
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
-}
-
-func TestParseProtobufRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
-		}
-
-		data := tlp.pr.MarshalProtobuf(nil)
-
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"432\",\"parent_id\":\"qwerty\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"432","parent_id":"qwerty"}`)
-}
-
-func TestParsePromLabels_Success(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		var a []string
-		for _, f := range fields {
-			a = append(a, fmt.Sprintf("%s=%q", f.Name, f.Value))
-		}
-		result := "{" + strings.Join(a, ", ") + "}"
-		if result != s {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, s)
-		}
-	}
-
-	f("{}")
-	f(`{foo="bar"}`)
-	f(`{foo="bar", baz="x", y="z"}`)
-	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
-}
-
-func TestParsePromLabels_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(fields) > 0 {
-			t.Fatalf("unexpected non-empty fields: %s", fields)
-		}
-	}
-
-	f("")
-	f("{")
-	f(`{foo}`)
-	f(`{foo=bar}`)
-	f(`{foo="bar}`)
-	f(`{foo="ba\",r}`)
-	f(`{foo="bar" baz="aa"}`)
-	f(`foobar`)
-	f(`foo{bar="baz"}`)
-}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -1,80 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			if err := parseProtobufRequest(body, blp, nil, false, true); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
-	var b []byte
-	var entries []Entry
-	streams := make([]Stream, streamsCount)
-	for i := range streams {
-		b = b[:0]
-		b = append(b, '{')
-		for j := 0; j < labelsCount; j++ {
-			b = append(b, "label_"...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, `="value_`...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, '"')
-			if j < labelsCount-1 {
-				b = append(b, ',')
-			}
-		}
-		b = append(b, '}')
-		labels := string(b)
-
-		var rowsBuf []byte
-		entriesLen := len(entries)
-		for j := 0; j < rowsCount; j++ {
-			rowsBufLen := len(rowsBuf)
-			rowsBuf = append(rowsBuf, "value_"...)
-			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
-			entries = append(entries, Entry{
-				Timestamp: time.Now(),
-				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
-			})
-		}
-
-		st := &streams[i]
-		st.Labels = labels
-		st.Entries = entries[entriesLen:]
-	}
-	pr := PushRequest{
-		Streams: streams,
-	}
-
-	return pr.MarshalProtobuf(nil)
-}

app/vlinsert/loki/pb.go

@@ -1,302 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: push_request.proto
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package loki
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// PushRequest represents Loki PushRequest
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L14
-type PushRequest struct {
-	Streams []Stream
-
-	entriesBuf   []Entry
-	labelPairBuf []LabelPair
-}
-
-func (pr *PushRequest) reset() {
-	pr.Streams = pr.Streams[:0]
-
-	pr.entriesBuf = pr.entriesBuf[:0]
-	pr.labelPairBuf = pr.labelPairBuf[:0]
-}
-
-// UnmarshalProtobuf unmarshals pr from protobuf message at src.
-//
-// pr remains valid until src is modified.
-func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
-	pr.reset()
-	var err error
-	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
-	return err
-}
-
-// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
-func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	pr.marshalProtobuf(m.MessageMarshaler())
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, s := range pr.Streams {
-		s.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message PushRequest {
-	//   repeated Stream streams = 1;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
-			}
-			pr.Streams = append(pr.Streams, Stream{})
-			s := &pr.Streams[len(pr.Streams)-1]
-			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
-			}
-		}
-	}
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Stream represents Loki stream.
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L23
-type Stream struct {
-	Labels  string
-	Entries []Entry
-}
-
-func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, s.Labels)
-	for _, e := range s.Entries {
-		e.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message Stream {
-	//   string labels = 1;
-	//   repeated Entry entries = 2;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	entriesBufLen := len(entriesBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			labels, ok := fc.String()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
-			}
-			s.Labels = labels
-		case 2:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
-			}
-			entriesBuf = append(entriesBuf, Entry{})
-			e := &entriesBuf[len(entriesBuf)-1]
-			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
-			}
-		}
-	}
-	s.Entries = entriesBuf[entriesBufLen:]
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Entry represents Loki entry.
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L38
-type Entry struct {
-	Timestamp          time.Time
-	Line               string
-	StructuredMetadata []LabelPair
-}
-
-func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	marshalTime(mm, 1, e.Timestamp)
-	mm.AppendString(2, e.Line)
-	for _, lp := range e.StructuredMetadata {
-		lp.marshalProtobuf(mm.AppendMessage(3))
-	}
-}
-
-func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
-	// message Entry {
-	//   Timestamp timestamp = 1;
-	//   string line = 2;
-	//   repeated LabelPair structuredMetadata = 3;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	labelPairBufLen := len(labelPairBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
-			}
-			timestamp, err := unmarshalTime(data)
-			if err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
-			}
-			e.Timestamp = timestamp
-		case 2:
-			line, ok := fc.String()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Line")
-			}
-			e.Line = line
-		case 3:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
-			}
-			labelPairBuf = append(labelPairBuf, LabelPair{})
-			lp := &labelPairBuf[len(labelPairBuf)-1]
-			if err := lp.unmarshalProtobuf(data); err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
-			}
-		}
-	}
-	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
-	return labelPairBuf, nil
-}
-
-// LabelPair represents Loki label pair.
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L33
-type LabelPair struct {
-	Name  string
-	Value string
-}
-
-func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, lp.Name)
-	mm.AppendString(2, lp.Value)
-}
-
-func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
-	// message LabelPair {
-	//   string name = 1;
-	//   string value = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			name, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot read name")
-			}
-			lp.Name = name
-		case 2:
-			value, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot unmarshal value")
-			}
-			lp.Value = value
-		}
-	}
-	return nil
-}
-
-func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
-	nsecs := timestamp.UnixNano()
-	ts := Timestamp{
-		Seconds: nsecs / 1e9,
-		Nanos:   int32(nsecs % 1e9),
-	}
-	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
-}
-
-func unmarshalTime(src []byte) (time.Time, error) {
-	var ts Timestamp
-	if err := ts.unmarshalProtobuf(src); err != nil {
-		return time.Time{}, err
-	}
-	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	return timestamp, nil
-}
-
-// Timestamp is protobuf well-known timestamp type.
-type Timestamp struct {
-	Seconds int64
-	Nanos   int32
-}
-
-func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendInt64(1, ts.Seconds)
-	mm.AppendInt32(2, ts.Nanos)
-}
-
-func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
-	// message Timestamp {
-	//   int64 seconds = 1;
-	//   int32 nanos = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			seconds, ok := fc.Int64()
-			if !ok {
-				return fmt.Errorf("cannot read Seconds")
-			}
-			ts.Seconds = seconds
-		case 2:
-			nanos, ok := fc.Int32()
-			if !ok {
-				return fmt.Errorf("cannot read Nanos")
-			}
-			ts.Nanos = nanos
-		}
-	}
-	return nil
-}

app/vlinsert/main.go

@@ -1,88 +0,0 @@
-package vlinsert
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-)
-
-var (
-	disableInsert   = flag.Bool("insert.disable", false, "Whether to disable /insert/* HTTP endpoints")
-	disableInternal = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
-)
-
-// Init initializes vlinsert
-func Init() {
-	syslog.MustInit()
-}
-
-// Stop stops vlinsert
-func Stop() {
-	syslog.MustStop()
-}
-
-// RequestHandler handles insert requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
-
-	if strings.HasPrefix(path, "/insert/") {
-		if *disableInsert {
-			httpserver.Errorf(w, r, "requests to /insert/* are disabled with -insert.disable command-line flag")
-			return true
-		}
-
-		return insertHandler(w, r, path)
-	}
-
-	if path == "/internal/insert" {
-		if *disableInternal || *disableInsert {
-			httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable or -insert.disable command-line flag")
-			return true
-		}
-		internalinsert.RequestHandler(w, r)
-		return true
-	}
-
-	return false
-}
-
-func insertHandler(w http.ResponseWriter, r *http.Request, path string) bool {
-	switch path {
-	case "/insert/jsonline":
-		jsonline.RequestHandler(w, r)
-		return true
-	case "/insert/ready":
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(200)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	}
-	switch {
-	// some clients may omit trailing slash at elasticsearch protocol.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
-	case strings.HasPrefix(path, "/insert/elasticsearch"):
-		return elasticsearch.RequestHandler(path, w, r)
-
-	case strings.HasPrefix(path, "/insert/loki/"):
-		return loki.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/opentelemetry/"):
-		return opentelemetry.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/journald/"):
-		return journald.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/datadog/"):
-		return datadog.RequestHandler(path, w, r)
-	}
-
-	return false
-}

app/vlinsert/opentelemetry/opentelemetry.go

@@ -1,153 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var maxRequestSize = flagutil.NewBytes("opentelemetry.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single OpenTelemetry request")
-
-// RequestHandler processes Opentelemetry insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	// use the same path as opentelemetry collector
-	// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
-	case "/insert/opentelemetry/v1/logs":
-		if r.Header.Get("Content-Type") == "application/json" {
-			httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-			return true
-		}
-		handleProtobuf(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf", false)
-		useDefaultStreamFields := len(cp.StreamFields) == 0
-		err := pushProtobufRequest(data, lmp, cp.MsgFields, useDefaultStreamFields)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read OpenTelemetry protocol data: %s", err)
-		return
-	}
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-
-	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-)
-
-func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {
-	var req pb.ExportLogsServiceRequest
-	if err := req.UnmarshalProtobuf(data); err != nil {
-		errorsTotal.Inc()
-		return fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
-	}
-
-	var commonFields []logstorage.Field
-	for _, rl := range req.ResourceLogs {
-		commonFields = commonFields[:0]
-		commonFields = appendKeyValues(commonFields, rl.Resource.Attributes, "")
-		commonFieldsLen := len(commonFields)
-		for _, sc := range rl.ScopeLogs {
-			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, msgFields, useDefaultStreamFields)
-		}
-	}
-
-	return nil
-}
-
-func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) []logstorage.Field {
-	fields := commonFields
-	for _, lr := range sc.LogRecords {
-		fields = fields[:len(commonFields)]
-		if lr.Body.KeyValueList != nil {
-			fields = appendKeyValues(fields, lr.Body.KeyValueList.Values, "")
-			logstorage.RenameField(fields[len(commonFields):], msgFields, "_msg")
-		} else {
-			fields = append(fields, logstorage.Field{
-				Name:  "_msg",
-				Value: lr.Body.FormatString(true),
-			})
-		}
-		fields = appendKeyValues(fields, lr.Attributes, "")
-		if len(lr.TraceID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "trace_id",
-				Value: lr.TraceID,
-			})
-		}
-		if len(lr.SpanID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "span_id",
-				Value: lr.SpanID,
-			})
-		}
-		fields = append(fields, logstorage.Field{
-			Name:  "severity",
-			Value: lr.FormatSeverity(),
-		})
-
-		var streamFields []logstorage.Field
-		if useDefaultStreamFields {
-			streamFields = commonFields
-		}
-		lmp.AddRow(lr.ExtractTimestampNano(), fields, streamFields)
-	}
-	return fields
-}
-
-func appendKeyValues(fields []logstorage.Field, kvs []*pb.KeyValue, parentField string) []logstorage.Field {
-	for _, attr := range kvs {
-		fieldName := attr.Key
-		if parentField != "" {
-			fieldName = parentField + "." + fieldName
-		}
-
-		if attr.Value.KeyValueList != nil {
-			fields = appendKeyValues(fields, attr.Value.KeyValueList.Values, fieldName)
-		} else {
-			fields = append(fields, logstorage.Field{
-				Name:  fieldName,
-				Value: attr.Value.FormatString(true),
-			})
-		}
-	}
-	return fields
-}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -1,208 +0,0 @@
-package opentelemetry
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func TestPushProtoOk(t *testing.T) {
-	f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-		lr := pb.ExportLogsServiceRequest{
-			ResourceLogs: src,
-		}
-
-		pData := lr.MarshalProtobuf(nil)
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := pushProtobufRequest(pData, tlp, nil, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// single line without resource attributes
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234},
-		`{"_msg":"log-line-message","severity":"Trace"}`,
-	)
-
-	// severities mapping
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 13, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 24, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1234, 1234},
-		`{"_msg":"log-line-message","severity":"Trace"}
-{"_msg":"log-line-message","severity":"Warn"}
-{"_msg":"log-line-message","severity":"Fatal4"}`,
-	)
-
-	// multi-line with resource attributes
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 25, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
-	)
-
-	// multi-scope with resource attributes and multi-line
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
-						{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
-					},
-				},
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
-						{TraceID: "1234", SpanID: "45", ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
-						{TraceID: "4bf92f3577b34da6a3ce929d0e0e4736", SpanID: "00f067aa0ba902b7", ObservedTimeUnixNano: 3333, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 2345, 2346, 2347, 2348, 3333},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Debug"}
-{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-1","trace_id":"1234","span_id":"45","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-2","trace_id":"4bf92f3577b34da6a3ce929d0e0e4736","span_id":"00f067aa0ba902b7","severity":"Unspecified"}`,
-	)
-
-	// nested fields
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{
-							TimeUnixNano: 1234,
-							Body:         pb.AnyValue{StringValue: ptrTo("nested fields")},
-							Attributes: []*pb.KeyValue{
-								{Key: "error", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-									{
-										Key:   "type",
-										Value: &pb.AnyValue{StringValue: ptrTo("document_parsing_exception")},
-									},
-									{
-										Key:   "reason",
-										Value: &pb.AnyValue{StringValue: ptrTo("failed to parse field [_msg] of type [text]")},
-									},
-									{
-										Key: "caused_by",
-										Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-											{
-												Key:   "type",
-												Value: &pb.AnyValue{StringValue: ptrTo("x_content_parse_exception")},
-											},
-											{
-												Key:   "reason",
-												Value: &pb.AnyValue{StringValue: ptrTo("unexpected end-of-input in VALUE_STRING")},
-											},
-											{
-												Key: "caused_by",
-												Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-													{
-														Key:   "type",
-														Value: &pb.AnyValue{StringValue: ptrTo("json_e_o_f_exception")},
-													},
-													{
-														Key:   "reason",
-														Value: &pb.AnyValue{StringValue: ptrTo("eof")},
-													},
-												}}},
-											},
-										}}},
-									},
-								}}}},
-							},
-						},
-					},
-				},
-			},
-		},
-	}, []int64{1234},
-		`{"_msg":"nested fields","error.type":"document_parsing_exception","error.reason":"failed to parse field [_msg] of type [text]",`+
-			`"error.caused_by.type":"x_content_parse_exception","error.caused_by.reason":"unexpected end-of-input in VALUE_STRING",`+
-			`"error.caused_by.caused_by.type":"json_e_o_f_exception","error.caused_by.caused_by.reason":"eof","severity":"Unspecified"}`)
-}
-
-func ptrTo[T any](s T) *T {
-	return &s
-}

app/vlinsert/opentelemetry/opentelemetry_timing_test.go

@@ -1,78 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, scopes := range []int{1, 2} {
-		for _, rows := range []int{100, 1000} {
-			for _, attributes := range []int{5, 10} {
-				b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, scopes, rows, attributes)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			if err := pushProtobufRequest(body, blp, nil, false); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
-	msg := "12345678910"
-
-	attrValues := []*pb.AnyValue{
-		{StringValue: ptrTo("string-attribute")},
-		{IntValue: ptrTo[int64](12345)},
-		{DoubleValue: ptrTo(3.14)},
-	}
-	attrs := make([]*pb.KeyValue, attributesCount)
-	for j := 0; j < attributesCount; j++ {
-		attrs[j] = &pb.KeyValue{
-			Key:   fmt.Sprintf("key-%d", j),
-			Value: attrValues[j%3],
-		}
-	}
-	entries := make([]pb.LogRecord, rowsCount)
-	for j := 0; j < rowsCount; j++ {
-		entries[j] = pb.LogRecord{
-			TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
-		}
-	}
-	scopes := make([]pb.ScopeLogs, scopesCount)
-
-	for j := 0; j < scopesCount; j++ {
-		scopes[j] = pb.ScopeLogs{
-			LogRecords: entries,
-		}
-	}
-
-	pr := pb.ExportLogsServiceRequest{
-		ResourceLogs: []pb.ResourceLogs{
-			{
-				Resource: pb.Resource{
-					Attributes: attrs,
-				},
-				ScopeLogs: scopes,
-			},
-		},
-	}
-
-	return pr.MarshalProtobuf(nil)
-}

app/vlinsert/syslog/syslog.go

@@ -1,607 +0,0 @@
-package syslog
-
-import (
-	"bufio"
-	"crypto/tls"
-	"encoding/json"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"net"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
-		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	streamFieldsTCP = flagutil.NewArrayString("syslog.streamFields.tcp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-	streamFieldsUDP = flagutil.NewArrayString("syslog.streamFields.udp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-
-	ignoreFieldsTCP = flagutil.NewArrayString("syslog.ignoreFields.tcp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-	ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-
-	decolorizeFieldsTCP = flagutil.NewArrayString("syslog.decolorizeFields.tcp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-	decolorizeFieldsUDP = flagutil.NewArrayString("syslog.decolorizeFields.udp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-
-	extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-	extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-
-	tenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
-	tenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
-
-	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-	listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+
-		"The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
-		"Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
-		"The provided key file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+
-		"See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+
-		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+
-		"Supported values: TLS10, TLS11, TLS12, TLS13. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-
-	compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-	compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-
-	useLocalTimestampTCP = flagutil.NewArrayBool("syslog.useLocalTimestamp.tcp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
-	useLocalTimestampUDP = flagutil.NewArrayBool("syslog.useLocalTimestamp.udp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
-)
-
-// MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports
-//
-// This function must be called after flag.Parse().
-//
-// MustStop() must be called in order to free up resources occupied by the initialized syslog parser.
-func MustInit() {
-	if workersStopCh != nil {
-		logger.Panicf("BUG: MustInit() called twice without MustStop() call")
-	}
-	workersStopCh = make(chan struct{})
-
-	for argIdx, addr := range *listenAddrTCP {
-		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runTCPListener(addr, argIdx)
-			workersWG.Done()
-		}(addr, argIdx)
-	}
-
-	for argIdx, addr := range *listenAddrUDP {
-		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runUDPListener(addr, argIdx)
-			workersWG.Done()
-		}(addr, argIdx)
-	}
-
-	currentYear := time.Now().Year()
-	globalCurrentYear.Store(int64(currentYear))
-	workersWG.Add(1)
-	go func() {
-		ticker := time.NewTicker(time.Minute)
-		for {
-			select {
-			case <-workersStopCh:
-				ticker.Stop()
-				workersWG.Done()
-				return
-			case <-ticker.C:
-				currentYear := time.Now().Year()
-				globalCurrentYear.Store(int64(currentYear))
-			}
-		}
-	}()
-
-	if *syslogTimezone != "" {
-		tz, err := time.LoadLocation(*syslogTimezone)
-		if err != nil {
-			logger.Fatalf("cannot parse -syslog.timezone=%q: %s", *syslogTimezone, err)
-		}
-		globalTimezone = tz
-	} else {
-		globalTimezone = time.Local
-	}
-}
-
-var (
-	globalCurrentYear atomic.Int64
-	globalTimezone    *time.Location
-)
-
-var (
-	workersWG     sync.WaitGroup
-	workersStopCh chan struct{}
-)
-
-// MustStop stops syslog parser initialized via MustInit()
-func MustStop() {
-	close(workersStopCh)
-	workersWG.Wait()
-	workersStopCh = nil
-}
-
-func runUDPListener(addr string, argIdx int) {
-	ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr)
-	if err != nil {
-		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
-	}
-
-	tenantIDStr := tenantIDUDP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodUDP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "udp")
-
-	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
-
-	streamFieldsStr := streamFieldsUDP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.udp=%q for -syslog.listenAddr.udp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsUDP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	decolorizeFieldsStr := decolorizeFieldsUDP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.udp=%q for -syslog.listenAddr.udp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.udp=%q for -syslog.listenAddr.udp=%q: %s", extraFieldsStr, addr, err)
-	}
-
-	doneCh := make(chan struct{})
-	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
-		close(doneCh)
-	}()
-
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
-	<-workersStopCh
-	if err := ln.Close(); err != nil {
-		logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err)
-	}
-	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
-}
-
-func runTCPListener(addr string, argIdx int) {
-	var tlsConfig *tls.Config
-	if tlsEnable.GetOptionalArg(argIdx) {
-		certFile := tlsCertFile.GetOptionalArg(argIdx)
-		keyFile := tlsKeyFile.GetOptionalArg(argIdx)
-		tc, err := netutil.GetServerTLSConfig(certFile, keyFile, *tlsMinVersion, *tlsCipherSuites)
-		if err != nil {
-			logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s",
-				certFile, keyFile, *tlsMinVersion, *tlsCipherSuites, err)
-		}
-		tlsConfig = tc
-	}
-	ln, err := netutil.NewTCPListener("syslog", addr, false, tlsConfig)
-	if err != nil {
-		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
-	}
-
-	tenantIDStr := tenantIDTCP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodTCP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "tcp")
-
-	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
-
-	streamFieldsStr := streamFieldsTCP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsTCP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	decolorizeFieldsStr := decolorizeFieldsTCP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", extraFieldsStr, addr, err)
-	}
-
-	doneCh := make(chan struct{})
-	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
-		close(doneCh)
-	}()
-
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
-	<-workersStopCh
-	if err := ln.Close(); err != nil {
-		logger.Fatalf("syslog: cannot close TCP listener at %s: %s", addr, err)
-	}
-	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
-}
-
-func checkCompressMethod(compressMethod, addr, protocol string) {
-	switch compressMethod {
-	case "", "none", "zstd", "gzip", "deflate":
-		return
-	default:
-		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
-	}
-}
-
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
-	gomaxprocs := cgroup.AvailableCPUs()
-	var wg sync.WaitGroup
-	localAddr := ln.LocalAddr()
-	for i := 0; i < gomaxprocs; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
-			var bb bytesutil.ByteBuffer
-			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
-			for {
-				bb.Reset()
-				bb.B = bb.B[:cap(bb.B)]
-				n, remoteAddr, err := ln.ReadFrom(bb.B)
-				if err != nil {
-					udpErrorsTotal.Inc()
-					var ne net.Error
-					if errors.As(err, &ne) {
-						if ne.Temporary() {
-							logger.Errorf("syslog: temporary error when listening for UDP at %q: %s", localAddr, err)
-							time.Sleep(time.Second)
-							continue
-						}
-						if strings.Contains(err.Error(), "use of closed network connection") {
-							break
-						}
-					}
-					logger.Errorf("syslog: cannot read UDP data from %s at %s: %s", remoteAddr, localAddr, err)
-					continue
-				}
-				bb.B = bb.B[:n]
-				udpRequestsTotal.Inc()
-				if err := processStream("udp", bb.NewReader(), encoding, useLocalTimestamp, cp); err != nil {
-					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
-				}
-			}
-		}()
-	}
-	wg.Wait()
-}
-
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
-	var cm ingestserver.ConnsMap
-	cm.Init("syslog")
-
-	var wg sync.WaitGroup
-	addr := ln.Addr()
-	for {
-		c, err := ln.Accept()
-		if err != nil {
-			var ne net.Error
-			if errors.As(err, &ne) {
-				if ne.Temporary() {
-					logger.Errorf("syslog: temporary error when listening for TCP addr %q: %s", addr, err)
-					time.Sleep(time.Second)
-					continue
-				}
-				if strings.Contains(err.Error(), "use of closed network connection") {
-					break
-				}
-				logger.Fatalf("syslog: unrecoverable error when accepting TCP connections at %q: %s", addr, err)
-			}
-			logger.Fatalf("syslog: unexpected error when accepting TCP connections at %q: %s", addr, err)
-		}
-		if !cm.Add(c) {
-			_ = c.Close()
-			break
-		}
-
-		wg.Add(1)
-		go func() {
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
-			if err := processStream("tcp", c, encoding, useLocalTimestamp, cp); err != nil {
-				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
-			}
-
-			cm.Delete(c)
-			_ = c.Close()
-			wg.Done()
-		}()
-	}
-
-	cm.CloseAll(0)
-	wg.Wait()
-}
-
-// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutil.CommonParams) error {
-	if err := insertutil.CanWriteData(); err != nil {
-		return err
-	}
-
-	lmp := cp.NewLogMessageProcessor("syslog_"+protocol, true)
-	err := processStreamInternal(r, encoding, useLocalTimestamp, lmp)
-	lmp.MustClose()
-
-	return err
-}
-
-func processStreamInternal(r io.Reader, encoding string, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return fmt.Errorf("cannot decode syslog data: %w", err)
-	}
-	defer protoparserutil.PutUncompressedReader(reader)
-
-	return processUncompressedStream(reader, useLocalTimestamp, lmp)
-}
-
-func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	slr := getSyslogLineReader(wcr)
-	defer putSyslogLineReader(slr)
-
-	n := 0
-	for {
-		ok := slr.nextLine()
-		wcr.DecConcurrency()
-		if !ok {
-			break
-		}
-
-		currentYear := int(globalCurrentYear.Load())
-		err := processLine(slr.line, currentYear, globalTimezone, useLocalTimestamp, lmp)
-		if err != nil {
-			errorsTotal.Inc()
-			return fmt.Errorf("cannot read line #%d: %s", n, err)
-		}
-		n++
-	}
-	return slr.Error()
-}
-
-type syslogLineReader struct {
-	line []byte
-
-	br  *bufio.Reader
-	err error
-}
-
-func (slr *syslogLineReader) reset(r io.Reader) {
-	slr.line = slr.line[:0]
-	slr.br.Reset(r)
-	slr.err = nil
-}
-
-// Error returns the last error occurred in slr.
-func (slr *syslogLineReader) Error() error {
-	if slr.err == nil || slr.err == io.EOF {
-		return nil
-	}
-	return slr.err
-}
-
-// nextLine reads the next syslog line from slr and stores it at slr.line.
-//
-// false is returned if the next line cannot be read. Error() must be called in this case
-// in order to verify whether there is an error or just slr stream has been finished.
-func (slr *syslogLineReader) nextLine() bool {
-	if slr.err != nil {
-		return false
-	}
-
-again:
-	prefix, err := slr.br.ReadSlice(' ')
-	if err != nil {
-		if err != io.EOF {
-			slr.err = fmt.Errorf("cannot read message frame prefix: %w", err)
-			return false
-		}
-		if len(prefix) == 0 {
-			slr.err = err
-			return false
-		}
-	}
-	// skip empty lines
-	for len(prefix) > 0 && prefix[0] == '\n' {
-		prefix = prefix[1:]
-	}
-	if len(prefix) == 0 {
-		// An empty prefix or a prefix with empty lines - try reading yet another prefix.
-		goto again
-	}
-
-	if prefix[0] >= '0' && prefix[0] <= '9' {
-		// This is octet-counting method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#msgxfer
-		msgLenStr := bytesutil.ToUnsafeString(prefix[:len(prefix)-1])
-		msgLen, err := strconv.ParseUint(msgLenStr, 10, 64)
-		if err != nil {
-			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
-			return false
-		}
-		if maxMsgLen := insertutil.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
-			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
-			return false
-		}
-		slr.line = slicesutil.SetLength(slr.line, int(msgLen))
-		if _, err := io.ReadFull(slr.br, slr.line); err != nil {
-			slr.err = fmt.Errorf("cannot read message with size %d bytes: %w", msgLen, err)
-			return false
-		}
-		return true
-	}
-
-	// This is octet-stuffing method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#octet-stuffing-legacy
-	slr.line = append(slr.line[:0], prefix...)
-	for {
-		line, err := slr.br.ReadSlice('\n')
-		if err == nil {
-			slr.line = append(slr.line, line[:len(line)-1]...)
-			return true
-		}
-		if err == io.EOF {
-			slr.line = append(slr.line, line...)
-			return true
-		}
-		if err == bufio.ErrBufferFull {
-			slr.line = append(slr.line, line...)
-			continue
-		}
-		slr.err = fmt.Errorf("cannot read message in octet-stuffing method: %w", err)
-		return false
-	}
-}
-
-func getSyslogLineReader(r io.Reader) *syslogLineReader {
-	v := syslogLineReaderPool.Get()
-	if v == nil {
-		br := bufio.NewReaderSize(r, 64*1024)
-		return &syslogLineReader{
-			br: br,
-		}
-	}
-	slr := v.(*syslogLineReader)
-	slr.reset(r)
-	return slr
-}
-
-func putSyslogLineReader(slr *syslogLineReader) {
-	syslogLineReaderPool.Put(slr)
-}
-
-var syslogLineReaderPool sync.Pool
-
-func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	p := logstorage.GetSyslogParser(currentYear, timezone)
-	lineStr := bytesutil.ToUnsafeString(line)
-	p.Parse(lineStr)
-
-	var ts int64
-	if useLocalTimestamp {
-		ts = time.Now().UnixNano()
-	} else {
-		nsecs, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
-		if err != nil {
-			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
-		}
-		ts = nsecs
-	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
-	logstorage.PutSyslogParser(p)
-
-	return nil
-}
-
-var timeFields = []string{"timestamp"}
-var msgFields = []string{"message"}
-
-var (
-	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
-
-	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
-	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
-)
-
-func parseFieldsList(s string) ([]string, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var a []string
-	err := json.Unmarshal([]byte(s), &a)
-	return a, err
-}
-
-func parseExtraFields(s string) ([]logstorage.Field, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var m map[string]string
-	if err := json.Unmarshal([]byte(s), &m); err != nil {
-		return nil, err
-	}
-	fields := make([]logstorage.Field, 0, len(m))
-	for k, v := range m {
-		fields = append(fields, logstorage.Field{
-			Name:  k,
-			Value: v,
-		})
-	}
-	sort.Slice(fields, func(i, j int) bool {
-		return fields[i].Name < fields[j].Name
-	})
-	return fields, nil
-}

app/vlinsert/syslog/syslog_test.go

@@ -1,129 +0,0 @@
-package syslog
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestSyslogLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		var lines []string
-		for slr.nextLine() {
-			lines = append(lines, string(slr.line))
-		}
-		if err := slr.Error(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines read;\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", nil)
-	f("\n\n\n", nil)
-
-	f("foobar", []string{"foobar"})
-	f("foobar\n", []string{"foobar\n"})
-	f("\n\nfoo\n\nbar\n\n", []string{"foo\n\nbar\n\n"})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...`, []string{"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches..."})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
-
-`, []string{
-		"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...",
-		"<165>Jun  4 12:08:33 abcd systemd[345]: abc defg",
-		`<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`,
-	})
-}
-
-func TestSyslogLineReader_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		if slr.nextLine() {
-			t.Fatalf("expecting failure to read the first line")
-		}
-		if err := slr.Error(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 aa")
-	f("1233423432 abc")
-}
-
-func TestProcessStreamInternal_Success(t *testing.T) {
-	f := func(data string, currentYear int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		MustInit()
-		defer MustStop()
-
-		globalTimezone = time.UTC
-		globalCurrentYear.Store(int64(currentYear))
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	data := `Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
-`
-	currentYear := 2023
-	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
-	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
-	f(data, currentYear, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		MustInit()
-		defer MustStop()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 foo")
-	f("123456789 bar")
-}

app/vlogscli/Makefile

@@ -1,109 +0,0 @@
-# All these commands must run from repository root.
-
-vlogscli:
-	APP_NAME=vlogscli $(MAKE) app-local
-
-vlogscli-race:
-	APP_NAME=vlogscli RACE=-race $(MAKE) app-local
-
-vlogscli-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker
-
-vlogscli-pure-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-pure
-
-vlogscli-linux-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-amd64
-
-vlogscli-linux-arm-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm
-
-vlogscli-linux-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm64
-
-vlogscli-linux-ppc64le-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-ppc64le
-
-vlogscli-linux-386-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-386
-
-vlogscli-darwin-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-amd64
-
-vlogscli-darwin-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-arm64
-
-vlogscli-freebsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-freebsd-amd64
-
-vlogscli-openbsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-openbsd-amd64
-
-vlogscli-windows-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-windows-amd64
-
-package-vlogscli:
-	APP_NAME=vlogscli $(MAKE) package-via-docker
-
-package-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-pure
-
-package-vlogscli-amd64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-amd64
-
-package-vlogscli-arm:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm
-
-package-vlogscli-arm64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm64
-
-package-vlogscli-ppc64le:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-ppc64le
-
-package-vlogscli-386:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-386
-
-publish-vlogscli:
-	APP_NAME=vlogscli $(MAKE) publish-via-docker
-
-vlogscli-linux-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-ppc64le:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-s390x:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-loong64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-386:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-freebsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-openbsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-windows-amd64:
-	GOARCH=amd64 APP_NAME=vlogscli $(MAKE) app-local-windows-goarch
-
-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) app-local-pure
-
-run-vlogscli:
-	APP_NAME=vlogscli $(MAKE) run-via-docker

app/vlogscli/README.md

@@ -1,5 +1 @@
-# vlogscli
-
-Command-line utility for querying [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlogscli/deployment/Dockerfile

@@ -1,6 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-ENTRYPOINT ["/vlogscli-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vlogscli-prod

app/vlogscli/json_prettifier.go

@@ -1,245 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-type outputMode int
-
-const (
-	outputModeJSONMultiline  = outputMode(0)
-	outputModeJSONSingleline = outputMode(1)
-	outputModeLogfmt         = outputMode(2)
-	outputModeCompact        = outputMode(3)
-)
-
-func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
-	switch outputMode {
-	case outputModeJSONMultiline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, true)
-		}
-	case outputModeJSONSingleline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, false)
-		}
-	case outputModeLogfmt:
-		return writeLogfmtObject
-	case outputModeCompact:
-		return writeCompactObject
-	default:
-		panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
-	}
-}
-
-type jsonPrettifier struct {
-	r         io.ReadCloser
-	formatter func(w io.Writer, fields []logstorage.Field) error
-
-	d *json.Decoder
-
-	pr *io.PipeReader
-	pw *io.PipeWriter
-	bw *bufio.Writer
-
-	wg sync.WaitGroup
-}
-
-func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
-	d := json.NewDecoder(r)
-	pr, pw := io.Pipe()
-	bw := bufio.NewWriter(pw)
-
-	formatter := getOutputFormatter(outputMode)
-
-	jp := &jsonPrettifier{
-		r:         r,
-		formatter: formatter,
-
-		d: d,
-
-		pr: pr,
-		pw: pw,
-		bw: bw,
-	}
-
-	jp.wg.Add(1)
-	go func() {
-		defer jp.wg.Done()
-		err := jp.prettifyJSONLines()
-		jp.closePipesWithError(err)
-	}()
-
-	return jp
-}
-
-func (jp *jsonPrettifier) closePipesWithError(err error) {
-	_ = jp.pr.CloseWithError(err)
-	_ = jp.pw.CloseWithError(err)
-}
-
-func (jp *jsonPrettifier) prettifyJSONLines() error {
-	for jp.d.More() {
-		fields, err := readNextJSONObject(jp.d)
-		if err != nil {
-			return err
-		}
-		sort.Slice(fields, func(i, j int) bool {
-			return fields[i].Name < fields[j].Name
-		})
-		if err := jp.formatter(jp.bw, fields); err != nil {
-			return err
-		}
-
-		// Flush bw after every output line in order to show results as soon as they appear.
-		if err := jp.bw.Flush(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (jp *jsonPrettifier) Close() error {
-	jp.closePipesWithError(io.ErrUnexpectedEOF)
-	err := jp.r.Close()
-	jp.wg.Wait()
-	return err
-}
-
-func (jp *jsonPrettifier) Read(p []byte) (int, error) {
-	return jp.pr.Read(p)
-}
-
-func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
-	t, err := d.Token()
-	if err != nil {
-		return nil, fmt.Errorf("cannot read '{': %w", err)
-	}
-	delim, ok := t.(json.Delim)
-	if !ok || delim.String() != "{" {
-		return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
-	}
-
-	var fields []logstorage.Field
-	for {
-		// Read object key
-		t, err := d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object key or closing brace: %w", err)
-		}
-		delim, ok := t.(json.Delim)
-		if ok {
-			if delim.String() == "}" {
-				return fields, nil
-			}
-			return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
-		}
-		key, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object key: %v; want string or '}'", t)
-		}
-
-		// read object value
-		t, err = d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object value: %w", err)
-		}
-		value, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object value: %v; want string", t)
-		}
-
-		fields = append(fields, logstorage.Field{
-			Name:  key,
-			Value: value,
-		})
-	}
-}
-
-func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
-	data := logstorage.MarshalFieldsToLogfmt(nil, fields)
-	_, err := fmt.Fprintf(w, "%s\n", data)
-	return err
-}
-
-func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
-	if len(fields) == 1 {
-		// Just write field value as is without name
-		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
-		return err
-	}
-	if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
-		// Write _time\tfieldValue as is
-		if fields[0].Name == "_time" {
-			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)
-			return err
-		}
-		_, err := fmt.Fprintf(w, "%s\t%s\n", fields[1].Value, fields[0].Value)
-		return err
-	}
-
-	// Fall back to logfmt
-	return writeLogfmtObject(w, fields)
-}
-
-func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
-	if len(fields) == 0 {
-		fmt.Fprintf(w, "{}\n")
-		return nil
-	}
-
-	fmt.Fprintf(w, "{")
-	writeNewlineIfNeeded(w, isMultiline)
-	if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
-		return err
-	}
-	for _, f := range fields[1:] {
-		fmt.Fprintf(w, ",")
-		writeNewlineIfNeeded(w, isMultiline)
-		if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
-			return err
-		}
-	}
-	writeNewlineIfNeeded(w, isMultiline)
-	fmt.Fprintf(w, "}\n")
-	return nil
-}
-
-func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
-	if isMultiline {
-		fmt.Fprintf(w, "\n")
-	}
-}
-
-func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
-	key := getJSONString(f.Name)
-	value := getJSONString(f.Value)
-	if isMultiline {
-		_, err := fmt.Fprintf(w, "  %s: %s", key, value)
-		return err
-	}
-	_, err := fmt.Fprintf(w, "%s:%s", key, value)
-	return err
-}
-
-func getJSONString(s string) string {
-	data, err := json.Marshal(s)
-	if err != nil {
-		panic(fmt.Errorf("unexpected error when marshaling string to JSON: %w", err))
-	}
-	return jsonHTMLReplacer.Replace(string(data))
-}
-
-var jsonHTMLReplacer = strings.NewReplacer(
-	`\u003c`, "\u003c",
-	`\u003e`, "\u003e",
-	`\u0026`, "\u0026",
-)

app/vlogscli/less_wrapper.go

@@ -1,123 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-
-	"github.com/mattn/go-isatty"
-)
-
-func isTerminal() bool {
-	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
-}
-
-func readWithLess(r io.Reader, disableColors, wrapLongLines bool) error {
-	if !isTerminal() {
-		// Just write everything to stdout if no terminal is available.
-		_, err := io.Copy(os.Stdout, r)
-		if err != nil && !isErrPipe(err) {
-			return fmt.Errorf("error when forwarding data to stdout: %w", err)
-		}
-		if err := os.Stdout.Sync(); err != nil {
-			return fmt.Errorf("cannot sync data to stdout: %w", err)
-		}
-		return nil
-	}
-
-	pr, pw, err := os.Pipe()
-	if err != nil {
-		return fmt.Errorf("cannot create pipe: %w", err)
-	}
-	defer func() {
-		_ = pr.Close()
-		_ = pw.Close()
-	}()
-
-	// Ignore Ctrl+C in the current process, so 'less' could handle it properly
-	cancel := ignoreSignals(os.Interrupt)
-	defer cancel()
-
-	// Start 'less' process
-	path, err := exec.LookPath("less")
-	if err != nil {
-		return fmt.Errorf("cannot find 'less' command: %w", err)
-	}
-	opts := []string{"less", "-F", "-X"}
-	if !disableColors {
-		opts = append(opts, "-R")
-	}
-	if !wrapLongLines {
-		opts = append(opts, "-S")
-	}
-	p, err := os.StartProcess(path, opts, &os.ProcAttr{
-		Env:   append(os.Environ(), "LESSCHARSET=utf-8"),
-		Files: []*os.File{pr, os.Stdout, os.Stderr},
-	})
-	if err != nil {
-		return fmt.Errorf("cannot start 'less' process: %w", err)
-	}
-
-	// Close pr after 'less' finishes in a parallel goroutine
-	// in order to unblock forwarding data to stopped 'less' below.
-	waitch := make(chan *os.ProcessState)
-	go func() {
-		// Wait for 'less' process to finish.
-		ps, err := p.Wait()
-		if err != nil {
-			fatalf("unexpected error when waiting for 'less' process: %w", err)
-		}
-		_ = pr.Close()
-		waitch <- ps
-	}()
-
-	// Forward data from r to 'less'
-	_, err = io.Copy(pw, r)
-	_ = pw.Sync()
-	_ = pw.Close()
-
-	// Wait until 'less' finished
-	ps := <-waitch
-
-	// Verify 'less' status.
-	if !ps.Success() {
-		return fmt.Errorf("'less' finished with unexpected code %d", ps.ExitCode())
-	}
-
-	if err != nil && !isErrPipe(err) {
-		return fmt.Errorf("error when forwarding data to 'less': %w", err)
-	}
-
-	return nil
-}
-
-func isErrPipe(err error) bool {
-	return errors.Is(err, syscall.EPIPE) || errors.Is(err, io.ErrClosedPipe)
-}
-
-func ignoreSignals(sigs ...os.Signal) func() {
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, sigs...)
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for {
-			_, ok := <-ch
-			if !ok {
-				return
-			}
-		}
-	}()
-	return func() {
-		signal.Stop(ch)
-		close(ch)
-		wg.Wait()
-	}
-}

app/vlogscli/main.go

@@ -1,457 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strconv"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/ergochat/readline"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
-		"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs . See also -tail.url")
-	tailURL = flag.String("tail.url", "", "URL for live tailing queries to VictoriaLogs; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing ."+
-		"The url is automatically detected from -datasource.url by replacing /query with /tail at the end if -tail.url is empty")
-	historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
-	header      = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
-	accountID   = flag.Int("accountID", 0, "Account ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-	projectID   = flag.Int("projectID", 0, "Project ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-)
-
-const (
-	firstLinePrompt = ";> "
-	nextLinePrompt  = ""
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.InitNoLogFlags()
-
-	hes, err := parseHeaders(*header)
-	if err != nil {
-		fatalf("cannot parse -header command-line flag: %s", err)
-	}
-	headers = hes
-
-	incompleteLine := ""
-	cfg := &readline.Config{
-		Prompt:                 firstLinePrompt,
-		DisableAutoSaveHistory: true,
-		Listener: func(line []rune, pos int, _ rune) ([]rune, int, bool) {
-			incompleteLine = string(line)
-			return line, pos, false
-		},
-	}
-	rl, err := readline.NewFromConfig(cfg)
-	if err != nil {
-		fatalf("cannot initialize readline: %s", err)
-	}
-
-	fmt.Fprintf(rl, "sending queries to -datasource.url=%s\n", *datasourceURL)
-	fmt.Fprintf(rl, `type ? and press enter to see available commands`+"\n")
-	runReadlineLoop(rl, &incompleteLine)
-
-	if err := rl.Close(); err != nil {
-		fatalf("cannot close readline: %s", err)
-	}
-
-}
-
-func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
-	historyLines, err := loadFromHistory(*historyFile)
-	if err != nil {
-		fatalf("cannot load query history: %s", err)
-	}
-	for _, line := range historyLines {
-		if err := rl.SaveToHistory(line); err != nil {
-			fatalf("cannot initialize query history: %s", err)
-		}
-	}
-
-	outputMode := outputModeJSONMultiline
-	disableColors := true
-	wrapLongLines := false
-	s := ""
-	for {
-		line, err := rl.ReadLine()
-		if err != nil {
-			switch err {
-			case io.EOF:
-				if s != "" {
-					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode, disableColors, wrapLongLines)
-				}
-				return
-			case readline.ErrInterrupt:
-				if s == "" && *incompleteLine == "" {
-					fmt.Fprintf(rl, "interrupted\n")
-					os.Exit(128 + int(syscall.SIGINT))
-				}
-				// Default value for Ctrl+C - clear the prompt and store the incompletely entered line into history
-				s += *incompleteLine
-				historyLines = pushToHistory(rl, historyLines, s)
-				s = ""
-				rl.SetPrompt(firstLinePrompt)
-				continue
-			default:
-				fatalf("unexpected error in readline: %s", err)
-			}
-		}
-
-		s += line
-		if s == "" {
-			// Skip empty lines
-			continue
-		}
-
-		if isQuitCommand(s) {
-			fmt.Fprintf(rl, "bye!\n")
-			_ = pushToHistory(rl, historyLines, s)
-			return
-		}
-		if isHelpCommand(s) {
-			printCommandsHelp(rl)
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\s` {
-			fmt.Fprintf(rl, "singleline json output mode\n")
-			outputMode = outputModeJSONSingleline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\m` {
-			fmt.Fprintf(rl, "multiline json output mode\n")
-			outputMode = outputModeJSONMultiline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\c` {
-			fmt.Fprintf(rl, "compact output mode\n")
-			outputMode = outputModeCompact
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\logfmt` {
-			fmt.Fprintf(rl, "logfmt output mode\n")
-			outputMode = outputModeLogfmt
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\wrap_long_lines` {
-			if wrapLongLines {
-				wrapLongLines = false
-				fmt.Fprintf(rl, "wrapping of long lines is disabled\n")
-			} else {
-				wrapLongLines = true
-				fmt.Fprintf(rl, "wrapping of long lines is enabled\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\disable_colors` {
-			if !disableColors {
-				disableColors = true
-				fmt.Fprintf(rl, `disabled colors in compact output mode; enter \enable_colors for enabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\enable_colors` {
-			if disableColors {
-				disableColors = false
-				fmt.Fprintf(rl, `enabled colors in compact output mode; type \disable_colors for disabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if line != "" && !strings.HasSuffix(line, ";") {
-			// Assume the query is incomplete and allow the user finishing the query on the next line
-			s += "\n"
-			rl.SetPrompt(nextLinePrompt)
-			continue
-		}
-
-		// Execute the query
-		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode, disableColors, wrapLongLines)
-		cancel()
-
-		historyLines = pushToHistory(rl, historyLines, s)
-		s = ""
-		rl.SetPrompt(firstLinePrompt)
-	}
-}
-
-func pushToHistory(rl *readline.Instance, historyLines []string, s string) []string {
-	s = strings.TrimSpace(s)
-	if len(historyLines) == 0 || historyLines[len(historyLines)-1] != s {
-		historyLines = append(historyLines, s)
-		if len(historyLines) > 500 {
-			historyLines = historyLines[len(historyLines)-500:]
-		}
-		if err := saveToHistory(*historyFile, historyLines); err != nil {
-			fatalf("cannot save query history: %s", err)
-		}
-	}
-	if err := rl.SaveToHistory(s); err != nil {
-		fatalf("cannot update query history: %s", err)
-	}
-	return historyLines
-}
-
-func loadFromHistory(filePath string) ([]string, error) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	linesQuoted := strings.Split(string(data), "\n")
-	lines := make([]string, 0, len(linesQuoted))
-	i := 0
-	for _, lineQuoted := range linesQuoted {
-		i++
-		if lineQuoted == "" {
-			continue
-		}
-		line, err := strconv.Unquote(lineQuoted)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse line #%d at %s: %w; line: [%s]", i, filePath, err, line)
-		}
-		lines = append(lines, line)
-	}
-	return lines, nil
-}
-
-func saveToHistory(filePath string, lines []string) error {
-	linesQuoted := make([]string, len(lines))
-	for i, line := range lines {
-		lineQuoted := strconv.Quote(line)
-		linesQuoted[i] = lineQuoted
-	}
-	data := strings.Join(linesQuoted, "\n")
-	return os.WriteFile(filePath, []byte(data), 0600)
-}
-
-func isQuitCommand(s string) bool {
-	switch s {
-	case `\q`, "q", "quit", "exit":
-		return true
-	default:
-		return false
-	}
-}
-
-func isHelpCommand(s string) bool {
-	switch s {
-	case `\h`, "h", "help", "?":
-		return true
-	default:
-		return false
-	}
-}
-
-func printCommandsHelp(w io.Writer) {
-	fmt.Fprintf(w, "%s", `Available commands:
-\q - quit
-\h - show this help
-\s - singleline json output mode
-\m - multiline json output mode
-\c - compact output mode
-\logfmt - logfmt output mode
-\wrap_long_lines - toggles wrapping long lines
-\enable_colors - enable ANSI colors in compact output mode
-\disable_colors - disable ANSI colors in compact output mode
-\tail <query> - live tail <query> results
-
-See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
-`)
-}
-
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, disableColors, wrapLongLines bool) {
-	if strings.HasPrefix(qStr, `\tail `) {
-		tailQuery(ctx, output, qStr, outputMode)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, *datasourceURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if err := readWithLess(respBody, disableColors, wrapLongLines); err != nil {
-		fmt.Fprintf(output, "error when reading query response: %s\n", err)
-		return
-	}
-}
-
-func tailQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
-	qStr = strings.TrimPrefix(qStr, `\tail `)
-	qURL, err := getTailURL()
-	if err != nil {
-		fmt.Fprintf(output, "%s\n", err)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, qURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if _, err := io.Copy(output, respBody); err != nil {
-		if !errors.Is(err, context.Canceled) && !isErrPipe(err) {
-			fmt.Fprintf(output, "error when live tailing query response: %s\n", err)
-		}
-		fmt.Fprintf(output, "\n")
-		return
-	}
-}
-
-func getTailURL() (string, error) {
-	if *tailURL != "" {
-		return *tailURL, nil
-	}
-
-	u, err := url.Parse(*datasourceURL)
-	if err != nil {
-		return "", fmt.Errorf("cannot parse -datasource.url=%q: %w", *datasourceURL, err)
-	}
-	if !strings.HasSuffix(u.Path, "/query") {
-		return "", fmt.Errorf("cannot find /query suffix in -datasource.url=%q", *datasourceURL)
-	}
-	u.Path = u.Path[:len(u.Path)-len("/query")] + "/tail"
-	return u.String(), nil
-}
-
-func getQueryResponse(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, qURL string) io.ReadCloser {
-	// Parse the query and convert it to canonical view.
-	qStr = strings.TrimSuffix(qStr, ";")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		fmt.Fprintf(output, "cannot parse query: %s\n", err)
-		return nil
-	}
-	qStr = q.String()
-	fmt.Fprintf(output, "executing [%s]...", qStr)
-
-	// Prepare HTTP request for qURL
-	args := make(url.Values)
-	args.Set("query", qStr)
-	data := strings.NewReader(args.Encode())
-
-	req, err := http.NewRequestWithContext(ctx, "POST", qURL, data)
-	if err != nil {
-		panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	for _, h := range headers {
-		req.Header.Set(h.Name, h.Value)
-	}
-	req.Header.Set("AccountID", strconv.Itoa(*accountID))
-	req.Header.Set("ProjectID", strconv.Itoa(*projectID))
-
-	// Execute HTTP request at qURL
-	startTime := time.Now()
-	resp, err := httpClient.Do(req)
-	queryDuration := time.Since(startTime)
-	fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
-	if err != nil {
-		if errors.Is(err, context.Canceled) {
-			fmt.Fprintf(output, "\n")
-		} else {
-			fmt.Fprintf(output, "cannot execute query: %s\n", err)
-		}
-		return nil
-	}
-
-	// Verify response code
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			body = []byte(fmt.Sprintf("cannot read response body: %s", err))
-		}
-		fmt.Fprintf(output, "unexpected status code: %d; response body:\n%s\n", resp.StatusCode, body)
-		return nil
-	}
-
-	// Prettify the response body
-	jp := newJSONPrettifier(resp.Body, outputMode)
-
-	return jp
-}
-
-var httpClient = &http.Client{}
-
-var headers []headerEntry
-
-type headerEntry struct {
-	Name  string
-	Value string
-}
-
-func parseHeaders(a []string) ([]headerEntry, error) {
-	hes := make([]headerEntry, len(a))
-	for i, s := range a {
-		a := strings.SplitN(s, ":", 2)
-		if len(a) != 2 {
-			return nil, fmt.Errorf("cannot parse header=%q; it must contain at least one ':'; for example, 'Cookie: foo'", s)
-		}
-		hes[i] = headerEntry{
-			Name:  strings.TrimSpace(a[0]),
-			Value: strings.TrimSpace(a[1]),
-		}
-	}
-	return hes, nil
-}
-
-func fatalf(format string, args ...any) {
-	fmt.Fprintf(os.Stderr, format+"\n", args...)
-	os.Exit(1)
-}
-
-func usage() {
-	const s = `
-vlogscli is a command-line tool for querying VictoriaLogs.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/querying/vlogscli/
-`
-	flagutil.Usage(s)
-}

app/vlogscli/multiarch/Dockerfile

@@ -1,11 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-ENTRYPOINT ["/vlogscli-prod"]
-ARG TARGETARCH
-COPY vlogscli-linux-${TARGETARCH}-prod ./vlogscli-prod

app/vlogsgenerator/Makefile

@@ -1,7 +0,0 @@
-# All these commands must run from repository root.
-
-vlogsgenerator:
-	APP_NAME=vlogsgenerator $(MAKE) app-local
-
-vlogsgenerator-race:
-	APP_NAME=vlogsgenerator RACE=-race $(MAKE) app-local

app/vlogsgenerator/README.md

@@ -1,158 +1 @@
-# vlogsgenerator
-
-Logs generator for [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-## How to build vlogsgenerator?
-
-Run `make vlogsgenerator` from the repository root. This builds `bin/vlogsgenerator` binary.
-
-## How run vlogsgenerator?
-
-`vlogsgenerator` generates logs in [JSON line format](https://jsonlines.org/) suitable for the ingestion
-via [`/insert/jsonline` endpoint at VictoriaLogs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api).
-
-By default it writes the generated logs into `stdout`. For example, the following command writes generated logs to `stdout`:
-
-```
-bin/vlogsgenerator
-```
-
-It is possible to redirect the generated logs to file. For example, the following command writes the generated logs to `logs.json` file:
-
-```
-bin/vlogsgenerator > logs.json
-```
-
-The generated logs at `logs.json` file can be inspected with the following command:
-
-```
-head logs.json | jq .
-```
-
-Below is an example output:
-
-```json
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 8 and worker 0; ip=185.69.136.129; uuid=b4fe8f1a-c93c-dea3-ba11-5b9f0509291e; u64=8996587920687045253",
-  "host": "host_8",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 8",
-  "const_1": "some value 1 8",
-  "const_2": "some value 2 8",
-  "var_0": "some value 0 12752539384823438260",
-  "dict_0": "warn",
-  "dict_1": "info",
-  "u8_0": "6",
-  "u16_0": "35202",
-  "u32_0": "1964973739",
-  "u64_0": "4810489083243239145",
-  "float_0": "1.868",
-  "ip_0": "250.34.75.125",
-  "timestamp_0": "1799-03-16T01:34:18.311Z",
-  "json_0": "{\"foo\":\"bar_3\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":32}"
-}
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 9 and worker 0; ip=164.244.254.194; uuid=7e8373b1-ce0d-1ce7-8e96-4bcab8955598; u64=13949903463741076522",
-  "host": "host_9",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 9",
-  "const_1": "some value 1 9",
-  "const_2": "some value 2 9",
-  "var_0": "some value 0 5371555382075206134",
-  "dict_0": "INFO",
-  "dict_1": "FATAL",
-  "u8_0": "219",
-  "u16_0": "31459",
-  "u32_0": "3918836777",
-  "u64_0": "6593354256620219850",
-  "float_0": "1.085",
-  "ip_0": "253.151.88.158",
-  "timestamp_0": "2042-10-05T16:42:57.082Z",
-  "json_0": "{\"foo\":\"bar_5\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":27}"
-}
-```
-
-The `run_id` field uniquely identifies every `vlogsgenerator` invocation.
-
-### How to write logs to VictoriaLogs?
-
-The generated logs can be written directly to VictoriaLogs by passing the address of [`/insert/jsonline` endpoint](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api)
-to `-addr` command-line flag. For example, the following command writes the generated logs to VictoriaLogs running at `localhost`:
-
-```
-bin/vlogsgenerator -addr=http://localhost:9428/insert/jsonline
-```
-
-### Configuration
-
-`vlogsgenerator` accepts various command-line flags, which can be used for configuring the number and the shape of the generated logs.
-These flags can be inspected by running `vlogsgenerator -help`. Below are the most interesting flags:
-
-* `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to generate.
-* `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
-
-The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
-
-For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
-[log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields), where every logs stream contains `10_000` log entries,
-and writes them to `http://localhost:9428/insert/jsonline`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-### Churn rate
-
-It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
-by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
-logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
-but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -totalStreams=1_000 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-In this case the total number of generated logs equals to `-totalStreams` * `-logsPerStream` = `10_000_000`.
-
-### Benchmark tuning
-
-By default `vlogsgenerator` generates and writes logs by a single worker. This may limit the maximum data ingestion rate during benchmarks.
-The number of workers can be changed via `-workers` command-line flag. For example, the following command generates and writes logs with `16` workers:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -workers=16
-```
-
-### Output statistics
-
-Every 10 seconds `vlogsgenerator` writes statistics about the generated logs into `stderr`. The frequency of the generated statistics can be adjusted via `-statInterval` command-line flag.
-For example, the following command writes statistics every 2 seconds:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -statInterval=2s
-```
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlogsgenerator/main.go

@@ -1,354 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"math"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"os"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
-	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
-
-	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
-	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
-	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
-	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
-		"during data generation")
-	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
-	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constant values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	i64FieldsPerLog = flag.Int("i64FieldsPerLog", 1, "The number of fields with int64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-
-	statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	var remoteWriteURL *url.URL
-	if *addr != "stdout" {
-		urlParsed, err := url.Parse(*addr)
-		if err != nil {
-			logger.Fatalf("cannot parse -addr=%q: %s", *addr, err)
-		}
-		qs, err := url.ParseQuery(urlParsed.RawQuery)
-		if err != nil {
-			logger.Fatalf("cannot parse query string in -addr=%q: %w", *addr, err)
-		}
-		qs.Set("_stream_fields", "host,worker_id")
-		urlParsed.RawQuery = qs.Encode()
-		remoteWriteURL = urlParsed
-	}
-
-	if start.nsec >= end.nsec {
-		logger.Fatalf("-start=%s must be smaller than -end=%s", start, end)
-	}
-	if *activeStreams <= 0 {
-		logger.Fatalf("-activeStreams must be bigger than 0; got %d", *activeStreams)
-	}
-	if *logsPerStream <= 0 {
-		logger.Fatalf("-logsPerStream must be bigger than 0; got %d", *logsPerStream)
-	}
-	if *totalStreams < *activeStreams {
-		*totalStreams = *activeStreams
-	}
-
-	cfg := &workerConfig{
-		url:           remoteWriteURL,
-		activeStreams: *activeStreams,
-		totalStreams:  *totalStreams,
-	}
-
-	// divide total and active streams among workers
-	if *workers <= 0 {
-		logger.Fatalf("-workers must be bigger than 0; got %d", *workers)
-	}
-	if *workers > *activeStreams {
-		logger.Fatalf("-workers=%d cannot exceed -activeStreams=%d", *workers, *activeStreams)
-	}
-	cfg.activeStreams /= *workers
-	cfg.totalStreams /= *workers
-
-	logger.Infof("start -workers=%d workers for ingesting -logsPerStream=%d log entries per each -totalStreams=%d (-activeStreams=%d) on a time range -start=%s, -end=%s to -addr=%s",
-		*workers, *logsPerStream, *totalStreams, *activeStreams, toRFC3339(start.nsec), toRFC3339(end.nsec), *addr)
-
-	startTime := time.Now()
-	var wg sync.WaitGroup
-	for i := 0; i < *workers; i++ {
-		wg.Add(1)
-		go func(workerID int) {
-			defer wg.Done()
-			generateAndPushLogs(cfg, workerID)
-		}(i)
-	}
-
-	go func() {
-		prevEntries := uint64(0)
-		prevBytes := uint64(0)
-		ticker := time.NewTicker(*statInterval)
-		for range ticker.C {
-			currEntries := logEntriesCount.Load()
-			deltaEntries := currEntries - prevEntries
-			rateEntries := float64(deltaEntries) / statInterval.Seconds()
-
-			currBytes := bytesGenerated.Load()
-			deltaBytes := currBytes - prevBytes
-			rateBytes := float64(deltaBytes) / statInterval.Seconds()
-			logger.Infof("generated %dK log entries (%dK total) at %.0fK entries/sec, %dMB (%dMB total) at %.0fMB/sec",
-				deltaEntries/1e3, currEntries/1e3, rateEntries/1e3, deltaBytes/1e6, currBytes/1e6, rateBytes/1e6)
-
-			prevEntries = currEntries
-			prevBytes = currBytes
-		}
-	}()
-
-	wg.Wait()
-
-	dSecs := time.Since(startTime).Seconds()
-	currEntries := logEntriesCount.Load()
-	currBytes := bytesGenerated.Load()
-	rateEntries := float64(currEntries) / dSecs
-	rateBytes := float64(currBytes) / dSecs
-	logger.Infof("ingested %dK log entries (%dMB) in %.3f seconds; avg ingestion rate: %.0fK entries/sec, %.0fMB/sec", currEntries/1e3, currBytes/1e6, dSecs, rateEntries/1e3, rateBytes/1e6)
-}
-
-var logEntriesCount atomic.Uint64
-
-var bytesGenerated atomic.Uint64
-
-type workerConfig struct {
-	url           *url.URL
-	activeStreams int
-	totalStreams  int
-}
-
-type statWriter struct {
-	w io.Writer
-}
-
-func (sw *statWriter) Write(p []byte) (int, error) {
-	bytesGenerated.Add(uint64(len(p)))
-	return sw.w.Write(p)
-}
-
-func generateAndPushLogs(cfg *workerConfig, workerID int) {
-	pr, pw := io.Pipe()
-	sw := &statWriter{
-		w: pw,
-	}
-
-	// The 1MB write buffer increases data ingestion performance by reducing the number of send() syscalls
-	bw := bufio.NewWriterSize(sw, 1024*1024)
-
-	doneCh := make(chan struct{})
-	go func() {
-		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
-		_ = bw.Flush()
-		_ = pw.Close()
-		close(doneCh)
-	}()
-
-	if cfg.url == nil {
-		_, err := io.Copy(os.Stdout, pr)
-		if err != nil {
-			logger.Fatalf("unexpected error when writing logs to stdout: %s", err)
-		}
-		return
-	}
-
-	req, err := http.NewRequest("POST", cfg.url.String(), pr)
-	if err != nil {
-		logger.Fatalf("cannot create request to %q: %s", cfg.url, err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 != 2 {
-		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
-	}
-
-	// Wait until all the generateLogs goroutine is finished.
-	<-doneCh
-}
-
-func generateLogs(bw *bufio.Writer, workerID, activeStreams, totalStreams int) {
-	streamLifetime := int64(float64(end.nsec-start.nsec) * (float64(activeStreams) / float64(totalStreams)))
-	streamStep := int64(float64(end.nsec-start.nsec) / float64(totalStreams-activeStreams+1))
-	step := streamLifetime / (*logsPerStream - 1)
-
-	currNsec := start.nsec
-	for currNsec < end.nsec {
-		firstStreamID := int((currNsec - start.nsec) / streamStep)
-		generateLogsAtTimestamp(bw, workerID, currNsec, firstStreamID, activeStreams)
-		currNsec += step
-	}
-}
-
-var runID = toUUID(rand.Uint64(), rand.Uint64())
-
-func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStreamID, activeStreams int) {
-	streamID := firstStreamID
-	timeStr := toRFC3339(ts)
-	for i := 0; i < activeStreams; i++ {
-		ip := toIPv4(rand.Uint32())
-		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
-			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
-		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
-		for j := 0; j < *constFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"const_%d":"some value %d %d"`, j, j, streamID)
-		}
-		for j := 0; j < *varFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"var_%d":"some value %d %d"`, j, j, rand.Uint64())
-		}
-		for j := 0; j < *dictFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"dict_%d":"%s"`, j, dictValues[rand.Intn(len(dictValues))])
-		}
-		for j := 0; j < *u8FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u8_%d":"%d"`, j, uint8(rand.Uint32()))
-		}
-		for j := 0; j < *u16FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u16_%d":"%d"`, j, uint16(rand.Uint32()))
-		}
-		for j := 0; j < *u32FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u32_%d":"%d"`, j, rand.Uint32())
-		}
-		for j := 0; j < *u64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
-		}
-		for j := 0; j < *i64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"i64_%d":"%d"`, j, int64(rand.Uint64()))
-		}
-		for j := 0; j < *floatFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
-		}
-		for j := 0; j < *ipFieldsPerLog; j++ {
-			ip := toIPv4(rand.Uint32())
-			fmt.Fprintf(bw, `,"ip_%d":"%s"`, j, ip)
-		}
-		for j := 0; j < *timestampFieldsPerLog; j++ {
-			timestamp := toISO8601(int64(rand.Uint64()))
-			fmt.Fprintf(bw, `,"timestamp_%d":"%s"`, j, timestamp)
-		}
-		for j := 0; j < *jsonFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"json_%d":"{\"foo\":\"bar_%d\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":%d}"`, j, rand.Intn(10), rand.Intn(100))
-		}
-		fmt.Fprintf(bw, "}\n")
-
-		logEntriesCount.Add(1)
-		streamID++
-	}
-}
-
-var dictValues = []string{
-	"debug",
-	"info",
-	"warn",
-	"error",
-	"fatal",
-	"ERROR",
-	"FATAL",
-	"INFO",
-}
-
-func newTimeFlag(name, defaultValue, description string) *timeFlag {
-	var tf timeFlag
-	if err := tf.Set(defaultValue); err != nil {
-		logger.Panicf("invalid defaultValue=%q for flag %q: %w", defaultValue, name, err)
-	}
-	flag.Var(&tf, name, description)
-	return &tf
-}
-
-type timeFlag struct {
-	s    string
-	nsec int64
-}
-
-func (tf *timeFlag) Set(s string) error {
-	msec, err := timeutil.ParseTimeMsec(s)
-	if err != nil {
-		return fmt.Errorf("cannot parse time from %q: %w", s, err)
-	}
-	tf.s = s
-	tf.nsec = msec * 1e6
-	return nil
-}
-
-func (tf *timeFlag) String() string {
-	return tf.s
-}
-
-func toRFC3339(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format(time.RFC3339Nano)
-}
-
-func toISO8601(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format("2006-01-02T15:04:05.000Z")
-}
-
-func toIPv4(n uint32) string {
-	dst := make([]byte, 0, len("255.255.255.255"))
-	dst = marshalUint64(dst, uint64(n>>24))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>16)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>8)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64(n&0xff))
-	return string(dst)
-}
-
-func toUUID(a, b uint64) string {
-	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", a&(1<<32-1), (a>>32)&(1<<16-1), (a >> 48), b&(1<<16-1), b>>16)
-}
-
-// marshalUint64 appends string representation of n to dst and returns the result.
-func marshalUint64(dst []byte, n uint64) []byte {
-	return strconv.AppendUint(dst, n, 10)
-}

app/vlselect/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlselect/internalselect/internalselect.go

@@ -1,316 +0,0 @@
-package internalselect
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-)
-
-// RequestHandler processes requests to /internal/select/*
-func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-
-	path := r.URL.Path
-	rh := requestHandlers[path]
-	if rh == nil {
-		httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
-		return
-	}
-
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
-	if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		// The return is skipped intentionally in order to track the duration of failed queries.
-	}
-	metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
-}
-
-var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
-	"/internal/select/query":               processQueryRequest,
-	"/internal/select/field_names":         processFieldNamesRequest,
-	"/internal/select/field_values":        processFieldValuesRequest,
-	"/internal/select/stream_field_names":  processStreamFieldNamesRequest,
-	"/internal/select/stream_field_values": processStreamFieldValuesRequest,
-	"/internal/select/streams":             processStreamsRequest,
-	"/internal/select/stream_ids":          processStreamIDsRequest,
-}
-
-func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	var wLock sync.Mutex
-	var dataLenBuf []byte
-
-	sendBuf := func(bb *bytesutil.ByteBuffer) error {
-		if len(bb.B) == 0 {
-			return nil
-		}
-
-		data := bb.B
-		if !cp.DisableCompression {
-			bufLen := len(bb.B)
-			bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
-			data = bb.B[bufLen:]
-		}
-
-		wLock.Lock()
-		dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
-		_, err := w.Write(dataLenBuf)
-		if err == nil {
-			_, err = w.Write(data)
-		}
-		wLock.Unlock()
-
-		// Reset the sent buf
-		bb.Reset()
-
-		return err
-	}
-
-	var bufs atomicutil.Slice[bytesutil.ByteBuffer]
-
-	var errGlobalLock sync.Mutex
-	var errGlobal error
-
-	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
-		if errGlobal != nil {
-			return
-		}
-
-		bb := bufs.Get(workerID)
-
-		bb.B = db.Marshal(bb.B)
-
-		if len(bb.B) < 1024*1024 {
-			// Fast path - the bb is too small to be sent to the client yet.
-			return
-		}
-
-		// Slow path - the bb must be sent to the client.
-		if err := sendBuf(bb); err != nil {
-			errGlobalLock.Lock()
-			if errGlobal != nil {
-				errGlobal = err
-			}
-			errGlobalLock.Unlock()
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
-		return err
-	}
-	if errGlobal != nil {
-		return errGlobal
-	}
-
-	// Send the remaining data
-	for _, bb := range bufs.All() {
-		if err := sendBuf(bb); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streams, cp.DisableCompression)
-}
-
-func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
-}
-
-type commonParams struct {
-	TenantIDs []logstorage.TenantID
-	Query     *logstorage.Query
-
-	DisableCompression bool
-}
-
-func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
-	version := r.FormValue("version")
-	if version != expectedProtocolVersion {
-		return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
-	}
-
-	tenantIDsStr := r.FormValue("tenant_ids")
-	tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
-	}
-
-	timestamp, err := getInt64FromRequest(r, "timestamp")
-	if err != nil {
-		return nil, err
-	}
-
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
-	}
-
-	s := r.FormValue("disable_compression")
-	disableCompression, err := strconv.ParseBool(s)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
-	}
-
-	cp := &commonParams{
-		TenantIDs: tenantIDs,
-		Query:     q,
-
-		DisableCompression: disableCompression,
-	}
-	return cp, nil
-}
-
-func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
-	var b []byte
-	for i := range vhs {
-		b = vhs[i].Marshal(b)
-	}
-
-	if !disableCompression {
-		b = zstd.CompressLevel(nil, b, 1)
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	if _, err := w.Write(b); err != nil {
-		return fmt.Errorf("cannot send response to the client: %w", err)
-	}
-
-	return nil
-}
-
-func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
-	s := r.FormValue(argName)
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
-	}
-	return n, nil
-}

app/vlselect/logsql/facets_response.qtpl

@@ -1,49 +0,0 @@
-{% import (
-	"slices"
-) %}
-
-{% stripspace %}
-
-{% func FacetsResponse(m map[string][]facetEntry) %}
-{
-	{% code
-		sortedKeys := make([]string, 0, len(m))
-		for k := range m {
-			sortedKeys = append(sortedKeys, k)
-		}
-		slices.Sort(sortedKeys)
-	%}
-	"facets":[
-		{% if len(sortedKeys) > 0 %}
-			{%= facetsLine(m, sortedKeys[0]) %}
-			{% for _, k := range sortedKeys[1:] %}
-				,{%= facetsLine(m, k) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetsLine(m map[string][]facetEntry, k string) %}
-{
-	"field_name":{%q= k %},
-	"values":[
-		{% code	fes := m[k] %}
-		{% if len(fes) > 0 %}
-			{%= facetLine(fes[0]) %}
-			{% for _, fe := range fes[1:] %}
-				,{%= facetLine(fe) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetLine(fe facetEntry) %}
-{
-	"field_value":{%q= fe.value %},
-	"hits":{%s= fe.hits %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/facets_response.qtpl.go

@@ -1,178 +0,0 @@
-// Code generated by qtc from "facets_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-import (
-	"slices"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-func StreamFacetsResponse(qw422016 *qt422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:7
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/facets_response.qtpl:10
-	sortedKeys := make([]string, 0, len(m))
-	for k := range m {
-		sortedKeys = append(sortedKeys, k)
-	}
-	slices.Sort(sortedKeys)
-
-//line app/vlselect/logsql/facets_response.qtpl:15
-	qw422016.N().S(`"facets":[`)
-//line app/vlselect/logsql/facets_response.qtpl:17
-	if len(sortedKeys) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:18
-		streamfacetsLine(qw422016, m, sortedKeys[0])
-//line app/vlselect/logsql/facets_response.qtpl:19
-		for _, k := range sortedKeys[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:19
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:20
-			streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:21
-		}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func WriteFacetsResponse(qq422016 qtio422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	StreamFacetsResponse(qw422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func FacetsResponse(m map[string][]facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:25
-	WriteFacetsResponse(qb422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:27
-func streamfacetsLine(qw422016 *qt422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:27
-	qw422016.N().S(`{"field_name":`)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().Q(k)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().S(`,"values":[`)
-//line app/vlselect/logsql/facets_response.qtpl:31
-	fes := m[k]
-
-//line app/vlselect/logsql/facets_response.qtpl:32
-	if len(fes) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:33
-		streamfacetLine(qw422016, fes[0])
-//line app/vlselect/logsql/facets_response.qtpl:34
-		for _, fe := range fes[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:34
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:35
-			streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:36
-		}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func writefacetsLine(qq422016 qtio422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func facetsLine(m map[string][]facetEntry, k string) string {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:40
-	writefacetsLine(qb422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:42
-func streamfacetLine(qw422016 *qt422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:42
-	qw422016.N().S(`{"field_value":`)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().Q(fe.value)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().S(`,"hits":`)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(fe.hits)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func writefacetLine(qq422016 qtio422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func facetLine(fe facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:47
-	writefacetLine(qb422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:47
-}

app/vlselect/logsql/hits_response.qtpl

@@ -1,70 +0,0 @@
-{% import (
-	"slices"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// FieldsForHits formats labels for /select/logsql/hits response
-{% func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) %}
-{
-	{% if len(columns) > 0 %}
-		{%q= columns[0].Name %}:{%q= columns[0].Values[rowIdx] %}
-		{% for _, c := range columns[1:] %}
-			,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-		{% endfor %}
-	{% endif %}
-}
-{% endfunc %}
-
-{% func HitsSeries(m map[string]*hitsSeries) %}
-{
-	{% code
-		sortedKeys := make([]string, 0, len(m))
-		for k := range m {
-			sortedKeys = append(sortedKeys, k)
-		}
-		slices.Sort(sortedKeys)
-	%}
-	"hits":[
-		{% if len(sortedKeys) > 0 %}
-			{%= hitsSeriesLine(m, sortedKeys[0]) %}
-			{% for _, k := range sortedKeys[1:] %}
-				,{%= hitsSeriesLine(m, k) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func hitsSeriesLine(m map[string]*hitsSeries, k string) %}
-{
-	{% code
-		hs := m[k]
-		hs.sort()
-		timestamps := hs.timestamps
-		hits := hs.hits
-	%}
-	"fields":{%s= k %},
-	"timestamps":[
-		{% if len(timestamps) > 0 %}
-			{%q= timestamps[0] %}
-			{% for _, ts := range timestamps[1:] %}
-				,{%q= ts %}
-			{% endfor %}
-		{% endif %}
-	],
-	"values":[
-		{% if len(hits) > 0 %}
-			{%dul= hits[0] %}
-			{% for _, v := range hits[1:] %}
-				,{%dul= v %}
-			{% endfor %}
-		{% endif %}
-	],
-	"total":{%dul= hs.hitsTotal %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/hits_response.qtpl.go

@@ -1,223 +0,0 @@
-// Code generated by qtc from "hits_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/hits_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/hits_response.qtpl:1
-import (
-	"slices"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// FieldsForHits formats labels for /select/logsql/hits response
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-func StreamFieldsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/hits_response.qtpl:10
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:12
-	if len(columns) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().Q(columns[0].Name)
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().Q(columns[0].Values[rowIdx])
-//line app/vlselect/logsql/hits_response.qtpl:14
-		for _, c := range columns[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/hits_response.qtpl:16
-		}
-//line app/vlselect/logsql/hits_response.qtpl:17
-	}
-//line app/vlselect/logsql/hits_response.qtpl:17
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:19
-func WriteFieldsForHits(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	StreamFieldsForHits(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:19
-func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:19
-	WriteFieldsForHits(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:21
-func StreamHitsSeries(qw422016 *qt422016.Writer, m map[string]*hitsSeries) {
-//line app/vlselect/logsql/hits_response.qtpl:21
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:24
-	sortedKeys := make([]string, 0, len(m))
-	for k := range m {
-		sortedKeys = append(sortedKeys, k)
-	}
-	slices.Sort(sortedKeys)
-
-//line app/vlselect/logsql/hits_response.qtpl:29
-	qw422016.N().S(`"hits":[`)
-//line app/vlselect/logsql/hits_response.qtpl:31
-	if len(sortedKeys) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:32
-		streamhitsSeriesLine(qw422016, m, sortedKeys[0])
-//line app/vlselect/logsql/hits_response.qtpl:33
-		for _, k := range sortedKeys[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:33
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:34
-			streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:35
-		}
-//line app/vlselect/logsql/hits_response.qtpl:36
-	}
-//line app/vlselect/logsql/hits_response.qtpl:36
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:39
-func WriteHitsSeries(qq422016 qtio422016.Writer, m map[string]*hitsSeries) {
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	StreamHitsSeries(qw422016, m)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:39
-func HitsSeries(m map[string]*hitsSeries) string {
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:39
-	WriteHitsSeries(qb422016, m)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:41
-func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:41
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:44
-	hs := m[k]
-	hs.sort()
-	timestamps := hs.timestamps
-	hits := hs.hits
-
-//line app/vlselect/logsql/hits_response.qtpl:48
-	qw422016.N().S(`"fields":`)
-//line app/vlselect/logsql/hits_response.qtpl:49
-	qw422016.N().S(k)
-//line app/vlselect/logsql/hits_response.qtpl:49
-	qw422016.N().S(`,"timestamps":[`)
-//line app/vlselect/logsql/hits_response.qtpl:51
-	if len(timestamps) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:52
-		qw422016.N().Q(timestamps[0])
-//line app/vlselect/logsql/hits_response.qtpl:53
-		for _, ts := range timestamps[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:53
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:54
-			qw422016.N().Q(ts)
-//line app/vlselect/logsql/hits_response.qtpl:55
-		}
-//line app/vlselect/logsql/hits_response.qtpl:56
-	}
-//line app/vlselect/logsql/hits_response.qtpl:56
-	qw422016.N().S(`],"values":[`)
-//line app/vlselect/logsql/hits_response.qtpl:59
-	if len(hits) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:60
-		qw422016.N().DUL(hits[0])
-//line app/vlselect/logsql/hits_response.qtpl:61
-		for _, v := range hits[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:61
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:62
-			qw422016.N().DUL(v)
-//line app/vlselect/logsql/hits_response.qtpl:63
-		}
-//line app/vlselect/logsql/hits_response.qtpl:64
-	}
-//line app/vlselect/logsql/hits_response.qtpl:64
-	qw422016.N().S(`],"total":`)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().DUL(hs.hitsTotal)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/hits_response.qtpl:68
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:68
-func writehitsSeriesLine(qq422016 qtio422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:68
-func hitsSeriesLine(m map[string]*hitsSeries, k string) string {
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:68
-	writehitsSeriesLine(qb422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:68
-}

app/vlselect/logsql/logsql.qtpl

@@ -1,32 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// ValuesWithHitsJSON generates JSON from the given values.
-{% func ValuesWithHitsJSON(values []logstorage.ValueWithHits) %}
-{
-	"values":{%= valuesWithHitsJSONArray(values) %}
-}
-{% endfunc %}
-
-{% func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) %}
-[
-	{% if len(values) > 0 %}
-		{%= valueWithHitsJSON(values[0]) %}
-		{% for _, v := range values[1:] %}
-			,{%= valueWithHitsJSON(v) %}
-		{% endfor %}
-	{% endif %}
-]
-{% endfunc %}
-
-{% func valueWithHitsJSON(v logstorage.ValueWithHits) %}
-{
-	"value":{%q= v.Value %},
-	"hits":{%dul= v.Hits %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/logsql.qtpl.go

@@ -1,152 +0,0 @@
-// Code generated by qtc from "logsql.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/logsql.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/logsql.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ValuesWithHitsJSON generates JSON from the given values.
-
-//line app/vlselect/logsql/logsql.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/logsql.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/logsql.qtpl:8
-func StreamValuesWithHitsJSON(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:8
-	qw422016.N().S(`{"values":`)
-//line app/vlselect/logsql/logsql.qtpl:10
-	streamvaluesWithHitsJSONArray(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:10
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:12
-func WriteValuesWithHitsJSON(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:12
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-	StreamValuesWithHitsJSON(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:12
-func ValuesWithHitsJSON(values []logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:12
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:12
-	WriteValuesWithHitsJSON(qb422016, values)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:14
-func streamvaluesWithHitsJSONArray(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:14
-	qw422016.N().S(`[`)
-//line app/vlselect/logsql/logsql.qtpl:16
-	if len(values) > 0 {
-//line app/vlselect/logsql/logsql.qtpl:17
-		streamvalueWithHitsJSON(qw422016, values[0])
-//line app/vlselect/logsql/logsql.qtpl:18
-		for _, v := range values[1:] {
-//line app/vlselect/logsql/logsql.qtpl:18
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/logsql.qtpl:19
-			streamvalueWithHitsJSON(qw422016, v)
-//line app/vlselect/logsql/logsql.qtpl:20
-		}
-//line app/vlselect/logsql/logsql.qtpl:21
-	}
-//line app/vlselect/logsql/logsql.qtpl:21
-	qw422016.N().S(`]`)
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:23
-func writevaluesWithHitsJSONArray(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:23
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-	streamvaluesWithHitsJSONArray(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:23
-func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:23
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:23
-	writevaluesWithHitsJSONArray(qb422016, values)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:25
-func streamvalueWithHitsJSON(qw422016 *qt422016.Writer, v logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:25
-	qw422016.N().S(`{"value":`)
-//line app/vlselect/logsql/logsql.qtpl:27
-	qw422016.N().Q(v.Value)
-//line app/vlselect/logsql/logsql.qtpl:27
-	qw422016.N().S(`,"hits":`)
-//line app/vlselect/logsql/logsql.qtpl:28
-	qw422016.N().DUL(v.Hits)
-//line app/vlselect/logsql/logsql.qtpl:28
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/logsql.qtpl:30
-}
-
-//line app/vlselect/logsql/logsql.qtpl:30
-func writevalueWithHitsJSON(qq422016 qtio422016.Writer, v logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:30
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-	streamvalueWithHitsJSON(qw422016, v)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-}
-
-//line app/vlselect/logsql/logsql.qtpl:30
-func valueWithHitsJSON(v logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:30
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:30
-	writevalueWithHitsJSON(qb422016, v)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:30
-}

app/vlselect/logsql/logsql_test.go

@@ -1,103 +0,0 @@
-package logsql
-
-import (
-	"testing"
-)
-
-func TestParseExtraFilters_Success(t *testing.T) {
-	f := func(s, resultExpected string) {
-		t.Helper()
-
-		f, err := parseExtraFilters(s)
-		if err != nil {
-			t.Fatalf("unexpected error in parseExtraFilters: %s", err)
-		}
-		result := f.String()
-		if result != resultExpected {
-			t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	f("", "")
-
-	// JSON string
-	f(`{"foo":"bar"}`, `foo:=bar`)
-	f(`{"foo":["bar","baz"]}`, `foo:in(bar,baz)`)
-	f(`{"z":"=b ","c":["d","e,"],"a":[],"_msg":"x"}`, `z:="=b " c:in(d,"e,") =x`)
-
-	// LogsQL filter
-	f(`foobar`, `foobar`)
-	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
-}
-
-func TestParseExtraFilters_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := parseExtraFilters(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// Invalid JSON
-	f(`{"foo"}`)
-	f(`[1,2]`)
-	f(`{"foo":[1]}`)
-
-	// Invliad LogsQL filter
-	f(`foo:(bar`)
-
-	// excess pipe
-	f(`foo | count()`)
-}
-
-func TestParseExtraStreamFilters_Success(t *testing.T) {
-	f := func(s, resultExpected string) {
-		t.Helper()
-
-		f, err := parseExtraStreamFilters(s)
-		if err != nil {
-			t.Fatalf("unexpected error in parseExtraStreamFilters: %s", err)
-		}
-		result := f.String()
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	f("", "")
-
-	// JSON string
-	f(`{"foo":"bar"}`, `{foo="bar"}`)
-	f(`{"foo":["bar","baz"]}`, `{foo=~"bar|baz"}`)
-	f(`{"z":"b","c":["d","e|\""],"a":[],"_msg":"x"}`, `{z="b",c=~"d|e\\|\"",_msg="x"}`)
-
-	// LogsQL filter
-	f(`foobar`, `foobar`)
-	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
-}
-
-func TestParseExtraStreamFilters_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := parseExtraStreamFilters(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// Invalid JSON
-	f(`{"foo"}`)
-	f(`[1,2]`)
-	f(`{"foo":[1]}`)
-
-	// Invliad LogsQL filter
-	f(`foo:(bar`)
-
-	// excess pipe
-	f(`foo | count()`)
-}

app/vlselect/logsql/query_response.qtpl

@@ -1,64 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// JSONRow creates JSON row from the given fields.
-{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-	{% code
-		i := 0
-		for i < len(columns) && columns[i].Values[rowIdx] == "" {
-			i++
-		}
-		columns = columns[i:]
-	%}
-	{% if len(columns) == 0 %}
-		{% return %}
-	{% endif %}
-	{
-	{% code	c := &columns[0] %}
-	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% code columns = columns[1:] %}
-	{% for colIdx := range columns %}
-		{% code
-			c := &columns[colIdx]
-			v := c.Values[rowIdx]
-		%}
-		{% if v == "" %}
-			{% continue %}
-		{% endif %}
-		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% endfor %}
-	}{% newline %}
-{% endfunc %}
-
-// JSONRows prints formatted rows
-{% func JSONRows(rows [][]logstorage.Field) %}
-	{% if len(rows) == 0 %}
-		{% return %}
-	{% endif %}
-	{% for _, fields := range rows %}
-		{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
-		{% if len(fields) == 0 %}
-			{% continue %}
-		{% endif %}
-		{
-		{% if len(fields) > 0 %}
-			{% code
-				f := fields[0]
-				fields = fields[1:]
-			%}
-			{%q= f.Name %}:{%q= f.Value %}
-			{% for _, f := range fields %}
-				{% if f.Value == "" %}
-					{% continue %}
-				{% endif %}
-				,{%q= f.Name %}:{%q= f.Value %}
-			{% endfor %}
-		{% endif %}
-		}{% newline %}
-	{% endfor %}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/query_response.qtpl.go

@@ -1,201 +0,0 @@
-// Code generated by qtc from "query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/query_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/query_response.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// JSONRow creates JSON row from the given fields.
-
-//line app/vlselect/logsql/query_response.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:10
-	i := 0
-	for i < len(columns) && columns[i].Values[rowIdx] == "" {
-		i++
-	}
-	columns = columns[i:]
-
-//line app/vlselect/logsql/query_response.qtpl:16
-	if len(columns) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:17
-		return
-//line app/vlselect/logsql/query_response.qtpl:18
-	}
-//line app/vlselect/logsql/query_response.qtpl:18
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:20
-	c := &columns[0]
-
-//line app/vlselect/logsql/query_response.qtpl:21
-	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:21
-	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:21
-	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:22
-	columns = columns[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:23
-	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:25
-		c := &columns[colIdx]
-		v := c.Values[rowIdx]
-
-//line app/vlselect/logsql/query_response.qtpl:28
-		if v == "" {
-//line app/vlselect/logsql/query_response.qtpl:29
-			continue
-//line app/vlselect/logsql/query_response.qtpl:30
-		}
-//line app/vlselect/logsql/query_response.qtpl:30
-		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:31
-		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:31
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:31
-		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:32
-	}
-//line app/vlselect/logsql/query_response.qtpl:32
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:33
-	qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/query_response.qtpl:34
-func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:34
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:34
-	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/query_response.qtpl:34
-func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:34
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:34
-	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:34
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:34
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:34
-}
-
-// JSONRows prints formatted rows
-
-//line app/vlselect/logsql/query_response.qtpl:37
-func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:38
-	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:39
-		return
-//line app/vlselect/logsql/query_response.qtpl:40
-	}
-//line app/vlselect/logsql/query_response.qtpl:41
-	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:42
-		fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
-
-//line app/vlselect/logsql/query_response.qtpl:43
-		if len(fields) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:44
-			continue
-//line app/vlselect/logsql/query_response.qtpl:45
-		}
-//line app/vlselect/logsql/query_response.qtpl:45
-		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:47
-		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:49
-			f := fields[0]
-			fields = fields[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:52
-			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:52
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:52
-			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:53
-			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:54
-				if f.Value == "" {
-//line app/vlselect/logsql/query_response.qtpl:55
-					continue
-//line app/vlselect/logsql/query_response.qtpl:56
-				}
-//line app/vlselect/logsql/query_response.qtpl:56
-				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:57
-				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:57
-				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:57
-				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:58
-			}
-//line app/vlselect/logsql/query_response.qtpl:59
-		}
-//line app/vlselect/logsql/query_response.qtpl:59
-		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:60
-		qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:61
-	}
-//line app/vlselect/logsql/query_response.qtpl:62
-}
-
-//line app/vlselect/logsql/query_response.qtpl:62
-func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:62
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:62
-	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:62
-}
-
-//line app/vlselect/logsql/query_response.qtpl:62
-func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:62
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:62
-	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:62
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:62
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:62
-}

app/vlselect/logsql/stats_query_range_response.qtpl

@@ -1,52 +0,0 @@
-{% stripspace %}
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-{% func StatsQueryRangeResponse(rows []*statsSeries) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"matrix",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsSeries(rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsSeries(rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsSeries(ss *statsSeries) %}
-{
-	"metric":{
-		"__name__":{%q= ss.Name %}
-		{% if len(ss.Labels) > 0 %}
-			{% for _, label := range ss.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"values":[
-		{% code points := ss.Points %}
-		{% if len(points) > 0 %}
-			{%= formatStatsPoint(&points[0]) %}
-			{% code points = points[1:] %}
-			{% for i := range points %}
-				,{%= formatStatsPoint(&points[i]) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func formatStatsPoint(p *statsPoint) %}
-[
-	{%f= float64(p.Timestamp)/1e9 %},
-	{%q= p.Value %}
-]
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_range_response.qtpl.go

@@ -1,188 +0,0 @@
-// Code generated by qtc from "stats_query_range_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-func StreamStatsQueryRangeResponse(qw422016 *qt422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:11
-		streamformatStatsSeries(qw422016, rows[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:14
-			streamformatStatsSeries(qw422016, rows[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func WriteStatsQueryRangeResponse(qq422016 qtio422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	StreamStatsQueryRangeResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func StatsQueryRangeResponse(rows []*statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	WriteStatsQueryRangeResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-func streamformatStatsSeries(qw422016 *qt422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:25
-	qw422016.N().Q(ss.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:26
-	if len(ss.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-		for _, label := range ss.Labels {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	qw422016.N().S(`},"values":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:33
-	points := ss.Points
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:34
-	if len(points) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:35
-		streamformatStatsPoint(qw422016, &points[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:36
-		points = points[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-		for i := range points {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:38
-			streamformatStatsPoint(qw422016, &points[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:39
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func writeformatStatsSeries(qq422016 qtio422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	streamformatStatsSeries(qw422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func formatStatsSeries(ss *statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	writeformatStatsSeries(qb422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-func streamformatStatsPoint(qw422016 *qt422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-	qw422016.N().S(`[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().F(float64(p.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().Q(p.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().S(`]`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func writeformatStatsPoint(qq422016 qtio422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	streamformatStatsPoint(qw422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func formatStatsPoint(p *statsPoint) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	writeformatStatsPoint(qb422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}

app/vlselect/logsql/stats_query_response.qtpl

@@ -1,36 +0,0 @@
-{% stripspace %}
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-{% func StatsQueryResponse(rows []statsRow) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"vector",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsRow(&rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsRow(&rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsRow(r *statsRow) %}
-{
-	"metric":{
-		"__name__":{%q= r.Name %}
-		{% if len(r.Labels) > 0 %}
-			{% for _, label := range r.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"value":[{%f= float64(r.Timestamp)/1e9 %},{%q= r.Value %}]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_response.qtpl.go

@@ -1,133 +0,0 @@
-// Code generated by qtc from "stats_query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-func StreamStatsQueryResponse(qw422016 *qt422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:11
-		streamformatStatsRow(qw422016, &rows[0])
-//line app/vlselect/logsql/stats_query_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:14
-			streamformatStatsRow(qw422016, &rows[i])
-//line app/vlselect/logsql/stats_query_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func WriteStatsQueryResponse(qq422016 qtio422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	StreamStatsQueryResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func StatsQueryResponse(rows []statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	WriteStatsQueryResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-func streamformatStatsRow(qw422016 *qt422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_response.qtpl:25
-	qw422016.N().Q(r.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:26
-	if len(r.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-		for _, label := range r.Labels {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	qw422016.N().S(`},"value":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().F(float64(r.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().Q(r.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func writeformatStatsRow(qq422016 qtio422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	streamformatStatsRow(qw422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func formatStatsRow(r *statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	writeformatStatsRow(qb422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}

app/vlselect/main.go

@@ -1,324 +0,0 @@
-package vlselect
-
-import (
-	"context"
-	"embed"
-	"flag"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/internalselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration")
-	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
-		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden to a smaller value on a per-query basis via 'timeout' query arg")
-
-	disableSelect   = flag.Bool("select.disable", false, "Whether to disable /select/* HTTP endpoints")
-	disableInternal = flag.Bool("internalselect.disable", false, "Whether to disable /internal/select/* HTTP endpoints")
-)
-
-func getDefaultMaxConcurrentRequests() int {
-	n := cgroup.AvailableCPUs()
-	if n <= 4 {
-		n *= 2
-	}
-	if n > 16 {
-		// A single request can saturate all the CPU cores, so there is no sense
-		// in allowing higher number of concurrent requests - they will just contend
-		// for unavailable CPU time.
-		n = 16
-	}
-	return n
-}
-
-// Init initializes vlselect
-func Init() {
-	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
-}
-
-// Stop stops vlselect
-func Stop() {
-}
-
-var concurrencyLimitCh chan struct{}
-
-var (
-	concurrencyLimitReached = metrics.NewCounter(`vl_concurrent_select_limit_reached_total`)
-	concurrencyLimitTimeout = metrics.NewCounter(`vl_concurrent_select_limit_timeout_total`)
-
-	_ = metrics.NewGauge(`vl_concurrent_select_capacity`, func() float64 {
-		return float64(cap(concurrencyLimitCh))
-	})
-	_ = metrics.NewGauge(`vl_concurrent_select_current`, func() float64 {
-		return float64(len(concurrencyLimitCh))
-	})
-)
-
-//go:embed vmui
-var vmuiFiles embed.FS
-
-var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
-
-// RequestHandler handles select requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
-
-	if strings.HasPrefix(path, "/select/") {
-		if *disableSelect {
-			httpserver.Errorf(w, r, "requests to /select/* are disabled with -select.disable command-line flag")
-			return true
-		}
-
-		return selectHandler(w, r, path)
-	}
-
-	if strings.HasPrefix(path, "/internal/select/") {
-		if *disableInternal || *disableSelect {
-			httpserver.Errorf(w, r, "requests to /internal/select/* are disabled with -internalselect.disable or -select.disable command-line flag")
-			return true
-		}
-		internalselect.RequestHandler(r.Context(), w, r)
-		return true
-	}
-
-	return false
-}
-
-func selectHandler(w http.ResponseWriter, r *http.Request, path string) bool {
-	ctx := r.Context()
-
-	if path == "/select/vmui" {
-		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
-		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
-		// is hidden behind vmauth or similar proxy.
-		_ = r.ParseForm()
-		newURL := "vmui/?" + r.Form.Encode()
-		httpserver.Redirect(w, newURL)
-		return true
-	}
-	if strings.HasPrefix(path, "/select/vmui/") {
-		if strings.HasPrefix(path, "/select/vmui/static/") {
-			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
-			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
-			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
-			w.Header().Set("Cache-Control", "max-age=31536000")
-		}
-		r.URL.Path = strings.TrimPrefix(path, "/select")
-		vmuiFileServer.ServeHTTP(w, r)
-		return true
-	}
-
-	if path == "/select/logsql/tail" {
-		logsqlTailRequests.Inc()
-		// Process live tailing request without timeout, since it is OK to run live tailing requests for very long time.
-		// Also do not apply concurrency limit to tail requests, since these limits are intended for non-tail requests.
-		logsql.ProcessLiveTailRequest(ctx, w, r)
-		return true
-	}
-
-	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
-	startTime := time.Now()
-	d := getMaxQueryDuration(r)
-	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
-	defer cancel()
-
-	if !incRequestConcurrency(ctxWithTimeout, w, r) {
-		return true
-	}
-	defer decRequestConcurrency()
-
-	ok := processSelectRequest(ctxWithTimeout, w, r, path)
-	if !ok {
-		return false
-	}
-
-	logRequestErrorIfNeeded(ctxWithTimeout, w, r, startTime)
-	return true
-}
-
-func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http.Request, startTime time.Time) {
-	err := ctx.Err()
-	switch err {
-	case nil:
-		// nothing to do
-	case context.Canceled:
-		// do not log canceled requests, since they are expected and legal.
-	case context.DeadlineExceeded:
-		err = &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
-				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", time.Since(startTime).Seconds(), maxQueryDuration),
-			StatusCode: http.StatusServiceUnavailable,
-		}
-		httpserver.Errorf(w, r, "%s", err)
-	default:
-		httpserver.Errorf(w, r, "unexpected error: %s", err)
-	}
-}
-
-func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {
-	startTime := time.Now()
-	stopCh := ctx.Done()
-	select {
-	case concurrencyLimitCh <- struct{}{}:
-		return true
-	default:
-		// Sleep for a while until giving up. This should resolve short bursts in requests.
-		concurrencyLimitReached.Inc()
-		select {
-		case concurrencyLimitCh <- struct{}{}:
-			return true
-		case <-stopCh:
-			switch ctx.Err() {
-			case context.Canceled:
-				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-				requestURI := httpserver.GetRequestURI(r)
-				logger.Infof("client has canceled the pending request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-					time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			case context.DeadlineExceeded:
-				concurrencyLimitTimeout.Inc()
-				err := &httpserver.ErrorWithStatusCode{
-					Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-						"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-						"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
-						"to pass bigger value to 'timeout' query arg",
-						time.Since(startTime).Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
-					StatusCode: http.StatusServiceUnavailable,
-				}
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return false
-		}
-	}
-}
-
-func decRequestConcurrency() {
-	<-concurrencyLimitCh
-}
-
-func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
-	httpserver.EnableCORS(w, r)
-	startTime := time.Now()
-	switch path {
-	case "/select/logsql/facets":
-		logsqlFacetsRequests.Inc()
-		logsql.ProcessFacetsRequest(ctx, w, r)
-		logsqlFacetsDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/field_names":
-		logsqlFieldNamesRequests.Inc()
-		logsql.ProcessFieldNamesRequest(ctx, w, r)
-		logsqlFieldNamesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/field_values":
-		logsqlFieldValuesRequests.Inc()
-		logsql.ProcessFieldValuesRequest(ctx, w, r)
-		logsqlFieldValuesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/hits":
-		logsqlHitsRequests.Inc()
-		logsql.ProcessHitsRequest(ctx, w, r)
-		logsqlHitsDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/query":
-		logsqlQueryRequests.Inc()
-		logsql.ProcessQueryRequest(ctx, w, r)
-		logsqlQueryDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stats_query":
-		logsqlStatsQueryRequests.Inc()
-		logsql.ProcessStatsQueryRequest(ctx, w, r)
-		logsqlStatsQueryDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stats_query_range":
-		logsqlStatsQueryRangeRequests.Inc()
-		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
-		logsqlStatsQueryRangeDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stream_field_names":
-		logsqlStreamFieldNamesRequests.Inc()
-		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
-		logsqlStreamFieldNamesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stream_field_values":
-		logsqlStreamFieldValuesRequests.Inc()
-		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
-		logsqlStreamFieldValuesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stream_ids":
-		logsqlStreamIDsRequests.Inc()
-		logsql.ProcessStreamIDsRequest(ctx, w, r)
-		logsqlStreamIDsDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/streams":
-		logsqlStreamsRequests.Inc()
-		logsql.ProcessStreamsRequest(ctx, w, r)
-		logsqlStreamsDuration.UpdateDuration(startTime)
-		return true
-	default:
-		return false
-	}
-}
-
-// getMaxQueryDuration returns the maximum duration for query from r.
-func getMaxQueryDuration(r *http.Request) time.Duration {
-	dms, err := httputil.GetDuration(r, "timeout", 0)
-	if err != nil {
-		dms = 0
-	}
-	d := time.Duration(dms) * time.Millisecond
-	if d <= 0 || d > *maxQueryDuration {
-		d = *maxQueryDuration
-	}
-	return d
-}
-
-var (
-	logsqlFacetsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
-	logsqlFacetsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/facets"}`)
-
-	logsqlFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
-	logsqlFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_names"}`)
-
-	logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
-	logsqlFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_values"}`)
-
-	logsqlHitsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
-	logsqlHitsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/hits"}`)
-
-	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/query"}`)
-
-	logsqlStatsQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
-	logsqlStatsQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query"}`)
-
-	logsqlStatsQueryRangeRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
-	logsqlStatsQueryRangeDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query_range"}`)
-
-	logsqlStreamFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
-	logsqlStreamFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_names"}`)
-
-	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
-	logsqlStreamFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_values"}`)
-
-	logsqlStreamIDsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
-	logsqlStreamIDsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_ids"}`)
-
-	logsqlStreamsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
-	logsqlStreamsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/streams"}`)
-
-	// no need to track duration for tail requests, as they usually take long time
-	logsqlTailRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
-)

app/vlselect/vmui/assets/vendor-D1GxaB_c.css

@@ -1 +0,0 @@
-.uplot,.uplot *,.uplot *:before,.uplot *:after{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol,"Noto Color Emoji";line-height:1.5;width:min-content}.u-title{text-align:center;font-size:18px;font-weight:700}.u-wrap{position:relative;-webkit-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;position:relative;width:100%;height:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{vertical-align:middle;display:inline-block}.u-legend .u-marker{width:1em;height:1em;margin-right:4px;background-clip:padding-box!important}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:#00000012;position:absolute;pointer-events:none}.u-cursor-x,.u-cursor-y{position:absolute;left:0;top:0;pointer-events:none;will-change:transform}.u-hz .u-cursor-x,.u-vt .u-cursor-y{height:100%;border-right:1px dashed #607D8B}.u-hz .u-cursor-y,.u-vt .u-cursor-x{width:100%;border-bottom:1px dashed #607D8B}.u-cursor-pt{position:absolute;top:0;left:0;border-radius:50%;border:0 solid;pointer-events:none;will-change:transform;background-clip:padding-box!important}.u-axis.u-off,.u-select.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-cursor-pt.u-off{display:none}

app/vlselect/vmui/config.json

@@ -1,5 +0,0 @@
-{
-  "license": {
-    "type": "opensource"
-  }
-}

app/vlselect/vmui/favicon.svg

@@ -1 +0,0 @@
-<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>

app/vlselect/vmui/index.html

@@ -1,57 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="./favicon.svg" />
-  <link rel="apple-touch-icon" href="./favicon.svg" />
-  <link rel="mask-icon" href="./favicon.svg" color="#000000">
-
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta name="description" content="Explore your log data with VictoriaLogs UI"/>
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="./manifest.json"/>
-  <!--
-    Notice the use of in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>UI for VictoriaLogs</title>
-
-  <meta name="twitter:card" content="summary">
-  <meta name="twitter:title" content="UI for VictoriaLogs">
-  <meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/">
-  <meta name="twitter:description" content="Explore your log data with VictoriaLogs UI">
-  <meta name="twitter:image" content="./preview.jpg">
-
-  <meta property="og:type" content="website">
-  <meta property="og:title" content="UI for VictoriaLogs">
-  <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
-  <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-721xTF8u.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-V4vnRsM-.js">
-  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C36SC0pJ.css">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-</body>
-</html>

app/vlselect/vmui/manifest.json

@@ -1,15 +0,0 @@
-{
-  "short_name": "vmui",
-  "name": "vmui",
-  "icons": [
-    {
-      "src": "favicon.svg",
-      "sizes": "any",
-      "type": "image/svg+xml"
-    }
-  ],
-  "start_url": ".",
-  "display": "standalone",
-  "theme_color": "#000000",
-  "background_color": "#ffffff"
-}

app/vlselect/vmui/robots.txt

@@ -1,3 +0,0 @@
-# https://www.robotstxt.org/robotstxt.html
-User-agent: *
-Disallow:

app/vlstorage/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlstorage/main.go

@@ -1,407 +0,0 @@
-package vlstorage
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-)
-
-var (
-	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
-		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
-	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
-		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
-	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
-		"see https://docs.victoriametrics.com/victorialogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
-		"see https://docs.victoriametrics.com/victorialogs/#storage")
-	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
-		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
-	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows")
-	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
-		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
-	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
-		"the storage stops accepting new data")
-
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge . It overrides -httpAuth.* . "+
-		"See https://docs.victoriametrics.com/victorialogs/#forced-merge")
-	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush . It overrides -httpAuth.* . "+
-		"See https://docs.victoriametrics.com/victorialogs/#forced-flush")
-
-	storageNodeAddrs = flagutil.NewArrayString("storageNode", "Comma-separated list of TCP addresses for storage nodes to route the ingested logs to and to send select queries to. "+
-		"If the list is empty, then the ingested logs are stored and queried locally from -storageDataPath")
-	insertConcurrency        = flag.Int("insert.concurrency", 2, "The average number of concurrent data ingestion requests, which can be sent to every -storageNode")
-	insertDisableCompression = flag.Bool("insert.disableCompression", false, "Whether to disable compression when sending the ingested data to -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-	selectDisableCompression = flag.Bool("select.disableCompression", false, "Whether to disable compression for select query responses received from -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-
-	storageNodeUsername     = flagutil.NewArrayString("storageNode.username", "Optional basic auth username to use for the corresponding -storageNode")
-	storageNodePassword     = flagutil.NewArrayString("storageNode.password", "Optional basic auth password to use for the corresponding -storageNode")
-	storageNodePasswordFile = flagutil.NewArrayString("storageNode.passwordFile", "Optional path to basic auth password to use for the corresponding -storageNode. "+
-		"The file is re-read every second")
-	storageNodeBearerToken     = flagutil.NewArrayString("storageNode.bearerToken", "Optional bearer auth token to use for the corresponding -storageNode")
-	storageNodeBearerTokenFile = flagutil.NewArrayString("storageNode.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -storageNode. "+
-		"The token is re-read from the file every second")
-
-	storageNodeTLS = flagutil.NewArrayBool("storageNode.tls", "Whether to use TLS (HTTPS) protocol for communicating with the corresponding -storageNode. "+
-		"By default communication is performed via HTTP")
-	storageNodeTLSCAFile = flagutil.NewArrayString("storageNode.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -storageNode. "+
-		"By default, system CA is used")
-	storageNodeTLSCertFile = flagutil.NewArrayString("storageNode.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
-		"to the corresponding -storageNode")
-	storageNodeTLSKeyFile    = flagutil.NewArrayString("storageNode.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -storageNode")
-	storageNodeTLSServerName = flagutil.NewArrayString("storageNode.tlsServerName", "Optional TLS server name to use for connections to the corresponding -storageNode. "+
-		"By default, the server name from -storageNode is used")
-	storageNodeTLSInsecureSkipVerify = flagutil.NewArrayBool("storageNode.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -storageNode")
-)
-
-var localStorage *logstorage.Storage
-var localStorageMetrics *metrics.Set
-
-var netstorageInsert *netinsert.Storage
-var netstorageSelect *netselect.Storage
-
-// Init initializes vlstorage.
-//
-// Stop must be called when vlstorage is no longer needed
-func Init() {
-	if len(*storageNodeAddrs) == 0 {
-		initLocalStorage()
-	} else {
-		initNetworkStorage()
-	}
-}
-
-func initLocalStorage() {
-	if localStorage != nil {
-		logger.Panicf("BUG: initLocalStorage() has been already called")
-	}
-
-	if retentionPeriod.Duration() < 24*time.Hour {
-		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
-	}
-	cfg := &logstorage.StorageConfig{
-		Retention:              retentionPeriod.Duration(),
-		MaxDiskSpaceUsageBytes: maxDiskSpaceUsageBytes.N,
-		FlushInterval:          *inmemoryDataFlushInterval,
-		FutureRetention:        futureRetention.Duration(),
-		LogNewStreams:          *logNewStreams,
-		LogIngestedRows:        *logIngestedRows,
-		MinFreeDiskSpaceBytes:  minFreeDiskSpaceBytes.N,
-	}
-	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
-	startTime := time.Now()
-	localStorage = logstorage.MustOpenStorage(*storageDataPath, cfg)
-
-	var ss logstorage.StorageStats
-	localStorage.UpdateStats(&ss)
-	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
-		"smallPartSize: %d bytes; bigPartSize: %d bytes",
-		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
-		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
-
-	// register local storage metrics
-	localStorageMetrics = metrics.NewSet()
-	localStorageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, localStorage)
-	})
-	metrics.RegisterSet(localStorageMetrics)
-}
-
-func initNetworkStorage() {
-	if netstorageInsert != nil || netstorageSelect != nil {
-		logger.Panicf("BUG: initNetworkStorage() has been already called")
-	}
-
-	authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
-	isTLSs := make([]bool, len(*storageNodeAddrs))
-	for i := range authCfgs {
-		authCfgs[i] = newAuthConfigForStorageNode(i)
-		isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
-	}
-
-	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
-	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
-
-	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
-	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
-
-	logger.Infof("initialized all the network services")
-}
-
-func newAuthConfigForStorageNode(argIdx int) *promauth.Config {
-	username := storageNodeUsername.GetOptionalArg(argIdx)
-	password := storageNodePassword.GetOptionalArg(argIdx)
-	passwordFile := storageNodePasswordFile.GetOptionalArg(argIdx)
-	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
-		basicAuthCfg = &promauth.BasicAuthConfig{
-			Username:     username,
-			Password:     promauth.NewSecret(password),
-			PasswordFile: passwordFile,
-		}
-	}
-
-	token := storageNodeBearerToken.GetOptionalArg(argIdx)
-	tokenFile := storageNodeBearerTokenFile.GetOptionalArg(argIdx)
-
-	tlsCfg := &promauth.TLSConfig{
-		CAFile:             storageNodeTLSCAFile.GetOptionalArg(argIdx),
-		CertFile:           storageNodeTLSCertFile.GetOptionalArg(argIdx),
-		KeyFile:            storageNodeTLSKeyFile.GetOptionalArg(argIdx),
-		ServerName:         storageNodeTLSServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: storageNodeTLSInsecureSkipVerify.GetOptionalArg(argIdx),
-	}
-
-	opts := &promauth.Options{
-		BasicAuth:       basicAuthCfg,
-		BearerToken:     token,
-		BearerTokenFile: tokenFile,
-		TLSConfig:       tlsCfg,
-	}
-	ac, err := opts.NewConfig()
-	if err != nil {
-		logger.Panicf("FATAL: cannot populate auth config for storage node #%d: %s", argIdx, err)
-	}
-
-	return ac
-}
-
-// Stop stops vlstorage.
-func Stop() {
-	if localStorage != nil {
-		metrics.UnregisterSet(localStorageMetrics, true)
-		localStorageMetrics = nil
-
-		localStorage.MustClose()
-		localStorage = nil
-	} else {
-		netstorageInsert.MustStop()
-		netstorageInsert = nil
-
-		netstorageSelect.MustStop()
-		netstorageSelect = nil
-	}
-}
-
-// RequestHandler is a storage request handler.
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	switch path {
-	case "/internal/force_merge":
-		return processForceMerge(w, r)
-	case "/internal/force_flush":
-		return processForceFlush(w, r)
-	}
-	return false
-}
-
-func processForceMerge(w http.ResponseWriter, r *http.Request) bool {
-	if localStorage == nil {
-		// Force merge isn't supported by non-local storage
-		return false
-	}
-
-	if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-		return true
-	}
-
-	// Run force merge in background
-	partitionNamePrefix := r.FormValue("partition_prefix")
-	go func() {
-		activeForceMerges.Inc()
-		defer activeForceMerges.Dec()
-		logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-		startTime := time.Now()
-		localStorage.MustForceMerge(partitionNamePrefix)
-		logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-	}()
-	return true
-}
-
-func processForceFlush(w http.ResponseWriter, r *http.Request) bool {
-	if localStorage == nil {
-		// Force merge isn't supported by non-local storage
-		return false
-	}
-
-	if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey) {
-		return true
-	}
-
-	logger.Infof("flushing storage to make pending data available for reading")
-	localStorage.DebugFlush()
-	return true
-}
-
-// Storage implements insertutil.LogRowsStorage interface
-type Storage struct{}
-
-// CanWriteData returns non-nil error if it cannot write data to vlstorage
-func (*Storage) CanWriteData() error {
-	if localStorage == nil {
-		// The data can be always written in non-local mode.
-		return nil
-	}
-
-	if localStorage.IsReadOnly() {
-		return &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
-				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
-			StatusCode: http.StatusTooManyRequests,
-		}
-	}
-	return nil
-}
-
-// MustAddRows adds lr to vlstorage
-//
-// It is advised to call CanWriteData() before calling MustAddRows()
-func (*Storage) MustAddRows(lr *logstorage.LogRows) {
-	if localStorage != nil {
-		// Store lr in the local storage.
-		localStorage.MustAddRows(lr)
-	} else {
-		// Store lr across the remote storage nodes.
-		lr.ForEachRow(netstorageInsert.AddRow)
-	}
-}
-
-// RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	if localStorage != nil {
-		return localStorage.RunQuery(ctx, tenantIDs, q, writeBlock)
-	}
-	return netstorageSelect.RunQuery(ctx, tenantIDs, q, writeBlock)
-}
-
-// GetFieldNames executes q and returns field names seen in results.
-func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetFieldNames(ctx, tenantIDs, q)
-}
-
-// GetFieldValues executes q and returns unique values for the fieldName seen in results.
-//
-// If limit > 0, then up to limit unique values are returned.
-func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
-}
-
-// GetStreamFieldNames executes q and returns stream field names seen in results.
-func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetStreamFieldNames(ctx, tenantIDs, q)
-}
-
-// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
-//
-// If limit > 0, then up to limit unique stream field values are returned.
-func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-}
-
-// GetStreams executes q and returns streams seen in query results.
-//
-// If limit > 0, then up to limit unique streams are returned.
-func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreams(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreams(ctx, tenantIDs, q, limit)
-}
-
-// GetStreamIDs executes q and returns streamIDs seen in query results.
-//
-// If limit > 0, then up to limit unique streamIDs are returned.
-func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamIDs(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreamIDs(ctx, tenantIDs, q, limit)
-}
-
-func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
-	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
-
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
-
-	isReadOnly := uint64(0)
-	if ss.IsReadOnly {
-		isReadOnly = 1
-	}
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), isReadOnly)
-
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/inmemory"}`, ss.InmemoryActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/small"}`, ss.SmallPartActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/big"}`, ss.BigPartActiveMerges)
-
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/inmemory"}`, ss.InmemoryMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/small"}`, ss.SmallPartMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/big"}`, ss.BigPartMergesTotal)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/inmemory"}`, ss.InmemoryRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/small"}`, ss.SmallPartRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/big"}`, ss.BigPartRowsCount)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/inmemory"}`, ss.InmemoryParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/small"}`, ss.SmallParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/big"}`, ss.BigParts)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/inmemory"}`, ss.InmemoryBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/small"}`, ss.SmallPartBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/big"}`, ss.BigPartBlocks)
-
-	metrics.WriteGaugeUint64(w, `vl_partitions`, ss.PartitionsCount)
-	metrics.WriteCounterUint64(w, `vl_streams_created_total`, ss.StreamsCreatedTotal)
-
-	metrics.WriteGaugeUint64(w, `vl_indexdb_rows`, ss.IndexdbItemsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_parts`, ss.IndexdbPartsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_blocks`, ss.IndexdbBlocksCount)
-
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="indexdb"}`, ss.IndexdbSizeBytes)
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="storage"}`, ss.CompressedInmemorySize+ss.CompressedSmallPartSize+ss.CompressedBigPartSize)
-
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/inmemory"}`, ss.CompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/small"}`, ss.CompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/big"}`, ss.CompressedBigPartSize)
-
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/inmemory"}`, ss.UncompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/small"}`, ss.UncompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/big"}`, ss.UncompressedBigPartSize)
-
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
-}
-
-var activeForceMerges = metrics.NewCounter("vl_active_force_merges")

app/vlstorage/netinsert/netinsert.go

@@ -1,369 +0,0 @@
-package netinsert
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/valyala/fastrand"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-)
-
-// the maximum size of a single data block sent to storage node.
-const maxInsertBlockSize = 2 * 1024 * 1024
-
-// ProtocolVersion is the version of the data ingestion protocol.
-//
-// It must be changed every time the data encoding at /internal/insert HTTP endpoint is changed.
-const ProtocolVersion = "v1"
-
-// Storage is a network storage for sending data to remote storage nodes in the cluster.
-type Storage struct {
-	sns []*storageNode
-
-	disableCompression bool
-
-	srt *streamRowsTracker
-
-	pendingDataBuffers chan *bytesutil.ByteBuffer
-
-	stopCh chan struct{}
-	wg     sync.WaitGroup
-}
-
-type storageNode struct {
-	// scheme is http or https scheme to communicate with addr
-	scheme string
-
-	// addr is TCP address of storage node to send the ingested data to
-	addr string
-
-	// s is a storage, which holds the given storageNode
-	s *Storage
-
-	// c is an http client used for sending data blocks to addr.
-	c *http.Client
-
-	// ac is auth config used for setting request headers such as Authorization and Host.
-	ac *promauth.Config
-
-	// pendingData contains pending data, which must be sent to the storage node at the addr.
-	pendingDataMu        sync.Mutex
-	pendingData          *bytesutil.ByteBuffer
-	pendingDataLastFlush time.Time
-
-	// the unix timestamp until the storageNode is disabled for data writing.
-	disabledUntil atomic.Uint64
-}
-
-func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
-	tr := httputil.NewTransport(false, "vlinsert_backend")
-	tr.TLSHandshakeTimeout = 20 * time.Second
-	tr.DisableCompression = true
-
-	scheme := "http"
-	if isTLS {
-		scheme = "https"
-	}
-
-	sn := &storageNode{
-		scheme: scheme,
-		addr:   addr,
-		s:      s,
-		c: &http.Client{
-			Transport: ac.NewRoundTripper(tr),
-		},
-		ac: ac,
-
-		pendingData: &bytesutil.ByteBuffer{},
-	}
-
-	s.wg.Add(1)
-	go func() {
-		defer s.wg.Done()
-		sn.backgroundFlusher()
-	}()
-
-	return sn
-}
-
-func (sn *storageNode) backgroundFlusher() {
-	t := time.NewTicker(time.Second)
-	defer t.Stop()
-
-	for {
-		select {
-		case <-sn.s.stopCh:
-			return
-		case <-t.C:
-			sn.flushPendingData()
-		}
-	}
-}
-
-func (sn *storageNode) flushPendingData() {
-	sn.pendingDataMu.Lock()
-	if time.Since(sn.pendingDataLastFlush) < time.Second {
-		// nothing to flush
-		sn.pendingDataMu.Unlock()
-		return
-	}
-
-	pendingData := sn.grabPendingDataForFlushLocked()
-	sn.pendingDataMu.Unlock()
-
-	sn.mustSendInsertRequest(pendingData)
-}
-
-func (sn *storageNode) addRow(r *logstorage.InsertRow) {
-	bb := bbPool.Get()
-	b := bb.B
-
-	b = r.Marshal(b)
-
-	if len(b) > maxInsertBlockSize {
-		logger.Warnf("skipping too long log entry, since its length exceeds %d bytes; the actual log entry length is %d bytes; log entry contents: %s", maxInsertBlockSize, len(b), b)
-		return
-	}
-
-	var pendingData *bytesutil.ByteBuffer
-	sn.pendingDataMu.Lock()
-	if sn.pendingData.Len()+len(b) > maxInsertBlockSize {
-		pendingData = sn.grabPendingDataForFlushLocked()
-	}
-	sn.pendingData.MustWrite(b)
-	sn.pendingDataMu.Unlock()
-
-	bb.B = b
-	bbPool.Put(bb)
-
-	if pendingData != nil {
-		sn.mustSendInsertRequest(pendingData)
-	}
-}
-
-var bbPool bytesutil.ByteBufferPool
-
-func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
-	sn.pendingDataLastFlush = time.Now()
-	pendingData := sn.pendingData
-	sn.pendingData = <-sn.s.pendingDataBuffers
-
-	return pendingData
-}
-
-func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
-	defer func() {
-		pendingData.Reset()
-		sn.s.pendingDataBuffers <- pendingData
-	}()
-
-	err := sn.sendInsertRequest(pendingData)
-	if err == nil {
-		return
-	}
-
-	if !errors.Is(err, errTemporarilyDisabled) {
-		logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
-	}
-	for !sn.s.sendInsertRequestToAnyNode(pendingData) {
-		logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
-
-		t := timerpool.Get(time.Second)
-		select {
-		case <-sn.s.stopCh:
-			timerpool.Put(t)
-			logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
-			return
-		case <-t.C:
-			timerpool.Put(t)
-		}
-	}
-}
-
-func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
-	dataLen := pendingData.Len()
-	if dataLen == 0 {
-		// Nothing to send.
-		return nil
-	}
-
-	if sn.disabledUntil.Load() > fasttime.UnixTimestamp() {
-		return errTemporarilyDisabled
-	}
-
-	ctx, cancel := contextutil.NewStopChanContext(sn.s.stopCh)
-	defer cancel()
-
-	var body io.Reader
-	if !sn.s.disableCompression {
-		bb := zstdBufPool.Get()
-		defer zstdBufPool.Put(bb)
-
-		bb.B = zstd.CompressLevel(bb.B[:0], pendingData.B, 1)
-		body = bb.NewReader()
-	} else {
-		body = pendingData.NewReader()
-	}
-
-	reqURL := sn.getRequestURL("/internal/insert")
-	req, err := http.NewRequestWithContext(ctx, "POST", reqURL, body)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating an http request: %s", err)
-	}
-	req.Header.Set("Content-Type", "application/octet-stream")
-	if !sn.s.disableCompression {
-		req.Header.Set("Content-Encoding", "zstd")
-	}
-	if err := sn.ac.SetHeaders(req, true); err != nil {
-		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
-	}
-
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		// Disable sn for data writing for 10 seconds.
-		sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-		return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 == 2 {
-		return nil
-	}
-
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		respBody = []byte(fmt.Sprintf("%s", err))
-	}
-
-	// Disable sn for data writing for 10 seconds.
-	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-	return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
-}
-
-func (sn *storageNode) getRequestURL(path string) string {
-	return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
-}
-
-var zstdBufPool bytesutil.ByteBufferPool
-
-// NewStorage returns new Storage for the given addrs with the given authCfgs.
-//
-// The concurrency is the average number of concurrent connections per every addr.
-//
-// If disableCompression is set, then the data is sent uncompressed to the remote storage.
-//
-// Call MustStop on the returned storage when it is no longer needed.
-func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
-	pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
-	for i := 0; i < cap(pendingDataBuffers); i++ {
-		pendingDataBuffers <- &bytesutil.ByteBuffer{}
-	}
-
-	s := &Storage{
-		disableCompression: disableCompression,
-		pendingDataBuffers: pendingDataBuffers,
-		stopCh:             make(chan struct{}),
-	}
-
-	sns := make([]*storageNode, len(addrs))
-	for i, addr := range addrs {
-		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
-	}
-	s.sns = sns
-
-	s.srt = newStreamRowsTracker(len(sns))
-
-	return s
-}
-
-// MustStop stops the s.
-func (s *Storage) MustStop() {
-	close(s.stopCh)
-	s.wg.Wait()
-	s.sns = nil
-}
-
-// AddRow adds the given log row into s.
-func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
-	idx := s.srt.getNodeIdx(streamHash)
-	sn := s.sns[idx]
-	sn.addRow(r)
-}
-
-func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
-	startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
-	for i := range s.sns {
-		idx := (startIdx + i) % len(s.sns)
-		sn := s.sns[idx]
-		err := sn.sendInsertRequest(pendingData)
-		if err == nil {
-			return true
-		}
-		if !errors.Is(err, errTemporarilyDisabled) {
-			logger.Warnf("cannot send pending data to the storage node %q: %s; trying to send it to another storage node", sn.addr, err)
-		}
-	}
-	return false
-}
-
-var errTemporarilyDisabled = fmt.Errorf("writing to the node is temporarily disabled")
-
-type streamRowsTracker struct {
-	mu sync.Mutex
-
-	nodesCount    int64
-	rowsPerStream map[uint64]uint64
-}
-
-func newStreamRowsTracker(nodesCount int) *streamRowsTracker {
-	return &streamRowsTracker{
-		nodesCount:    int64(nodesCount),
-		rowsPerStream: make(map[uint64]uint64),
-	}
-}
-
-func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
-	if srt.nodesCount == 1 {
-		// Fast path for a single node.
-		return 0
-	}
-
-	srt.mu.Lock()
-	defer srt.mu.Unlock()
-
-	streamRows := srt.rowsPerStream[streamHash] + 1
-	srt.rowsPerStream[streamHash] = streamRows
-
-	if streamRows <= 1000 {
-		// Write the initial rows for the stream to a single storage node for better locality.
-		// This should work great for log streams containing small number of logs, since will be distributed
-		// evenly among available storage nodes because they have different streamHash.
-		return streamHash % uint64(srt.nodesCount)
-	}
-
-	// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
-	// in order to improve query performance over this stream (the data for the log stream
-	// can be processed in parallel on all the storage nodes).
-	//
-	// The random distribution is preferred over round-robin distribution in order to avoid possible
-	// dependency between the order of the ingested logs and the number of storage nodes,
-	// which may lead to non-uniform distribution of logs among storage nodes.
-	return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
-}

app/vlstorage/netinsert/netinsert_test.go

@@ -1,57 +0,0 @@
-package netinsert
-
-import (
-	"fmt"
-	"math"
-	"math/rand"
-	"testing"
-
-	"github.com/cespare/xxhash/v2"
-)
-
-func TestStreamRowsTracker(t *testing.T) {
-	f := func(rowsCount, streamsCount, nodesCount int) {
-		t.Helper()
-
-		// generate stream hashes
-		streamHashes := make([]uint64, streamsCount)
-		for i := range streamHashes {
-			streamHashes[i] = xxhash.Sum64([]byte(fmt.Sprintf("stream %d.", i)))
-		}
-
-		srt := newStreamRowsTracker(nodesCount)
-
-		rng := rand.New(rand.NewSource(0))
-		rowsPerNode := make([]uint64, nodesCount)
-		for i := 0; i < rowsCount; i++ {
-			streamIdx := rng.Intn(streamsCount)
-			h := streamHashes[streamIdx]
-			nodeIdx := srt.getNodeIdx(h)
-			rowsPerNode[nodeIdx]++
-		}
-
-		// Verify that rows are uniformly distributed among nodes.
-		expectedRowsPerNode := float64(rowsCount) / float64(nodesCount)
-		for nodeIdx, nodeRows := range rowsPerNode {
-			if math.Abs(float64(nodeRows)-expectedRowsPerNode)/expectedRowsPerNode > 0.15 {
-				t.Fatalf("non-uniform distribution of rows among nodes; node %d has %d rows, while it must have %v rows; rowsPerNode=%d",
-					nodeIdx, nodeRows, expectedRowsPerNode, rowsPerNode)
-			}
-		}
-	}
-
-	rowsCount := 10000
-	streamsCount := 9
-	nodesCount := 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 10000
-	streamsCount = 100
-	nodesCount = 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 100000
-	streamsCount = 1000
-	nodesCount = 9
-	f(rowsCount, streamsCount, nodesCount)
-}